<commit_before># Copyright (c) 2017, Apple Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-3-clause license that can be
# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause

import coremltools
import keras
import sklearn
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.linear_model import LinearRegression
from sklearn.svm import SVC, SVR
import unittest
import numpy as np
from sklearn.datasets import load_boston
from nose.plugins.attrib import attr

def create_model(spec):
    """
    Create MLModel with specified types
    Parameters
    ----------
    spec: Pb spec from 3rd party converted model

    Returns
    -------
    MLModel
    """
    return coremltools.models.MLModel(spec)

class TestIODataTypes(unittest.TestCase):
    """
    This class tests for different I/O feature data types for an .mlmodel
    It will cover the following areas to test for:
    - All features must have a valid type
    - Multiarrays must have a valid dataType. Inputs must specify shape. Shape must have >= 0 elements
    - Images must have a valid colorspace. width & height have to be >= 0
    - Dictionaries must have a valid key type
    """

    @property
    def scikit_data(self):
        return load_boston()

    def _feature_data_type(self, dtype):
        feature_dict = {np.int32: 'INT32', np.float32: 'FLOAT32', np.float64: 'DOUBLE'}
        return feature_dict[dtype]

    @property
    def number_data_type(self):
        return dict(
            int8=np.int8,
            int16=np.int16,
            int32=np.int32,
            uint8=np.uint8,
            uint16=np.uint16,
            uint32=np.uint32,
            float=np.float32,
            double=np.double,
        )

    def _sklearn_setup(self, model, dtype, data, target):
        model.fit(data, target)
        spec = coremltools.converters.sklearn.convert(model, 'data', 'target').get_spec()
        return model, spec

    def _check_tree_model(self, spec, inputType, outputType, n_out):
        self.assertIsNotNone(spec)

        # Test the model class
        self.assertIsNotNone(spec.description)
        self.assertIsNotNone(spec.treeEnsembleClassifier)

        # Test the interface class
        self.assertEqual(spec.description.predictedFeatureName, 'target')

        # Test the inputs and outputs
        self.assertEqual(len(spec.description.output), n_out)
        self.assertEqual(spec.description.output[0].name, 'target')
        self.assertEqual(spec.description.output[0].type.WhichOneof('Type'), outputType)
        self.assertEqual(spec.description.input[0].name, 'data')
        self.assertEqual(spec.description.input[0].type.WhichOneof('Type'), inputType)

    def test_tree_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = DecisionTreeRegressor(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'doubleType', 1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 coreml_model.predict({'data': test_data})['target'],
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     coreml_model.predict({'data': test_data})['target'],
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_random_forest_classifier(self):
        for dtype in self.number_data_type.keys():
            scikit_model = RandomForestClassifier(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype) > self.scikit_data['target'].astype(dtype).mean()
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'int64Type', 2)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 bool(int(coreml_model.predict({'data': test_data})['target'])),
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     bool(int(coreml_model.predict({'data': test_data})['target'])),
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_random_forest_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = RandomForestRegressor(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'doubleType', 1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertAlmostEqual(scikit_model.predict(test_data)[0],
                                       coreml_model.predict({'data': test_data})['target'],
                                       msg="{} != {} for Dtype: {}".format(
                                           scikit_model.predict(test_data)[0],
                                           coreml_model.predict({'data': test_data})['target'],
                                           dtype
                                       )
                                       )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_support_vector_classifier(self):
        for dtype in self.number_data_type.keys():
            scikit_model = SVC(kernel='rbf', gamma=1.2, C=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype) > self.scikit_data['target'].astype(dtype).mean()
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            coreml_model = create_model(spec)
            for idx in range(0, 10):
                test_data = data[idx].reshape(1, -1)
                try:
                    self.assertEqual(scikit_model.predict(test_data)[0],
                                     bool(int(coreml_model.predict({'data': test_data})['target'])),
                                     msg="{} != {} for Dtype: {}".format(
                                         scikit_model.predict(test_data)[0],
                                         bool(int(coreml_model.predict({'data': test_data})['target'])),
                                         dtype
                                     )
                                     )
                except RuntimeError:
                    print("{} not supported. ".format(dtype))

    def test_support_vector_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = SVR(kernel='rbf')
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 coreml_model.predict({'data': test_data})['target'],
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     coreml_model.predict({'data': test_data})['target'],
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_linear_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = LinearRegression(normalize=True)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertAlmostEqual(scikit_model.predict(test_data)[0],
                                       coreml_model.predict({'data': test_data})['target'],
                                       msg="{} != {} for Dtype: {}".format(
                                           scikit_model.predict(test_data)[0],
                                           coreml_model.predict({'data': test_data})['target'],
                                           dtype
                                       )
                                       )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    @attr('keras2')
    def test_keras_dense_model(self):
        model = keras.models.Sequential()
        model.add(keras.layers.Dense(3, activation='sigmoid', kernel_initializer='random_uniform',
                                     bias_initializer='random_uniform', input_dim=3))
        for key, dtype in self.number_data_type.iteritems():
            try:
                input_data = np.random.rand(1, 3).astype(key)
                keras_out = model.predict(input_data)
                coreml_model = coremltools.converters.keras.convert(model, input_names=['data'],
                                                                    output_names=['target'])
                spec = coreml_model.get_spec()
                spec.description.output[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                spec.description.input[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                coreml_model = coremltools.models.MLModel(spec)
                coreml_out = coreml_model.predict({'data': np.expand_dims(input_data, 0)})['target']
                self.assertEqual(dtype,
                                 coreml_out.dtype)
                if dtype != np.int32:
                    for idx in range(0, len(keras_out)):
                        self.assertAlmostEqual(keras_out[0][idx], coreml_out[idx],
                                               msg="{}\n{} != {}".format(dtype, keras_out, coreml_out), places=2)
            except KeyError:
                print("{} not supported. ".format(dtype))

    def test_keras_embedding_model(self):

        model = keras.models.Sequential()
        model.add(keras.layers.Embedding(100, 3, input_length=5, input_dtype='float32'))
        for key, dtype in self.number_data_type.iteritems():
            try:
                input_data = np.random.randint(0, 100, size=(1, 5)).astype(key)
                keras_out = np.squeeze(model.predict(input_data)).flatten()
                coreml_model = coremltools.converters.keras.convert(model, input_names=['data'],
                                                                    output_names=['target'])

                spec = coreml_model.get_spec()
                spec.description.output[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                spec.description.input[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                coreml_model = coremltools.models.MLModel(spec)
                coreml_out = np.squeeze(
                    coreml_model.predict({'data': np.expand_dims(input_data, 0).T})['target']).flatten()
                self.assertEqual(dtype,
                                 coreml_out.dtype)
                if dtype != np.int32:
                    for idx in range(0, len(keras_out)):
                        self.assertAlmostEqual(keras_out[idx], coreml_out[idx],
                                               msg="{}\n{} != {}".format(dtype, keras_out, coreml_out), places=2)
            except KeyError:
                print("{} not supported. ".format(dtype))
<commit_msg>removed iteritems()<commit_after># Copyright (c) 2017, Apple Inc. All rights reserved.
#
# Use of this source code is governed by a BSD-3-clause license that can be
# found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause

import coremltools
import keras
import sklearn
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from sklearn.linear_model import LinearRegression
from sklearn.svm import SVC, SVR
import unittest
import numpy as np
from sklearn.datasets import load_boston
from nose.plugins.attrib import attr

def create_model(spec):
    """
    Create MLModel with specified types
    Parameters
    ----------
    spec: Pb spec from 3rd party converted model

    Returns
    -------
    MLModel
    """
    return coremltools.models.MLModel(spec)

class TestIODataTypes(unittest.TestCase):
    """
    This class tests for different I/O feature data types for an .mlmodel
    It will cover the following areas to test for:
    - All features must have a valid type
    - Multiarrays must have a valid dataType. Inputs must specify shape. Shape must have >= 0 elements
    - Images must have a valid colorspace. width & height have to be >= 0
    - Dictionaries must have a valid key type
    """

    @property
    def scikit_data(self):
        return load_boston()

    def _feature_data_type(self, dtype):
        feature_dict = {np.int32: 'INT32', np.float32: 'FLOAT32', np.float64: 'DOUBLE'}
        return feature_dict[dtype]

    @property
    def number_data_type(self):
        return dict(
            int8=np.int8,
            int16=np.int16,
            int32=np.int32,
            uint8=np.uint8,
            uint16=np.uint16,
            uint32=np.uint32,
            float=np.float32,
            double=np.double,
        )

    def _sklearn_setup(self, model, dtype, data, target):
        model.fit(data, target)
        spec = coremltools.converters.sklearn.convert(model, 'data', 'target').get_spec()
        return model, spec

    def _check_tree_model(self, spec, inputType, outputType, n_out):
        self.assertIsNotNone(spec)

        # Test the model class
        self.assertIsNotNone(spec.description)
        self.assertIsNotNone(spec.treeEnsembleClassifier)

        # Test the interface class
        self.assertEqual(spec.description.predictedFeatureName, 'target')

        # Test the inputs and outputs
        self.assertEqual(len(spec.description.output), n_out)
        self.assertEqual(spec.description.output[0].name, 'target')
        self.assertEqual(spec.description.output[0].type.WhichOneof('Type'), outputType)
        self.assertEqual(spec.description.input[0].name, 'data')
        self.assertEqual(spec.description.input[0].type.WhichOneof('Type'), inputType)

    def test_tree_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = DecisionTreeRegressor(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'doubleType', 1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 coreml_model.predict({'data': test_data})['target'],
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     coreml_model.predict({'data': test_data})['target'],
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_random_forest_classifier(self):
        for dtype in self.number_data_type.keys():
            scikit_model = RandomForestClassifier(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype) > self.scikit_data['target'].astype(dtype).mean()
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'int64Type', 2)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 bool(int(coreml_model.predict({'data': test_data})['target'])),
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     bool(int(coreml_model.predict({'data': test_data})['target'])),
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_random_forest_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = RandomForestRegressor(random_state=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            self._check_tree_model(spec, 'multiArrayType', 'doubleType', 1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertAlmostEqual(scikit_model.predict(test_data)[0],
                                       coreml_model.predict({'data': test_data})['target'],
                                       msg="{} != {} for Dtype: {}".format(
                                           scikit_model.predict(test_data)[0],
                                           coreml_model.predict({'data': test_data})['target'],
                                           dtype
                                       )
                                       )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_support_vector_classifier(self):
        for dtype in self.number_data_type.keys():
            scikit_model = SVC(kernel='rbf', gamma=1.2, C=1)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype) > self.scikit_data['target'].astype(dtype).mean()
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            coreml_model = create_model(spec)
            for idx in range(0, 10):
                test_data = data[idx].reshape(1, -1)
                try:
                    self.assertEqual(scikit_model.predict(test_data)[0],
                                     bool(int(coreml_model.predict({'data': test_data})['target'])),
                                     msg="{} != {} for Dtype: {}".format(
                                         scikit_model.predict(test_data)[0],
                                         bool(int(coreml_model.predict({'data': test_data})['target'])),
                                         dtype
                                     )
                                     )
                except RuntimeError:
                    print("{} not supported. ".format(dtype))

    def test_support_vector_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = SVR(kernel='rbf')
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0],
                                 coreml_model.predict({'data': test_data})['target'],
                                 msg="{} != {} for Dtype: {}".format(
                                     scikit_model.predict(test_data)[0],
                                     coreml_model.predict({'data': test_data})['target'],
                                     dtype
                                 )
                                 )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    def test_linear_regressor(self):
        for dtype in self.number_data_type.keys():
            scikit_model = LinearRegression(normalize=True)
            data = self.scikit_data['data'].astype(dtype)
            target = self.scikit_data['target'].astype(dtype)
            scikit_model, spec = self._sklearn_setup(scikit_model, dtype, data, target)
            test_data = data[0].reshape(1, -1)
            coreml_model = create_model(spec)
            try:
                self.assertEqual(scikit_model.predict(test_data)[0].dtype,
                                 type(coreml_model.predict({'data': test_data})['target']))
                self.assertAlmostEqual(scikit_model.predict(test_data)[0],
                                       coreml_model.predict({'data': test_data})['target'],
                                       msg="{} != {} for Dtype: {}".format(
                                           scikit_model.predict(test_data)[0],
                                           coreml_model.predict({'data': test_data})['target'],
                                           dtype
                                       )
                                       )
            except RuntimeError:
                print("{} not supported. ".format(dtype))

    @attr('keras2')
    def test_keras_dense_model(self):
        model = keras.models.Sequential()
        model.add(keras.layers.Dense(3, activation='sigmoid', kernel_initializer='random_uniform',
                                     bias_initializer='random_uniform', input_dim=3))
        for key, dtype in self.number_data_type.items():
            try:
                input_data = np.random.rand(1, 3).astype(key)
                keras_out = model.predict(input_data)
                coreml_model = coremltools.converters.keras.convert(model, input_names=['data'],
                                                                    output_names=['target'])
                spec = coreml_model.get_spec()
                spec.description.output[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                spec.description.input[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                coreml_model = coremltools.models.MLModel(spec)
                coreml_out = coreml_model.predict({'data': np.expand_dims(input_data, 0)})['target']
                self.assertEqual(dtype,
                                 coreml_out.dtype)
                if dtype != np.int32:
                    for idx in range(0, len(keras_out)):
                        self.assertAlmostEqual(keras_out[0][idx], coreml_out[idx],
                                               msg="{}\n{} != {}".format(dtype, keras_out, coreml_out), places=2)
            except KeyError:
                print("{} not supported. ".format(dtype))

    def test_keras_embedding_model(self):

        model = keras.models.Sequential()
        model.add(keras.layers.Embedding(100, 3, input_length=5, input_dtype='float32'))
        for key, dtype in self.number_data_type.items():
            try:
                input_data = np.random.randint(0, 100, size=(1, 5)).astype(key)
                keras_out = np.squeeze(model.predict(input_data)).flatten()
                coreml_model = coremltools.converters.keras.convert(model, input_names=['data'],
                                                                    output_names=['target'])

                spec = coreml_model.get_spec()
                spec.description.output[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                spec.description.input[
                    0].type.multiArrayType.dataType = coremltools.proto.FeatureTypes_pb2.ArrayFeatureType.ArrayDataType.Value(
                    self._feature_data_type(dtype))
                coreml_model = coremltools.models.MLModel(spec)
                coreml_out = np.squeeze(
                    coreml_model.predict({'data': np.expand_dims(input_data, 0).T})['target']).flatten()
                self.assertEqual(dtype,
                                 coreml_out.dtype)
                if dtype != np.int32:
                    for idx in range(0, len(keras_out)):
                        self.assertAlmostEqual(keras_out[idx], coreml_out[idx],
                                               msg="{}\n{} != {}".format(dtype, keras_out, coreml_out), places=2)
            except KeyError:
                print("{} not supported. ".format(dtype))
<|endoftext|>


<commit_before>#!/usr/bin/python
from browser.linkjs import link
import optparse
import os
import shutil
import stat
import subprocess
import sys
import zipfile

class package( object ):
    root = os.path.join('bin', 'js')
    archive = os.path.join('bin', 'compiled.zip')

    @staticmethod
    def pack():
        file = zipfile.ZipFile(package.archive, 'w')
        for f in os.listdir(package.root):
            file.write(os.path.join(package.root, f), f)

    @staticmethod
    def unpack():
        cleanup(package.root)
        file = zipfile.ZipFile(package.archive, 'r')
        file.extractall(package.root)

# http://stackoverflow.com/questions/1889597/deleting-directory-in-python
def remove_readonly(fn, path, excinfo):
    if fn is os.rmdir:
        os.chmod(path, stat.S_IWRITE)
        os.rmdir(path)
    elif fn is os.remove:
        os.chmod(path, stat.S_IWRITE)
        os.remove(path)

def norm_path(path):
    return os.path.normcase(os.path.normpath(os.path.realpath(os.path.abspath(path))))

def is_parent_for(parent, child):
    parent = norm_path(parent)
    child = norm_path(child)
    while True:
        if parent == child:
            return True
        next = os.path.dirname(child)
        if next == child:
            return False
        child = next

def cleanup(dir):
    if not os.path.exists(dir):
        return

    this_dir = os.path.dirname(__file__)
    if is_parent_for(dir, this_dir):
        raise Exception("cannot delete itself: %s" % this_dir)
    shutil.rmtree(dir, onerror=remove_readonly)

def copy(src, dst_dir):
    dst = os.path.join(dst_dir, os.path.basename(src))
    if os.path.exists(dst):
        os.chmod(dst, stat.S_IWRITE)
    print('%s -> %s' % (src, dst))
    shutil.copy(src, dst)

def copytree(src, dst):
    cleanup(dst)
    print('%s -> %s' % (src, dst))
    shutil.copytree(src, dst)

def run(cmd, env=None, cwd=None, print_output=False):
    p = subprocess.Popen(
        cmd,
        stdout=None if print_output else subprocess.PIPE,
        stderr=subprocess.STDOUT,
        #shell = True,
        env=env,
        cwd=cwd)

    result = None if print_output else p.stdout.read().decode()
    rc = p.wait()
    if rc:
        if result:
            print(result)
        print('"%s" failed with exit code %d' % (' '.join(cmd), rc))
        exit(rc)
    return result

root = os.path.dirname(os.path.abspath(__file__))

def make_js_search_dirs(bin):
    return [os.path.join(root, 'test'),
            os.path.join(root, 'src'),
            bin];

def run_node(args, js_search_dirs, cwd=None):
    node_exe, path_separator = ('node.exe', ';') if os.name == 'nt' else ('node', ':')
    node_env = dict(list(os.environ.items())
                  + [('NODE_PATH', path_separator.join(js_search_dirs))])

    run([node_exe] + args, node_env, cwd, print_output=True)

def run_tests(bin):
    print('run tests using "%s" ->' % bin)
    js_search_dirs = make_js_search_dirs(bin)

    unit_tests = os.path.join(root, 'test', 'test_unit.js')
    run_node([unit_tests], js_search_dirs)

    compile_tests = os.path.join(root, 'test', 'test_compile.js')
    run_node([compile_tests], js_search_dirs, cwd=os.path.join(root, 'test'))
    print('<-tests')

def recompile(bin):
    print('recompile oberon sources using "%s"...' % bin)
    compiler = os.path.join(root, 'src', 'oc_nodejs.js')
    sources = ['EberonSymbols.ob', 'EberonCast.ob', 'EberonOperator.ob',
               'OberonSymbols.ob', 'Lexer.ob', 'Module.ob']

    result = os.path.join(root, 'bin.recompile')
    cleanup(result)
    os.mkdir(result)
    out = os.path.join(result, 'js')
    os.mkdir(out)

    run_node([compiler,
              '--include=src/ob;src/eberon;src/oberon',
              '--out-dir=%s' % out,
              '--import-dir=js',
              '--timing=true']
              + sources,
             make_js_search_dirs(bin))
    return result

def compile_using_snapshot(src):
    snapshot_root = os.path.join(root, 'snapshot')
    out = os.path.join(root, 'bin', 'js')
    compiler = os.path.join(snapshot_root, 'oc_nodejs.js')
    js_search_dirs = [snapshot_root]
    run_node([  compiler,
                '--include=src/ob;src/eberon;src/oberon',
                '--out-dir=%s' % out,
                '--import-dir=js',
                src],
             js_search_dirs)

def build_html(options):
    version = None
    if options.set_version:
        print(run(['git', 'pull']))
        version = run(['git', 'log', '-1', '--format="%ci%n%H"'])

    out = options.out
    build_version = None
    build_version_path = os.path.join(out, 'version.txt')
    try:
        with open(build_version_path) as f:
            build_version = f.read()
    except:
        pass

    if (not build_version is None) and build_version == version:
        print("current html is up to date, do nothing")
        return

    print('unpacking compiled js to %s...' % package.root)
    package.unpack()

    if not os.path.exists(out):
        os.mkdir(out)

    link(['oc.js', 'oberon/oberon_grammar.js', 'eberon/eberon_grammar.js'],
         os.path.join(out, 'oc.js'),
         ['src', 'bin'],
         version)
    copy('browser/oberonjs.html', out)
    for d in ['codemirror', 'jslibs']:
        copytree(os.path.join('browser', d), os.path.join(out, d))

    if version is None:
        if os.path.exists(build_version_path):
            os.remove(build_version_path)
    else:
        with open(build_version_path, 'w') as f:
            f.write(version)

def pre_commit_check(options):
    bin = os.path.join(root, 'bin')
    run_tests(bin)
    recompiled = recompile(bin)
    run_tests(recompiled)

    print('%s -> %s' % (recompiled, bin))
    cleanup(bin)
    os.rename(recompiled, bin)

    print('packaging compiled js to %s...' % package.root)
    package.pack()

class compile_target(object):
    name = 'compile'
    description = 'compile oberon source file using the snapshot'

    @staticmethod
    def setup_options(parser):
        parser.add_option('--file', help='file to compile')

    def __init__(self, options):
        compile_using_snapshot(options.file)

class html_target(object):
    name = 'html'
    description = 'build html page'

    @staticmethod
    def setup_options(parser):
        parser.add_option('--out', help='output directory, default: "_out"', default='_out')
        parser.add_option('--set-version', action="store_true", help='include version in built html')

    def __init__(self, options):
        build_html(options)

class pre_commit_target(object):
    name = 'pre-commit'
    description = 'run tests, recompile oberon sources, run tests against just recompiled sources, pack compiled sources and build html'

    @staticmethod
    def setup_options(parser):
        pass

    def __init__(self, options):
        pre_commit_check(options)

targets = [compile_target, html_target, pre_commit_target]

def build(target, options):
    targets[target](options)

if __name__ == '__main__':
    description = 'Targets: %s' % '|'.join([t.name for t in targets])
    parser = optparse.OptionParser(
        description=description,
        usage='%prog [options] <target>'
        )
    for t in targets:
        group = optparse.OptionGroup(parser, 'target "%s"' % t.name, t.description)
        t.setup_options(group)
        parser.add_option_group(group)

    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.print_help();
        exit(-1)

    target_name = args[0]
    target = None
    for t in targets:
        if t.name == target_name:
            target = t
            break

    if target is None:
        print('uknown target: "%s"' % target_name)
        exit(-1)

    target(options)<commit_msg>run tests using build.py<commit_after>#!/usr/bin/python
from browser.linkjs import link
import optparse
import os
import shutil
import stat
import subprocess
import sys
import zipfile

class package( object ):
    root = os.path.join('bin', 'js')
    archive = os.path.join('bin', 'compiled.zip')

    @staticmethod
    def pack():
        file = zipfile.ZipFile(package.archive, 'w')
        for f in os.listdir(package.root):
            file.write(os.path.join(package.root, f), f)

    @staticmethod
    def unpack():
        cleanup(package.root)
        file = zipfile.ZipFile(package.archive, 'r')
        file.extractall(package.root)

# http://stackoverflow.com/questions/1889597/deleting-directory-in-python
def remove_readonly(fn, path, excinfo):
    if fn is os.rmdir:
        os.chmod(path, stat.S_IWRITE)
        os.rmdir(path)
    elif fn is os.remove:
        os.chmod(path, stat.S_IWRITE)
        os.remove(path)

def norm_path(path):
    return os.path.normcase(os.path.normpath(os.path.realpath(os.path.abspath(path))))

def is_parent_for(parent, child):
    parent = norm_path(parent)
    child = norm_path(child)
    while True:
        if parent == child:
            return True
        next = os.path.dirname(child)
        if next == child:
            return False
        child = next

def cleanup(dir):
    if not os.path.exists(dir):
        return

    this_dir = os.path.dirname(__file__)
    if is_parent_for(dir, this_dir):
        raise Exception("cannot delete itself: %s" % this_dir)
    shutil.rmtree(dir, onerror=remove_readonly)

def copy(src, dst_dir):
    dst = os.path.join(dst_dir, os.path.basename(src))
    if os.path.exists(dst):
        os.chmod(dst, stat.S_IWRITE)
    print('%s -> %s' % (src, dst))
    shutil.copy(src, dst)

def copytree(src, dst):
    cleanup(dst)
    print('%s -> %s' % (src, dst))
    shutil.copytree(src, dst)

def run(cmd, env=None, cwd=None, print_output=False):
    p = subprocess.Popen(
        cmd,
        stdout=None if print_output else subprocess.PIPE,
        stderr=subprocess.STDOUT,
        #shell = True,
        env=env,
        cwd=cwd)

    result = None if print_output else p.stdout.read().decode()
    rc = p.wait()
    if rc:
        if result:
            print(result)
        print('"%s" failed with exit code %d' % (' '.join(cmd), rc))
        exit(rc)
    return result

root = os.path.dirname(os.path.abspath(__file__))

def make_js_search_dirs(bin):
    return [os.path.join(root, 'test'),
            os.path.join(root, 'src'),
            bin];

def run_node(args, js_search_dirs, cwd=None):
    node_exe, path_separator = ('node.exe', ';') if os.name == 'nt' else ('node', ':')
    node_env = dict(list(os.environ.items())
                  + [('NODE_PATH', path_separator.join(js_search_dirs))])

    run([node_exe] + args, node_env, cwd, print_output=True)

def run_tests(bin, unit_test=None, code_test=None):
    print('run tests using "%s" ->' % bin)
    js_search_dirs = make_js_search_dirs(bin)

    if not code_test:
        unit_tests = os.path.join(root, 'test', 'test_unit.js')
        args = [unit_tests]
        if unit_test:
            args += [unit_test]
        run_node(args, js_search_dirs)

    if not unit_test:
        compile_tests = os.path.join(root, 'test', 'test_compile.js')
        args = [compile_tests]
        if code_test:
            args += [code_test]
        run_node(args, js_search_dirs, cwd=os.path.join(root, 'test'))
    print('<-tests')

def recompile(bin):
    print('recompile oberon sources using "%s"...' % bin)
    compiler = os.path.join(root, 'src', 'oc_nodejs.js')
    sources = ['EberonSymbols.ob', 'EberonCast.ob', 'EberonOperator.ob',
               'OberonSymbols.ob', 'Lexer.ob', 'Module.ob']

    result = os.path.join(root, 'bin.recompile')
    cleanup(result)
    os.mkdir(result)
    out = os.path.join(result, 'js')
    os.mkdir(out)

    run_node([compiler,
              '--include=src/ob;src/eberon;src/oberon',
              '--out-dir=%s' % out,
              '--import-dir=js',
              '--timing=true']
              + sources,
             make_js_search_dirs(bin))
    return result

def compile_using_snapshot(src):
    snapshot_root = os.path.join(root, 'snapshot')
    out = os.path.join(root, 'bin', 'js')
    compiler = os.path.join(snapshot_root, 'oc_nodejs.js')
    js_search_dirs = [snapshot_root]
    run_node([  compiler,
                '--include=src/ob;src/eberon;src/oberon',
                '--out-dir=%s' % out,
                '--import-dir=js',
                src],
             js_search_dirs)

def build_html(options):
    version = None
    if options.set_version:
        print(run(['git', 'pull']))
        version = run(['git', 'log', '-1', '--format="%ci%n%H"'])

    out = options.out
    build_version = None
    build_version_path = os.path.join(out, 'version.txt')
    try:
        with open(build_version_path) as f:
            build_version = f.read()
    except:
        pass

    if (not build_version is None) and build_version == version:
        print("current html is up to date, do nothing")
        return

    print('unpacking compiled js to %s...' % package.root)
    package.unpack()

    if not os.path.exists(out):
        os.mkdir(out)

    link(['oc.js', 'oberon/oberon_grammar.js', 'eberon/eberon_grammar.js'],
         os.path.join(out, 'oc.js'),
         ['src', 'bin'],
         version)
    copy('browser/oberonjs.html', out)
    for d in ['codemirror', 'jslibs']:
        copytree(os.path.join('browser', d), os.path.join(out, d))

    if version is None:
        if os.path.exists(build_version_path):
            os.remove(build_version_path)
    else:
        with open(build_version_path, 'w') as f:
            f.write(version)

def pre_commit_check(options):
    bin = os.path.join(root, 'bin')
    run_tests(bin)
    recompiled = recompile(bin)
    run_tests(recompiled)

    print('%s -> %s' % (recompiled, bin))
    cleanup(bin)
    os.rename(recompiled, bin)

    print('packaging compiled js to %s...' % package.root)
    package.pack()

class compile_target(object):
    name = 'compile'
    description = 'compile oberon source file using the snapshot'

    @staticmethod
    def setup_options(parser):
        parser.add_option('--file', help='file to compile')

    def __init__(self, options):
        compile_using_snapshot(options.file)

class html_target(object):
    name = 'html'
    description = 'build html page'

    @staticmethod
    def setup_options(parser):
        parser.add_option('--out', help='output directory, default: "_out"', default='_out')
        parser.add_option('--set-version', action="store_true", help='include version in built html')

    def __init__(self, options):
        build_html(options)

class tests_target(object):
    name = 'tests'
    description = 'run tests'

    @staticmethod
    def setup_options(parser):
        parser.add_option('--unit', help='run specific unit test')
        parser.add_option('--code', help='run specific code generator test')

    def __init__(self, options):
        bin = os.path.join(root, 'bin')
        run_tests(bin, options.unit, options.code)

class pre_commit_target(object):
    name = 'pre-commit'
    description = 'run tests, recompile oberon sources, run tests against just recompiled sources, pack compiled sources and build html'

    @staticmethod
    def setup_options(parser):
        pass

    def __init__(self, options):
        pre_commit_check(options)

targets = [compile_target, html_target, tests_target, pre_commit_target]

def build(target, options):
    targets[target](options)

if __name__ == '__main__':
    description = 'Targets: %s' % '|'.join([t.name for t in targets])
    parser = optparse.OptionParser(
        description=description,
        usage='%prog [options] <target>'
        )
    for t in targets:
        group = optparse.OptionGroup(parser, 'target "%s"' % t.name, t.description)
        t.setup_options(group)
        parser.add_option_group(group)

    (options, args) = parser.parse_args()
    if len(args) != 1:
        parser.print_help();
        exit(-1)

    target_name = args[0]
    target = None
    for t in targets:
        if t.name == target_name:
            target = t
            break

    if target is None:
        print('uknown target: "%s"' % target_name)
        exit(-1)

    target(options)<|endoftext|>


<commit_before>from __future__ import print_function
import xml.etree.ElementTree as ET

userAgent = "pyudl"

class TDSCatalog():

    def __init__(self, catalogUrl):
        # top level server url
        self.catalogUrl = catalogUrl
        self.base_tds_url = catalogUrl.split('/thredds/')[0]
        # get catalog.xml file
        xml_data = basic_http_request(catalogUrl, return_response=True)
        # begin parsing the xml doc
        tree = ET.parse(xml_data)
        root = tree.getroot()
        if "name" in root.attrib:
            self.catalog_name = root.attrib["name"]
        else:
            self.catalog_name = "No name found"

        self.datasets = {}
        self.services = []
        self.catalogRefs = {}
        for child in root.iter():
            tagType = child.tag.split('}')[-1]

            if tagType == "service":
                if child.attrib["serviceType"] != "Compound":
                    self.services.append(SimpleService(child))
            elif tagType == "dataset":
                if "urlPath" in child.attrib:
                    if child.attrib["urlPath"] == "latest.xml":
                        ds = Dataset(child, catalogUrl)
                    else:
                        ds = Dataset(child)
                    self.datasets[ds.name] = ds
            elif tagType == "catalogRef":
                catalogRef = CatalogRef(child)
                self.catalogRefs[catalogRef.title] = catalogRef

        self.numberOfDatasets = len(self.datasets.keys())
        for dsName in self.datasets.keys():
            self.datasets[dsName].makeAccessUrls(
                self.base_tds_url, self.services)

class CatalogRef():

    def __init__(self, elementNode):
        self.name = elementNode.attrib["name"]
        self.href = elementNode.attrib["{http://www.w3.org/1999/xlink}href"]
        if self.href[0] == '/':
            self.href = self.href[1:]
        self.title = elementNode.attrib["{http://www.w3.org/1999/xlink}title"]

class Dataset():

    def __init__(self, elementNode, catalogUrl=""):
        self.name = elementNode.attrib['name']
        self.urlPath = elementNode.attrib['urlPath']
        self.resolved = False
        self.resolverUrl = None
        # if latest.xml, resolve the latest url
        if self.urlPath == "latest.xml":
            if catalogUrl != "":
                self.resolved = True
                self.resolverUrl = self.urlPath
                self.urlPath = self.resolveUrl(catalogUrl)
            else:
                print("Must pass along the catalog URL to resolve the "
                      "latest.xml dataset!")

    def resolveUrl(self, catalogUrl):
        if catalogUrl != "":
            resolverBase = catalogUrl.split("catalog.xml")[0]
            resolverUrl = resolverBase + self.urlPath
            resolverXml = basic_http_request(resolverUrl, return_response=True)
            tree = ET.parse(resolverXml)
            root = tree.getroot()
            if "name" in root.attrib:
                self.catalog_name = root.attrib["name"]
            else:
                self.catalog_name = "No name found"

            found = False
            for child in root.iter():
                if not found:
                    tagType = child.tag.split('}')[-1]
                    if tagType == "dataset":
                        if "urlPath" in child.attrib:
                            ds = Dataset(child)
                            resolvedUrl = ds.urlPath
                            found = True
            if found:
                return resolvedUrl
            else:
                print("no dataset url path found in latest.xml!")

    def makeAccessUrls(self, catalogUrl, services):
        accessUrls = {}
        serverUrl = catalogUrl.split('/thredds/')[0]
        for service in services:
            if service.serviceType != 'Resolver':
                accessUrls[service.serviceType] = serverUrl + \
                    service.base + self.urlPath

        self.accessUrls = accessUrls

class SimpleService():

    def __init__(self, serviceNode):
        self.name = serviceNode.attrib['name']
        self.serviceType = serviceNode.attrib['serviceType']
        self.base = serviceNode.attrib['base']

class CompoundService():

    def __init__(self, serviceNode):
        self.name = serviceNode.attrib['name']
        self.serviceType = serviceNode.attrib['serviceType']
        self.base = serviceNode.attrib['base']
        services = []
        for child in list(serviceNode):
            services.append(SimpleService(child))

        self.services = services

def basic_http_request(full_url, return_response=False):
    try:
        from urllib2 import urlopen, Request
    except ImportError:
        from urllib.request import urlopen, Request

    url_request = Request(full_url)
    url_request.add_header('User-agent', userAgent)
    try:
        response = urlopen(url_request)
        if return_response:
            return response
        else:
            del response
    except IOError as e:
        if hasattr(e, 'reason'):
            print('We failed to reach a server.')
            print('Reason: {}'.format(e.reason))
            print('Full  url: {}'.format(full_url))
            raise
        elif hasattr(e, 'code'):
            print('The server couldn\'t fulfill the request.')
            print('Error code: {}'.format(e.code))
            print('TDS response: {}'.format(e.read()))
            print('Full  url: {}'.format(full_url))
            raise
        else:
            print('error not caught!')
            raise

def get_latest_cat(cat):
    for service in cat.services:
        if (service.name.lower() == "latest" and
                service.serviceType.lower() == "resolver"):
            return TDSCatalog(cat.catalogUrl.replace("catalog.xml", "latest.xml"))

    print("ERROR: Latest Dataset Not Found!")

def get_latest_access_url(catalog, access_method):
    """
    Get the data access url, using a specified method, to the latest data
    available from a top level dataset TDSCatalog object.

    Parameters
    ----------
    catalog : TDSCatalog
        Instance of TDSCatalog - the THREDDS Data Server Catalog Object. The
        URL of the top level data catalog from a dataset should be used to
        create the object.

    access_method : String
        desired data access method (i.e. OPENDAP)

    Returns
    -------
    String
        String representation of the URL to be used to access the latest
        data available from a given catalog

    """

    latestCat = get_latest_cat(catalog)
    if latestCat != "":
        if len(latestCat.datasets.keys()) == 1:
            latestDs = latestCat.datasets[latestCat.datasets.keys()[0]]
            return latestDs.accessUrls[access_method]
        else:
            print('ERROR: More than one access url matching the requested '
                  'access method...clearly this is an error')
<commit_msg>Make classes inherit from object.<commit_after>from __future__ import print_function
import xml.etree.ElementTree as ET

userAgent = "pyudl"

class TDSCatalog(object):

    def __init__(self, catalogUrl):
        # top level server url
        self.catalogUrl = catalogUrl
        self.base_tds_url = catalogUrl.split('/thredds/')[0]
        # get catalog.xml file
        xml_data = basic_http_request(catalogUrl, return_response=True)
        # begin parsing the xml doc
        tree = ET.parse(xml_data)
        root = tree.getroot()
        if "name" in root.attrib:
            self.catalog_name = root.attrib["name"]
        else:
            self.catalog_name = "No name found"

        self.datasets = {}
        self.services = []
        self.catalogRefs = {}
        for child in root.iter():
            tagType = child.tag.split('}')[-1]

            if tagType == "service":
                if child.attrib["serviceType"] != "Compound":
                    self.services.append(SimpleService(child))
            elif tagType == "dataset":
                if "urlPath" in child.attrib:
                    if child.attrib["urlPath"] == "latest.xml":
                        ds = Dataset(child, catalogUrl)
                    else:
                        ds = Dataset(child)
                    self.datasets[ds.name] = ds
            elif tagType == "catalogRef":
                catalogRef = CatalogRef(child)
                self.catalogRefs[catalogRef.title] = catalogRef

        self.numberOfDatasets = len(self.datasets.keys())
        for dsName in self.datasets.keys():
            self.datasets[dsName].makeAccessUrls(
                self.base_tds_url, self.services)

class CatalogRef(object):

    def __init__(self, elementNode):
        self.name = elementNode.attrib["name"]
        self.href = elementNode.attrib["{http://www.w3.org/1999/xlink}href"]
        if self.href[0] == '/':
            self.href = self.href[1:]
        self.title = elementNode.attrib["{http://www.w3.org/1999/xlink}title"]

class Dataset(object):

    def __init__(self, elementNode, catalogUrl=""):
        self.name = elementNode.attrib['name']
        self.urlPath = elementNode.attrib['urlPath']
        self.resolved = False
        self.resolverUrl = None
        # if latest.xml, resolve the latest url
        if self.urlPath == "latest.xml":
            if catalogUrl != "":
                self.resolved = True
                self.resolverUrl = self.urlPath
                self.urlPath = self.resolveUrl(catalogUrl)
            else:
                print("Must pass along the catalog URL to resolve the "
                      "latest.xml dataset!")

    def resolveUrl(self, catalogUrl):
        if catalogUrl != "":
            resolverBase = catalogUrl.split("catalog.xml")[0]
            resolverUrl = resolverBase + self.urlPath
            resolverXml = basic_http_request(resolverUrl, return_response=True)
            tree = ET.parse(resolverXml)
            root = tree.getroot()
            if "name" in root.attrib:
                self.catalog_name = root.attrib["name"]
            else:
                self.catalog_name = "No name found"

            found = False
            for child in root.iter():
                if not found:
                    tagType = child.tag.split('}')[-1]
                    if tagType == "dataset":
                        if "urlPath" in child.attrib:
                            ds = Dataset(child)
                            resolvedUrl = ds.urlPath
                            found = True
            if found:
                return resolvedUrl
            else:
                print("no dataset url path found in latest.xml!")

    def makeAccessUrls(self, catalogUrl, services):
        accessUrls = {}
        serverUrl = catalogUrl.split('/thredds/')[0]
        for service in services:
            if service.serviceType != 'Resolver':
                accessUrls[service.serviceType] = serverUrl + \
                    service.base + self.urlPath

        self.accessUrls = accessUrls

class SimpleService(object):

    def __init__(self, serviceNode):
        self.name = serviceNode.attrib['name']
        self.serviceType = serviceNode.attrib['serviceType']
        self.base = serviceNode.attrib['base']

class CompoundService(object):

    def __init__(self, serviceNode):
        self.name = serviceNode.attrib['name']
        self.serviceType = serviceNode.attrib['serviceType']
        self.base = serviceNode.attrib['base']
        services = []
        for child in list(serviceNode):
            services.append(SimpleService(child))

        self.services = services

def basic_http_request(full_url, return_response=False):
    try:
        from urllib2 import urlopen, Request
    except ImportError:
        from urllib.request import urlopen, Request

    url_request = Request(full_url)
    url_request.add_header('User-agent', userAgent)
    try:
        response = urlopen(url_request)
        if return_response:
            return response
        else:
            del response
    except IOError as e:
        if hasattr(e, 'reason'):
            print('We failed to reach a server.')
            print('Reason: {}'.format(e.reason))
            print('Full  url: {}'.format(full_url))
            raise
        elif hasattr(e, 'code'):
            print('The server couldn\'t fulfill the request.')
            print('Error code: {}'.format(e.code))
            print('TDS response: {}'.format(e.read()))
            print('Full  url: {}'.format(full_url))
            raise
        else:
            print('error not caught!')
            raise

def get_latest_cat(cat):
    for service in cat.services:
        if (service.name.lower() == "latest" and
                service.serviceType.lower() == "resolver"):
            return TDSCatalog(cat.catalogUrl.replace("catalog.xml", "latest.xml"))

    print("ERROR: Latest Dataset Not Found!")

def get_latest_access_url(catalog, access_method):
    """
    Get the data access url, using a specified method, to the latest data
    available from a top level dataset TDSCatalog object.

    Parameters
    ----------
    catalog : TDSCatalog
        Instance of TDSCatalog - the THREDDS Data Server Catalog Object. The
        URL of the top level data catalog from a dataset should be used to
        create the object.

    access_method : String
        desired data access method (i.e. OPENDAP)

    Returns
    -------
    String
        String representation of the URL to be used to access the latest
        data available from a given catalog

    """

    latestCat = get_latest_cat(catalog)
    if latestCat != "":
        if len(latestCat.datasets.keys()) == 1:
            latestDs = latestCat.datasets[latestCat.datasets.keys()[0]]
            return latestDs.accessUrls[access_method]
        else:
            print('ERROR: More than one access url matching the requested '
                  'access method...clearly this is an error')
<|endoftext|>


<commit_before>#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import sqlite3
import diff

class Database():
    def __init__(self,bitHopper):
        self.curs = None
        self.bitHopper = bitHopper
        self.pool = bitHopper.pool
        self.check_database()

    def check_database(self):
        self.bitHopper.log_msg('Checking Database')
        database = sqlite3.connect('stats.db')
        self.curs = database.cursor()

        for server_name in self.pool.get_servers():
            sql = "CREATE TABLE IF NOT EXISTS "+server_name +" (diff REAL, shares INTEGER, stored_payout REAL)"
            self.curs.execute(sql)

        for server in self.pool.get_servers():
            sql = 'select * from ' + server +' where diff = ' + str(diff.difficulty)
            rows = self.curs.execute(sql)
            rows = rows.fetchall()
            if len(rows) == 0:
                sql = 'INSERT INTO ' + server + '(diff,shares,stored_payout) values( '+str(diff.difficulty) +', '+str(0) + ', '+str(0)+ ')'
                self.curs.execute(sql)
        self.bitHopper.log_msg('Database Setup')

    def update_shares(self,server,shares):

        sql = 'UPDATE '+ server +'Set shares= shares + '+ shares +' WHERE diff='+ diff.difficulty
        self.curs.execute(sql)

    def get_shares(self,server):
        sql = 'select shares from ' + server
        self.curs.execute(sql)
        shares = 0
        for info in self.curs.fetchall():
            shares += int(info[0])
        return shares

    def update_payout(self,server,payout):

        sql = 'UPDATE '+ server +'Set stored_payout= stored_payout + '+ payout +' WHERE diff='+ diff.difficulty
        self.curs.execute(sql)

    def get_payout(self,server):
        sql = 'select stored_payout from ' + server
        self.curs.execute(sql)
        payout = 0
        for info in self.curs.fetchall():
            payout += float(info[0])
        return payout

<commit_msg>Database casting issue fix<commit_after>#License#
#bitHopper by Colin Rice is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License.
#Based on a work at github.com.

import sqlite3
import diff

class Database():
    def __init__(self,bitHopper):
        self.curs = None
        self.bitHopper = bitHopper
        self.pool = bitHopper.pool
        self.check_database()

    def check_database(self):
        self.bitHopper.log_msg('Checking Database')
        database = sqlite3.connect('stats.db')
        self.curs = database.cursor()

        for server_name in self.pool.get_servers():
            sql = "CREATE TABLE IF NOT EXISTS "+server_name +" (diff REAL, shares INTEGER, stored_payout REAL)"
            self.curs.execute(sql)

        for server in self.pool.get_servers():
            sql = 'select * from ' + server +' where diff = ' + str(diff.difficulty)
            rows = self.curs.execute(sql)
            rows = rows.fetchall()
            if len(rows) == 0:
                sql = 'INSERT INTO ' + server + '(diff,shares,stored_payout) values( '+str(diff.difficulty) +', '+str(0) + ', '+str(0)+ ')'
                self.curs.execute(sql)
        self.bitHopper.log_msg('Database Setup')

    def update_shares(self,server,shares):

        sql = 'UPDATE '+ server +'Set shares= shares + '+ str(shares) +' WHERE diff='+ diff.difficulty
        self.curs.execute(sql)

    def get_shares(self,server):
        sql = 'select shares from ' + server
        self.curs.execute(sql)
        shares = 0
        for info in self.curs.fetchall():
            shares += int(info[0])
        return shares

    def update_payout(self,server,payout):

        sql = 'UPDATE '+ server +'Set stored_payout= stored_payout + '+ str(payout) +' WHERE diff='+ diff.difficulty
        self.curs.execute(sql)

    def get_payout(self,server):
        sql = 'select stored_payout from ' + server
        self.curs.execute(sql)
        payout = 0
        for info in self.curs.fetchall():
            payout += float(info[0])
        return payout

<|endoftext|>


<commit_before>from notebook.utils import url_path_join
from notebook.base.handlers import IPythonHandler
from nbconvert.exporters.export import *
import base64
import json
import os.path
import requests

# This handler will save out the notebook to GitHub gists in either a new Gist
# or it will create a new revision for a gist that already contains these two files.
class GistHandler(IPythonHandler):
    client_id = None
    client_secret = None
    def get(self):
        print("Extracting code . . .")
        args = self.request.arguments
        access_code = args["code"][0].decode('ascii')
        response = requests.post("https://github.com/login/oauth/access_token",
            data = {
                "client_id": self.client_id,
                "client_secret" : self.client_secret,
                "code" : access_code
            },
            headers = {"Accept" : "application/json"})

        nb_path = base64.b64decode(args["nb_path"][0]).decode('utf-8').lstrip("/")

        args = json.loads(response.text)
        print(args)
        print("Building request. . .")

        # TODO: change these to .get to prevent exceptions
        access_token = args["access_token"]
        token_type = args["token_type"]
        scope = args["scope"]

        githubHeaders = { "Accept" : "application/json",
                            "Authorization" : "token " + access_token }

        print("Extracting file contents")
        filename = os.path.basename(nb_path)
        ext_start_ind = filename.rfind(".")
        if ext_start_ind == -1:
            filename_no_ext = filename
        else:
            filename_no_ext = filename[:ext_start_ind]
        notebook_output, _ = export_by_name("notebook", nb_path)
        python_output, _ = export_by_name("python", nb_path)

        pyFiles = {
            "description": filename_no_ext,
            "public": False,
            "files": {
                filename : {"filename" : filename, "content": notebook_output},
                filename_no_ext + ".py" : { "filename" : filename_no_ext + ".py",
                                "content": python_output }
            }
        }

        # Get the list of user's gists to see if this gist exists already
        response = requests.get("https://api.github.com/gists",
            headers = githubHeaders)
        args = json.loads(response.text)

        filenameWithPy = filename_no_ext + ".py"
        matchCounter = 0;
        matchID = None

        for gist in args:

            gistContents = gist["files"]
            if filename in gistContents and filenameWithPy in gistContents:
                matchCounter += 1

                if "id" in gist:
                    matchID = gist["id"]

        if matchID == None:

            print("Saving gist. . .")
            # TODO: Validate the token
            response = requests.post("https://api.github.com/gists",
                data = json.dumps(pyFiles),
                headers = githubHeaders)

        # If we have another gist with the same files, create a new revision
        elif matchCounter == 1:
            # create new rev for existing gist
            print ("Revising gist with id " + matchID)

            response = requests.patch("https://api.github.com/gists/" + matchID,
                data = json.dumps(pyFiles),
                headers = githubHeaders)

        # TODO: Some sort of error message if we have more than 1 gist with the same files
        else:
            print("OOPS")
            # put stuff here

        print("Redirecting...")
        self.redirect(response.json()["html_url"])

        print("All done. . .")

def load_jupyter_server_extension(nb_server_app):
    """
    Called when the extension is loaded.

    Args:
        nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance.
    """
    # Extract our gist client details from the config:
    cfg = nb_server_app.config["NotebookApp"]
    GistHandler.client_id = cfg["oauth_client_id"]
    GistHandler.client_secret = cfg["oauth_client_secret"]

    web_app = nb_server_app.web_app
    host_pattern = '.*$'
    route_pattern = url_path_join(web_app.settings['base_url'], '/create_gist')
    web_app.add_handlers(host_pattern, [(route_pattern, GistHandler)])
<commit_msg>Changing all camelCash to pothole variables.<commit_after>from notebook.utils import url_path_join
from notebook.base.handlers import IPythonHandler
from nbconvert.exporters.export import *
import base64
import json
import os.path
import requests

# This handler will save out the notebook to GitHub gists in either a new Gist
# or it will create a new revision for a gist that already contains these two files.
class GistHandler(IPythonHandler):
    client_id = None
    client_secret = None
    def get(self):
        print("Extracting code . . .")
        args = self.request.arguments
        access_code = args["code"][0].decode('ascii')
        response = requests.post("https://github.com/login/oauth/access_token",
            data = {
                "client_id": self.client_id,
                "client_secret" : self.client_secret,
                "code" : access_code
            },
            headers = {"Accept" : "application/json"})

        nb_path = base64.b64decode(args["nb_path"][0]).decode('utf-8').lstrip("/")

        args = json.loads(response.text)
        print(args)
        print("Building request. . .")

        # TODO: change these to .get to prevent exceptions
        access_token = args["access_token"]
        token_type = args["token_type"]
        scope = args["scope"]

        github_headers = { "Accept" : "application/json",
                            "Authorization" : "token " + access_token }

        print("Extracting file contents")
        filename = os.path.basename(nb_path)
        ext_start_ind = filename.rfind(".")
        if ext_start_ind == -1:
            filename_no_ext = filename
        else:
            filename_no_ext = filename[:ext_start_ind]
        notebook_output, _ = export_by_name("notebook", nb_path)
        python_output, _ = export_by_name("python", nb_path)

        filename_with_py = filename_no_ext + ".py"
        py_files = {
            "description": filename_no_ext,
            "public": False,
            "files": {
                filename : {"filename" : filename, "content": notebook_output},
                filename_with_py : { "filename" : filename_with_py,
                                "content": python_output }
            }
        }

        # Get the list of user's gists to see if this gist exists already
        response = requests.get("https://api.github.com/gists",
            headers = github_headers)
        args = json.loads(response.text)

        match_counter = 0;
        matchID = None

        for gist in args:

            gist_contents = gist["files"]
            if filename in gist_contents and filename_with_py in gist_contents:
                match_counter += 1

                if "id" in gist:
                    matchID = gist["id"]

        if matchID == None:

            print("Saving gist. . .")
            # TODO: Validate the token
            response = requests.post("https://api.github.com/gists",
                data = json.dumps(py_files),
                headers = github_headers)

        # If we have another gist with the same files, create a new revision
        elif match_counter == 1:
            # create new rev for existing gist
            print ("Revising gist with id " + matchID)

            response = requests.patch("https://api.github.com/gists/" + matchID,
                data = json.dumps(py_files),
                headers = github_headers)

        # TODO: Some sort of error message if we have more than 1 gist with the same files
        else:
            print("OOPS")
            # put stuff here

        print("Redirecting...")
        self.redirect(response.json()["html_url"])

        print("All done. . .")

def load_jupyter_server_extension(nb_server_app):
    """
    Called when the extension is loaded.

    Args:
        nb_server_app (NotebookWebApplication): handle to the Notebook webserver instance.
    """
    # Extract our gist client details from the config:
    cfg = nb_server_app.config["NotebookApp"]
    GistHandler.client_id = cfg["oauth_client_id"]
    GistHandler.client_secret = cfg["oauth_client_secret"]

    web_app = nb_server_app.web_app
    host_pattern = '.*$'
    route_pattern = url_path_join(web_app.settings['base_url'], '/create_gist')
    web_app.add_handlers(host_pattern, [(route_pattern, GistHandler)])
<|endoftext|>


<commit_before>import logging
import os
import sys
import threading
import time

import redis

from django.core.management.base import NoArgsCommand
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.importlib import import_module

from humfrey.longliving.base import LonglivingThread

logger = logging.getLogger(__name__)

class Command(NoArgsCommand):
    LOCK_NAME = 'longliving:lock'

    def get_threads(self, bail):
        try:
            longliving_classes = settings.LONGLIVING_CLASSES
        except AttributeError:
            raise ImproperlyConfigured("LONGLIVING_CLASSES setting missing.")

        threads = []
        for class_path in longliving_classes:
            module_name, class_name = class_path.rsplit('.', 1)
            try:
                module = import_module(module_name)
            except ImportError:
                raise ImproperlyConfigured("Could not import module %r" % module_name)
            try:
                cls = getattr(module, class_name)
            except AttributeError:
                raise ImproperlyConfigured("Module %r has no attribute %r" % (module_name, class_name))
            if not issubclass(cls, threading.Thread):
                raise ImproperlyConfigured("%r must be a subclass of threading.Thread" % class_path)
            thread = cls(bail)
            thread.name = class_path
            threads.append(thread)
        return threads

    def handle_noargs(self, **options):
        log_level = options.pop('log-level', None)
        if log_level:
            logging.basicConfig(stream=sys.stderr, level=getattr(logging, log_level.upper()))

        redis_client = redis.client.Redis(**settings.REDIS_PARAMS)

        existing_pid = redis_client.get(self.LOCK_NAME)
        if not redis_client.setnx(self.LOCK_NAME, os.getpid()):
            existing_pid = int(redis_client.get(self.LOCK_NAME))
            try:
                os.kill(existing_pid, 0)
            except OSError:
                redis_client.set(self.LOCK_NAME, os.getpid())
            else:
                logger.warning("Not starting as another instance detected.")
                sys.stderr.write("Already running\n")
                sys.exit(1)
        logger.info("Starting longliving process")

        try:
            bail = threading.Event()
            threads = self.get_threads(bail)

            for thread in threads:
                thread.start()

            logger.info("Longliving threads started")

            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                logger.info("Caught KeyboardInterrupt; shutting down.")
                bail.set()
                redis_client.publish(LonglivingThread.BAIL_CHANNEL, '')

            for i in range(5):
                for thread in threads[:]:
                    thread.join(5)
                    if thread.isAlive():
                        logger.warning("Couldn't join thread %r on attempt %i/5", thread.name, i + 1)
                    else:
                        threads.remove(thread)

            if threads:
                logger.error("Couldn't join all threads.")
            else:
                logger.info("All threads finished; stopping.")
        finally:
            redis_client.delete(self.LOCK_NAME)
<commit_msg>Fixed passing of --log-level arg to longliving command.<commit_after>import logging
from optparse import make_option
import os
import sys
import threading
import time

import redis

from django.core.management.base import BaseCommand
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.importlib import import_module

from humfrey.longliving.base import LonglivingThread

logger = logging.getLogger(__name__)

class Command(BaseCommand):
    LOCK_NAME = 'longliving:lock'

    option_list = BaseCommand.option_list + (
        make_option('--log-level',
            action='store',
            dest='level',
            default=None,
            help='Log level'),
        )

    def get_threads(self, bail):
        try:
            longliving_classes = settings.LONGLIVING_CLASSES
        except AttributeError:
            raise ImproperlyConfigured("LONGLIVING_CLASSES setting missing.")

        threads = []
        for class_path in longliving_classes:
            module_name, class_name = class_path.rsplit('.', 1)
            try:
                module = import_module(module_name)
            except ImportError:
                raise ImproperlyConfigured("Could not import module %r" % module_name)
            try:
                cls = getattr(module, class_name)
            except AttributeError:
                raise ImproperlyConfigured("Module %r has no attribute %r" % (module_name, class_name))
            if not issubclass(cls, threading.Thread):
                raise ImproperlyConfigured("%r must be a subclass of threading.Thread" % class_path)
            thread = cls(bail)
            thread.name = class_path
            threads.append(thread)
        return threads

    def handle_noargs(self, *args, **options):
        log_level = options.pop('level', None)
        if log_level:
            logging.basicConfig(stream=sys.stderr, level=getattr(logging, log_level.upper()))

        redis_client = redis.client.Redis(**settings.REDIS_PARAMS)

        existing_pid = redis_client.get(self.LOCK_NAME)
        if not redis_client.setnx(self.LOCK_NAME, os.getpid()):
            existing_pid = int(redis_client.get(self.LOCK_NAME))
            try:
                os.kill(existing_pid, 0)
            except OSError:
                redis_client.set(self.LOCK_NAME, os.getpid())
            else:
                logger.warning("Not starting as another instance detected.")
                sys.stderr.write("Already running\n")
                sys.exit(1)
        logger.info("Starting longliving process")

        try:
            bail = threading.Event()
            threads = self.get_threads(bail)

            for thread in threads:
                thread.start()

            logger.info("Longliving threads started")

            try:
                while True:
                    time.sleep(1)
            except KeyboardInterrupt:
                logger.info("Caught KeyboardInterrupt; shutting down.")
                bail.set()
                redis_client.publish(LonglivingThread.BAIL_CHANNEL, '')

            for i in range(5):
                for thread in threads[:]:
                    thread.join(5)
                    if thread.isAlive():
                        logger.warning("Couldn't join thread %r on attempt %i/5", thread.name, i + 1)
                    else:
                        threads.remove(thread)

            if threads:
                logger.error("Couldn't join all threads.")
            else:
                logger.info("All threads finished; stopping.")
        finally:
            redis_client.delete(self.LOCK_NAME)
<|endoftext|>


<commit_before>import os
import sqlite3
import sys
import logging
import datetime
import shutil
import dicom

from optparse import OptionParser
from django.core.management import setup_environ
import local_settings as local
setup_environ(local)

from django.core.exceptions import ObjectDoesNotExist
from dicom_models.staging.models import RadiologyStudy as RadStudy
from dicom_models.production.models import RadiologyStudy, DataAvailability

s_studies = RadStudy.objects.using('staging').filter(image_published=True)
count = 0
for study in s_studies:
    # Check to see if this study is in production
    study_in_prod = RadiologyStudy.objects.using('production').filter(study_uid=study.study_uid)
    if len(study_in_prod) == 0:
        new_prod_study = RadiologyStudy(study_uid = study.study_uid, modality=study.modality, sop_class=study.sop_class, number_of_series=study.number_of_series, total_images=study.total_images)
    else:
        new_prod_study = study_in_prod[0]
        new_prod_study.modality = study.modality
        new_prod_study.sop_class = study.sop_class
        new_prod_study.number_of_series = study.number_of_series
        new_prod_study.total_images = study.total_images

    new_prod_study.encounter_id = study.encounter_id
    new_prod_study.patient_id = study.encounter.patient_id
    new_prod_study.save()

    print "New study %s" % study.study_uid
    count+= 1

print "%d new studies." % count

data = DataAvailability.objects.all()
for d in data:
    d.reset_calculated_fields(save=True)

<commit_msg>Minor fix.<commit_after>import os
import sqlite3
import sys
import logging
import datetime
import shutil
import dicom

from optparse import OptionParser
from django.core.management import setup_environ
import local_settings as local
setup_environ(local)

from django.core.exceptions import ObjectDoesNotExist
from dicom_models.staging.models import RadiologyStudy as RadStudy
from dicom_models.production.models import RadiologyStudy, DataAvailability

s_studies = RadStudy.objects.using('staging').filter(image_published=True)
count = 0
for study in s_studies:
    # Check to see if this study is in production
    study_in_prod = RadiologyStudy.objects.using('production').filter(study_uid=study.study_uid)
    if len(study_in_prod) == 0:
        new_prod_study = RadiologyStudy(study_uid = study.study_uid, modality=study.modality, sop_class=study.sop_class, number_of_series=study.number_of_series, total_images=study.total_images)
    else:
        new_prod_study = study_in_prod[0]
        new_prod_study.modality = study.modality
        new_prod_study.sop_class = study.sop_class
        new_prod_study.number_of_series = study.number_of_series
        new_prod_study.total_images = study.total_images

    new_prod_study.encounter_id = study.encounter_id
    new_prod_study.patient_id = study.encounter.patient_id
    new_prod_study.save()

    print "New study %s" % study.study_uid
    count+= 1

print "%d new studies." % count

data = DataAvailability.objects.using('production').all()
for d in data:
    d.reset_calculated_fields(save=True)

<|endoftext|>