<commit_before># Copyright 2014 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from collections import OrderedDict

class EOSConf:

    def __init__(self, name):
        """
        You will probably not have to bother that much about this module yourself as it is usually easier to parse the
        configuration from a file and then use the "load_config" methods on the EOS class to get this object populated.
        However, if you understand how the eAPI handles the configuration in JSON mode you should be able to
        manipulate it in the same way.

        :param name: Name of the configuration
        """
        self.name = name
        self.cmds = OrderedDict()

    def __getitem__(self, item):
        return self.cmds[item]['cmds'].keys()

    def __str__(self):
        return '%s config' % self.name

    def __repr__(self):
        return "EOSConf: %s" % self.__str__()

    @staticmethod
    def _parse_config(config):
        cmds = OrderedDict()
        prev_key = None

        if isinstance(config, unicode) or isinstance(config, str):
            config = config.splitlines()

        for line in config:
            line = line.strip('\n')
            if line is '' or line.startswith('!'):
                pass
            elif line.startswith(' '):
                cmds[prev_key]['cmds'][line.strip()] = None
            else:
                prev_key = line
                cmds[line] = dict()
                cmds[line]['comments'] = list()
                cmds[line]['cmds'] = OrderedDict()

        return cmds

    def _load_file(self, filename):
        with open(filename, 'r') as f:
            return self._parse_config(f.readlines())

    def load_config(self, filename=None, config=None):
        """
        Reads the configuration from a file or from a string and loads the object. If you send both a filename and a
        string containing the configuration, the file takes precedence.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        """
        if isinstance(config, dict):
            self.cmds = config
        elif isinstance(config, str) or isinstance(config, unicode):
            self.cmds = self._parse_config(config)
        else:
            self.cmds = self._load_file(filename)

    def to_string(self):
        """

        :return: A string representation of the configuration.
        """
        txt = ''
        for key, value in self.cmds.iteritems():
            txt += '%s\n' % key

            for k in value['cmds'].keys():
                txt += '  %s\n' % k

        return txt

    def compare_config(self, other):
        """
        This method will compare the self object with the other object. The other object will be the target of the
        comparison.

        :param other: Configuration object you want to do the comparison with.
        :return: A string representation of the changes between the self object and other.
        """
        def _print(action, list, config):
            diff_text = ''
            for cmd in list:
                diff_text += '%s %s\n' % (action, cmd)
                for key in config.cmds[cmd]['cmds'].keys():
                    diff_text += '%s\n' % key

            return diff_text

        diff_text = ''

        added = set(other.cmds.keys()) - set(self.cmds.keys())
        removed = set(self.cmds.keys()) - set(other.cmds.keys())
        keep = set(other.cmds.keys()) & set(self.cmds.keys())

        diff_text += _print('+', added, other)
        diff_text += _print('-', removed, self)

        for cmd in keep:
            orig = set(self.cmds[cmd]['cmds'].keys())
            cand = set(other.cmds[cmd]['cmds'].keys())

            new = cand - orig
            old = orig - cand

            if len(new) > 0 or len(old) > 0:
                diff_text +=  "%s\n" % cmd

                for cmd in new:
                    diff_text +=  "  + %s\n" % cmd

                for cmd in old:
                    diff_text +=  "  - %s\n" % cmd

        return diff_text<commit_msg>Now we support subblocks. Nasty hack, I hope Arista implement the diff natively soon<commit_after># Copyright 2014 Spotify AB. All rights reserved.
#
# The contents of this file are licensed under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.

from collections import OrderedDict

class EOSConf:

    def __init__(self, name):
        """
        You will probably not have to bother that much about this module yourself as it is usually easier to parse the
        configuration from a file and then use the "load_config" methods on the EOS class to get this object populated.
        However, if you understand how the eAPI handles the configuration in JSON mode you should be able to
        manipulate it in the same way.

        :param name: Name of the configuration
        """
        self.name = name
        self.cmds = OrderedDict()

    def __getitem__(self, item):
        return self.cmds[item]['cmds'].keys()

    def __str__(self):
        return '%s config' % self.name

    def __repr__(self):
        return "EOSConf: %s" % self.__str__()

    @staticmethod
    def _parse_config(config):
        cmds = OrderedDict()
        prev_key = None
        separator = ' '

        if isinstance(config, unicode) or isinstance(config, str):
            config = config.splitlines()

        for line in config:
            line = line.strip('\n')
            if line is '' or line.startswith('!'):
                pass
            elif line.startswith('      '):
                cmds[prev_key]['cmds'][sub_prev_key]['cmds'][line.strip()] = None
            elif line.startswith('   '):
                sub_prev_key = line.strip()
                cmds[prev_key]['cmds'][sub_prev_key] = dict()
                cmds[prev_key]['cmds'][sub_prev_key]['comments'] = list()
                cmds[prev_key]['cmds'][sub_prev_key]['cmds'] = OrderedDict()
            else:
                prev_key = line
                cmds[line] = dict()
                cmds[line]['comments'] = list()
                cmds[line]['cmds'] = OrderedDict()

        return cmds

    def _load_file(self, filename):
        with open(filename, 'r') as f:
            return self._parse_config(f.readlines())

    def load_config(self, filename=None, config=None):
        """
        Reads the configuration from a file or from a string and loads the object. If you send both a filename and a
        string containing the configuration, the file takes precedence.

        :param filename: Path to the file containing the desired configuration. By default is None.
        :param config: String containing the desired configuration.
        """
        if isinstance(config, dict):
            self.cmds = config
        elif isinstance(config, str) or isinstance(config, unicode):
            self.cmds = self._parse_config(config)
        else:
            self.cmds = self._load_file(filename)

    def to_string(self):
        """

        :return: A string representation of the configuration.
        """
        txt = ''
        for key, value in self.cmds.iteritems():
            txt += '%s\n' % key

            for k in value['cmds'].keys():
                txt += '   %s\n' % k

                for sk in value['cmds'][k]['cmds'].keys():
                    txt += '      %s\n' % sk

        return txt

    def compare_config(self, other):
        """
        This method will compare the self object with the other object. The other object will be the target of the
        comparison.

        :param other: Configuration object you want to do the comparison with.
        :return: A string representation of the changes between the self object and other.
        """
        def _print(action, list, config):
            diff_text = ''
            for cmd in list:
                diff_text += '%s %s\n' % (action, cmd)
                for key in config.cmds[cmd]['cmds'].keys():
                    diff_text += '%s\n' % key

            return diff_text

        diff_text = ''

        added = set(other.cmds.keys()) - set(self.cmds.keys())
        removed = set(self.cmds.keys()) - set(other.cmds.keys())
        keep = set(other.cmds.keys()) & set(self.cmds.keys())

        diff_text += _print('+', added, other)
        diff_text += _print('-', removed, self)

        for cmd in keep:
            mine = self.cmds[cmd]['cmds']
            sother = other.cmds[cmd]['cmds']

            added = set(sother.keys()) - set(mine.keys())
            removed = set(mine.keys()) - set(sother.keys())
            keep = set(sother.keys()) & set(mine.keys())

            diff_text += _print('+', added, sother)
            diff_text += _print('-', removed, mine)

            for scmd in keep:
                orig = set(mine[scmd]['cmds'].keys())
                cand = set(sother[scmd]['cmds'].keys())

                new = cand - orig
                old = orig - cand

                if len(new) > 0 or len(old) > 0:
                    diff_text +=  "%s\n" % scmd

                    for cmd in new:
                        diff_text +=  "  + %s\n" % scmd

                    for cmd in old:
                        diff_text +=  "  - %s\n" % scmd

        return diff_text<|endoftext|>


<commit_before>from pygraphc.similarity.CosineSimilarity import CosineSimilarity

class CalinskiHarabaszIndex(object):
    def __init__(self, clusters, preprocessed_logs, log_length):
        self.clusters = clusters
        self.preprocessed_logs = preprocessed_logs
        self.log_length = log_length
        self.cluster_centroids = {}
        self.cluster_total_nodes = {}
        self.__get_all_cluster_properties()

    def __get_centroid(self, cluster=None):
        centroid = ''

        # centroid for a particular cluster
        if cluster:
            for log_id in cluster:
                centroid += self.preprocessed_logs[log_id]
        # centroid for the whole logs
        else:
            for log_id in self.preprocessed_logs:
                centroid += self.preprocessed_logs[log_id]

        return centroid

    def __get_all_cluster_properties(self):
        for cluster_id, log_ids in self.clusters.iteritems():
            self.cluster_centroids[cluster_id] = self.__get_centroid(log_ids)
            self.cluster_total_nodes[cluster_id] = len(log_ids)

    @staticmethod
    def __get_distance(source, dest):
        cs = CosineSimilarity()
        distance = cs.get_cosine_similarity(source, dest)
        return distance

    def __get_trace_b(self):
        traces_b = []
        logs_centroid = self.__get_centroid()
        for cluster_id, log_ids in self.clusters.iteritems():
            trace_b = self.cluster_total_nodes[cluster_id] * \
                      (self.__get_distance(self.cluster_centroids[cluster_id], logs_centroid) ** 2)
            traces_b.append(trace_b)

        total_trace_b = sum(traces_b)
        return total_trace_b

    def __get_trace_w(self):
        traces_w = []
        for cluster_id, log_ids in self.clusters.iteritems():
            trace_w_cluster = []
            for log_id in log_ids:
                trace_w = self.__get_distance(self.preprocessed_logs[log_id], self.cluster_centroids[cluster_id]) ** 2
                trace_w_cluster.append(trace_w)
            traces_w.append(sum(trace_w_cluster))

        total_traces_w = sum(traces_w)
        return total_traces_w

    def get_calinski_harabasz(self):
        total_cluster = len(self.clusters.keys())
        ch_index = (self.__get_trace_b() / (total_cluster - 1)) / \
                   (self.__get_trace_w() / (self.log_length - total_cluster))

        return ch_index

<commit_msg>Move get properties to main method<commit_after>from pygraphc.similarity.CosineSimilarity import CosineSimilarity

class CalinskiHarabaszIndex(object):
    def __init__(self, clusters, preprocessed_logs, log_length):
        self.clusters = clusters
        self.preprocessed_logs = preprocessed_logs
        self.log_length = log_length
        self.cluster_centroids = {}
        self.cluster_total_nodes = {}

    def __get_centroid(self, cluster=None):
        centroid = ''

        # centroid for a particular cluster
        if cluster:
            for log_id in cluster:
                centroid += self.preprocessed_logs[log_id]
        # centroid for the whole logs
        else:
            for log_id in self.preprocessed_logs:
                centroid += self.preprocessed_logs[log_id]

        return centroid

    def __get_all_cluster_properties(self):
        for cluster_id, log_ids in self.clusters.iteritems():
            self.cluster_centroids[cluster_id] = self.__get_centroid(log_ids)
            self.cluster_total_nodes[cluster_id] = len(log_ids)

    @staticmethod
    def __get_distance(source, dest):
        cs = CosineSimilarity()
        distance = cs.get_cosine_similarity(source, dest)
        return distance

    def __get_trace_b(self):
        traces_b = []
        logs_centroid = self.__get_centroid()
        for cluster_id, log_ids in self.clusters.iteritems():
            trace_b = self.cluster_total_nodes[cluster_id] * \
                      (self.__get_distance(self.cluster_centroids[cluster_id], logs_centroid) ** 2)
            traces_b.append(trace_b)

        total_trace_b = sum(traces_b)
        return total_trace_b

    def __get_trace_w(self):
        traces_w = []
        for cluster_id, log_ids in self.clusters.iteritems():
            trace_w_cluster = []
            for log_id in log_ids:
                trace_w = self.__get_distance(self.preprocessed_logs[log_id], self.cluster_centroids[cluster_id]) ** 2
                trace_w_cluster.append(trace_w)
            traces_w.append(sum(trace_w_cluster))

        total_traces_w = sum(traces_w)
        return total_traces_w

    def get_calinski_harabasz(self):
        self.__get_all_cluster_properties()
        total_cluster = len(self.clusters.keys())
        ch_index = (self.__get_trace_b() / (total_cluster - 1)) / \
                   (self.__get_trace_w() / (self.log_length - total_cluster))

        return ch_index
<|endoftext|>


<commit_before>#!/usr/bin/env python
#-*- coding:utf-8 -*-

import _winreg
import os, sys
import re

REGEX_PATH = re.compile(r"^[A-Z]{1}:(\\[A-Za-z\s]+){0,5}\\?$")
REGEX_HKEY = re.compile(r"")

class PackageInstall:
	def __init__(self, path="C:"):
		self.path = path.replace("\\", "_")

	def _fontRegister(self):
		_sub_key = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Console\\TrueTypeFont"
		try:
				_preKeyHandle = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, _sub_key, 0, _winreg.KEY_ALL_ACCESS)
				_winreg.SetValueEx(_preKeyHandle, "000", 0, _winreg.REG_SZ, "Source Code Pro")
		except WindowsError as err:
				print str(err)
				sys.exit(2)
		finally:
				_winreg.CloseKey(_preKeyHandle)

	def _copyFiles(self):
		pass

	def _mainRegister(self):
		_pairs = [("CodePage", _winreg.REG_DWORD, 850),\
				("ScreenBufferSize", _winreg.REG_DWORD, 19660909),\
				("WindowSize", _winreg.REG_DWORD, 2293869),\
				("FontSize", _winreg.REG_DWORD, 1179648), \
				("FontFamily", _winreg.REG_DWORD, 54),\
				("FontWeight", _winreg.REG_DWORD, 400), \
				("FaceName", _winreg.REG_SZ, "Source Code Pro"),
				]

		_suffix = "_MACQuery_main.exe"
		_hkey = self.path[-1] == "_" and self.path[:-1]+_suffix or self.path + _suffix

		try:
				_preKeyHandle = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Console", 0, _winreg.KEY_ALL_ACCESS)
				_newKeyHandle = _winreg.CreateKey(_preKeyHandle, _hkey)

				for _name, _type, _value in _pairs:
						_winreg.SetValueEx(_newKeyHandle, _name, 0, _type, _value)
		except WindowsError as err:
				print str(err)
				sys.exit(2)
		finally:
				_winreg.CloseKey(_newKeyHandle)
				_winreg.CloseKey(_preKeyHandle)

	def packed(self):
		pass

if __name__ == "__main__":
		pass
<commit_msg>modified winSetup.py<commit_after>#!/usr/bin/env python
#-*- coding:utf-8 -*-

import _winreg
import os, sys
import re

REGEX_PATH = re.compile(r"^[A-Z]{1}:(\\[A-Za-z\s]+){0,5}\\?$")
REGEX_FONT = re.compile(r"^Source Code Pro.*\.ttf$")

class BadPathFormat(Exception):
	_info = """
	INSTALLATION FAILED:
	Bad format of installation path, the following format is recommended:
		Drive:\sub_directory1\sub_directory2
	"""
class FontRelyError(Exception):
	_info="""
	INSTALLATION FAILED:
	Font <<Source Code Pro>> is needed,please install the font first.
	"""

class PackageInstall:
	def __init__(self, path="C:\\"):
		if REGEX_PATH.search(path):
				self.abspath = path.rstrip("\\")
				self.hkeypath = path.rstrip("\\").replace("\\", "_")
		else:
				raise BadPathFormat, BadPathFormat._info

	def _fontRegister(self):
		_fontList = os.listdir("C:\\Windows\\Fonts")
		for font in _fontList:
				if REGEX_FONT.search(font):
						break
				else:
						raise FontRelyError, FontRelyError._info

		_sub_key = "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion\\Console\\TrueTypeFont"
		try:
				_preKeyHandle = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, _sub_key, 0, _winreg.KEY_ALL_ACCESS)
				_winreg.SetValueEx(_preKeyHandle, "000", 0, _winreg.REG_SZ, "Source Code Pro")
		except WindowsError as err:
				print str(err)
				sys.exit(2)
		finally:
				_winreg.CloseKey(_preKeyHandle)

	def _copyFiles(self):
		_dir = self.abspath + "\\" + "MACQuery"

		try:
				os.mkdir(_dir)
		except (IOError, OSError) as err:
				print str(err)
				sys.exit(2)
		try:
				os.system("xcopy .\\* %s /EXCLUDE:winSetup.exe" % _dir)
		except (IOError, OSError) as err:
				print str(err)
				sys.exit(2)

	def _mainRegister(self):
		_pairs = [("CodePage", _winreg.REG_DWORD, 850),\
				("ScreenBufferSize", _winreg.REG_DWORD, 19660909),\
				("WindowSize", _winreg.REG_DWORD, 2293869),\
				("FontSize", _winreg.REG_DWORD, 1179648), \
				("FontFamily", _winreg.REG_DWORD, 54),\
				("FontWeight", _winreg.REG_DWORD, 400), \
				("FaceName", _winreg.REG_SZ, "Source Code Pro"),
				]

		_suffix = "_MACQuery_main.exe"
		_hkey = self.hkeypath + _suffix

		try:
				_preKeyHandle = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, "Console", 0, _winreg.KEY_ALL_ACCESS)
				_newKeyHandle = _winreg.CreateKey(_preKeyHandle, _hkey)

				for _name, _type, _value in _pairs:
						_winreg.SetValueEx(_newKeyHandle, _name, 0, _type, _value)
		except WindowsError as err:
				print str(err)
				sys.exit(2)
		finally:
				_winreg.CloseKey(_newKeyHandle)
				_winreg.CloseKey(_preKeyHandle)

	def packed(self):
		self._fontRegister()
		self._copyFiles()
		self._mainRegister()

if __name__ == "__main__":
		handler = PackageInstall("H:\\")
		handler.packed()
<|endoftext|>


<commit_before># Copyright 2013 Novo Nordisk Foundation Center for Biosustainability,
# Technical University of Denmark.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = '0.0.1'

from .util import list_available_solvers
available_solvers = list_available_solvers()

if available_solvers['GLPK']:
    from .glpk_interface import Model, Variable, Constraint, Objective
elif available_solvers['GUROBI']:
    from .gurobi_interface import Model, Variable, Constraint, Objective
elif available_solvers['CPLEX']:
    from .cplex_interface import Model, Variable, Constraint, Objective
else:
    raise Warning('No solvers available.')

__all__ = []<commit_msg>Log an error if no solvers are installed.<commit_after># Copyright 2013 Novo Nordisk Foundation Center for Biosustainability,
# Technical University of Denmark.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__version__ = '0.0.1'

import logging
log = logging.getLogger(__name__)

from .util import list_available_solvers
available_solvers = list_available_solvers()

if available_solvers['GLPK']:
    from .glpk_interface import Model, Variable, Constraint, Objective
elif available_solvers['GUROBI']:
    from .gurobi_interface import Model, Variable, Constraint, Objective
elif available_solvers['CPLEX']:
    from .cplex_interface import Model, Variable, Constraint, Objective
else:
    log.error('No solvers available.')

# __all__ = (
#     'Model',
#     'Variable',
#     'Constraint',
#     'Objective',
# )
<|endoftext|>


<commit_before>import numpy as np
from numpy.testing import *

import locale
import sys
from StringIO import StringIO

def check_float_type(tp):
    for x in [0, 1,-1, 1e20] :
        assert_equal(str(tp(x)), str(float(x)),
                     err_msg='Failed str formatting for type %s' % tp)

    if tp(1e10).itemsize > 4:
        assert_equal(str(tp(1e10)), str(float('1e10')),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '1e+10',
                     err_msg='Failed str formatting for type %s' % tp)

def test_float_types():
    """ Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_float_type, t

def check_nan_inf_float(tp):
    for x in [np.inf, -np.inf, np.nan]:
        assert_equal(str(tp(x)), str(float(x)),
                     err_msg='Failed str formatting for type %s' % tp)

def test_nan_inf_float():
    """ Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_nan_inf_float, t

def check_complex_type(tp):
    for x in [0, 1,-1, 1e20] :
        assert_equal(str(tp(x)), str(complex(x)),
                     err_msg='Failed str formatting for type %s' % tp)
        assert_equal(str(tp(x*1j)), str(complex(x*1j)),
                     err_msg='Failed str formatting for type %s' % tp)
        assert_equal(str(tp(x + x*1j)), str(complex(x + x*1j)),
                     err_msg='Failed str formatting for type %s' % tp)

    if tp(1e10).itemsize > 8:
        assert_equal(str(tp(1e10)), str(complex(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '(1e+10+0j)',
                     err_msg='Failed str formatting for type %s' % tp)

def test_complex_types():
    """Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.complex64, np.cdouble, np.clongdouble] :
        yield check_complex_type, t

# print tests
def _test_redirected_print(x, tp):
    file = StringIO()
    file_tp = StringIO()
    stdout = sys.stdout
    try:
        sys.stdout = file_tp
        print tp(x)
        sys.stdout = file
        print x
    finally:
        sys.stdout = stdout

    assert_equal(file.getvalue(), file_tp.getvalue(),
                 err_msg='print failed for type%s' % tp)

def check_float_type_print(tp):
    for x in [0, 1,-1, 1e20, np.inf, -np.inf, np.nan]:
        _test_redirected_print(float(x), tp)

    if tp(1e10).itemsize > 4:
        assert_equal(str(tp(1e10)), str(float(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '1e+10',
                     err_msg='Failed str formatting for type %s' % tp)

def check_complex_type_print(tp):
    # We do not create complex with inf/nan directly because the feature is
    # missing in python < 2.6
    for x in [0, 1, -1, 1e20, complex(np.inf, 1),
              complex(np.nan, 1), complex(-np.inf, 1)] :
        _test_redirected_print(complex(x), tp)

    if tp(1e10).itemsize > 8:
        assert_equal(str(tp(1e10)), str(complex(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '(1e+10+0j)',
                     err_msg='Failed str formatting for type %s' % tp)

def test_float_type_print():
    """Check formatting when using print """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_float_type_print, t

def test_complex_type_print():
    """Check formatting when using print """
    for t in [np.complex64, np.cdouble, np.clongdouble] :
        yield check_complex_type_print, t

# Locale tests: scalar types formatting should be independant of the locale
def has_french_locale():
    curloc = locale.getlocale(locale.LC_NUMERIC)
    try:
        if not sys.platform == 'win32':
            locale.setlocale(locale.LC_NUMERIC, 'fr_FR')
        else:
            locale.setlocale(locale.LC_NUMERIC, 'FRENCH')

        st = True
    except:
        st = False
    finally:
        locale.setlocale(locale.LC_NUMERIC, locale=curloc)

    return st

def _test_locale_independance(tp):
    # XXX: How to query locale on a given system ?

    # French is one language where the decimal is ',' not '.', and should be
    # relatively common on many systems
    curloc = locale.getlocale(locale.LC_NUMERIC)
    try:
        if not sys.platform == 'win32':
            locale.setlocale(locale.LC_NUMERIC, 'fr_FR')
        else:
            locale.setlocale(locale.LC_NUMERIC, 'FRENCH')

        assert_equal(str(tp(1.2)), str(float(1.2)),
                     err_msg='Failed locale test for type %s' % tp)
    finally:
        locale.setlocale(locale.LC_NUMERIC, locale=curloc)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_single():
    return _test_locale_independance(np.float32)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_double():
    return _test_locale_independance(np.double)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_longdouble():
    return _test_locale_independance(np.longdouble)

if __name__ == "__main__":
    run_module_suite()
<commit_msg>Special case float tests on windows: python 2.5 and below have >=3 digits in the exp.<commit_after>import numpy as np
from numpy.testing import *

import locale
import sys
from StringIO import StringIO

def check_float_type(tp):
    for x in [0, 1,-1, 1e20] :
        assert_equal(str(tp(x)), str(float(x)),
                     err_msg='Failed str formatting for type %s' % tp)

    if tp(1e10).itemsize > 4:
        assert_equal(str(tp(1e10)), str(float('1e10')),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        if sys.platform == 'win32' and sys.version_info[0] <= 2 and sys.version_info[1] <= 5:
            ref = '1e+010'
        else:
            ref = '1e+10'
        assert_equal(str(tp(1e10)), ref,
                     err_msg='Failed str formatting for type %s' % tp)

def test_float_types():
    """ Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_float_type, t

def check_nan_inf_float(tp):
    for x in [np.inf, -np.inf, np.nan]:
        assert_equal(str(tp(x)), str(float(x)),
                     err_msg='Failed str formatting for type %s' % tp)

def test_nan_inf_float():
    """ Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_nan_inf_float, t

def check_complex_type(tp):
    for x in [0, 1,-1, 1e20] :
        assert_equal(str(tp(x)), str(complex(x)),
                     err_msg='Failed str formatting for type %s' % tp)
        assert_equal(str(tp(x*1j)), str(complex(x*1j)),
                     err_msg='Failed str formatting for type %s' % tp)
        assert_equal(str(tp(x + x*1j)), str(complex(x + x*1j)),
                     err_msg='Failed str formatting for type %s' % tp)

    if tp(1e10).itemsize > 8:
        assert_equal(str(tp(1e10)), str(complex(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        if sys.platform == 'win32' and sys.version_info[0] <= 2 and sys.version_info[1] <= 5:
            ref = '(1e+010+0j)'
        else:
            ref = '(1e+10+0j)'
        assert_equal(str(tp(1e10)), ref,
                     err_msg='Failed str formatting for type %s' % tp)

def test_complex_types():
    """Check formatting.

        This is only for the str function, and only for simple types.
        The precision of np.float and np.longdouble aren't the same as the
        python float precision.

    """
    for t in [np.complex64, np.cdouble, np.clongdouble] :
        yield check_complex_type, t

# print tests
def _test_redirected_print(x, tp):
    file = StringIO()
    file_tp = StringIO()
    stdout = sys.stdout
    try:
        sys.stdout = file_tp
        print tp(x)
        sys.stdout = file
        print x
    finally:
        sys.stdout = stdout

    assert_equal(file.getvalue(), file_tp.getvalue(),
                 err_msg='print failed for type%s' % tp)

def check_float_type_print(tp):
    for x in [0, 1,-1, 1e20, np.inf, -np.inf, np.nan]:
        _test_redirected_print(float(x), tp)

    if tp(1e10).itemsize > 4:
        assert_equal(str(tp(1e10)), str(float(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '1e+10',
                     err_msg='Failed str formatting for type %s' % tp)

def check_complex_type_print(tp):
    # We do not create complex with inf/nan directly because the feature is
    # missing in python < 2.6
    for x in [0, 1, -1, 1e20, complex(np.inf, 1),
              complex(np.nan, 1), complex(-np.inf, 1)] :
        _test_redirected_print(complex(x), tp)

    if tp(1e10).itemsize > 8:
        assert_equal(str(tp(1e10)), str(complex(1e10)),
                     err_msg='Failed str formatting for type %s' % tp)
    else:
        assert_equal(str(tp(1e10)), '(1e+10+0j)',
                     err_msg='Failed str formatting for type %s' % tp)

def test_float_type_print():
    """Check formatting when using print """
    for t in [np.float32, np.double, np.longdouble] :
        yield check_float_type_print, t

def test_complex_type_print():
    """Check formatting when using print """
    for t in [np.complex64, np.cdouble, np.clongdouble] :
        yield check_complex_type_print, t

# Locale tests: scalar types formatting should be independant of the locale
def has_french_locale():
    curloc = locale.getlocale(locale.LC_NUMERIC)
    try:
        if not sys.platform == 'win32':
            locale.setlocale(locale.LC_NUMERIC, 'fr_FR')
        else:
            locale.setlocale(locale.LC_NUMERIC, 'FRENCH')

        st = True
    except:
        st = False
    finally:
        locale.setlocale(locale.LC_NUMERIC, locale=curloc)

    return st

def _test_locale_independance(tp):
    # XXX: How to query locale on a given system ?

    # French is one language where the decimal is ',' not '.', and should be
    # relatively common on many systems
    curloc = locale.getlocale(locale.LC_NUMERIC)
    try:
        if not sys.platform == 'win32':
            locale.setlocale(locale.LC_NUMERIC, 'fr_FR')
        else:
            locale.setlocale(locale.LC_NUMERIC, 'FRENCH')

        assert_equal(str(tp(1.2)), str(float(1.2)),
                     err_msg='Failed locale test for type %s' % tp)
    finally:
        locale.setlocale(locale.LC_NUMERIC, locale=curloc)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_single():
    return _test_locale_independance(np.float32)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_double():
    return _test_locale_independance(np.double)

@np.testing.dec.skipif(not has_french_locale(),
                       "Skipping locale test, French locale not found")
def test_locale_longdouble():
    return _test_locale_independance(np.longdouble)

if __name__ == "__main__":
    run_module_suite()
<|endoftext|>


<commit_before>'''
Created on Jan 10, 2017

@author: phusisian
'''
import random
from ObjAvoid import *

class Test:

    def __init__(self):
        self.massHolder = MassHolder()
        self.droneMass = DroneMass(self.massHolder, TestFunctions.getRandomPointsInBounds(2, 20, ((-700, 700), (-500,500))))#MultiDimPoint([0,0]), 1, [MultiDimPoint([200, 200]), MultiDimPoint([400, -200])])
        self.massHolder.appendDroneMass(self.droneMass)
        self.addRandomMasses(40)
        self.window = Window(self.massHolder, (1440,900))

    def addRandomMasses(self, numMasses):
        randPoints = TestFunctions.getRandomPointsInBounds(2, numMasses, ((-700, 700), (-500, 500)))
        for i in range(0, len(randPoints)):
            #self.massHolder.appendMass(Mass(self.massHolder, randPoints[i], 500))
            self.massHolder.appendMass(SafetyRadiusMass(self.massHolder, randPoints[i], 500, 20, Window.REFRESH_TIME))

    def drawStationaryObjects(self, win):
        self.massHolder.drawStationaryObjects(win)

    def draw(self, win):
        self.massHolder.tick()
        self.massHolder.draw(win)

if __name__ == '__main__':
    my_test = Test()
<commit_msg>committed to fix init error<commit_after>'''
Created on Jan 10, 2017

@author: phusisian
'''
import random
from ObjAvoid import *

class Test:

    def __init__(self):
        self.massHolder = MassHolder()
        self.droneMass = DroneMass(self.massHolder, MultiDimPoint([0,0]), 1, TestFunctions.getRandomPointsInBounds(2, 20, ((-700, 700), (-500,500))))#MultiDimPoint([0,0]), 1, [MultiDimPoint([200, 200]), MultiDimPoint([400, -200])])
        self.massHolder.appendDroneMass(self.droneMass)
        self.addRandomMasses(40)
        self.window = Window(self.massHolder, (1440,900))

    def addRandomMasses(self, numMasses):
        randPoints = TestFunctions.getRandomPointsInBounds(2, numMasses, ((-700, 700), (-500, 500)))
        for i in range(0, len(randPoints)):
            #self.massHolder.appendMass(Mass(self.massHolder, randPoints[i], 500))
            self.massHolder.appendMass(SafetyRadiusMass(self.massHolder, randPoints[i], 500, 20, Window.REFRESH_TIME))

    def drawStationaryObjects(self, win):
        self.massHolder.drawStationaryObjects(win)

    def draw(self, win):
        self.massHolder.tick()
        self.massHolder.draw(win)

if __name__ == '__main__':
    my_test = Test()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
from glob import glob
from logman import logger
import pickle
import os
import sys
from utils import load_reports

from pefinder import (
    mark_reports,
    analyze_reports,
    label_remapping
)

def get_parser():

    parser = argparse.ArgumentParser(description="generate predictions for PE for a set of reports (impressions)")

    # Name of the docker image, required
    parser.add_argument("--reports",
                        dest='reports',
                        help="Path to folder of reports, or tab separated text file",
                        type=str,
                        required=True)

    parser.add_argument("--report_field",
                        dest='report_field',
                        help="the header column that contains the text of interest (default is report_text)",
                        type=str,
                        default="report_text")

    parser.add_argument("--id_field",
                        dest='id_field',
                        help="the header column that contains the id of the report (default is report_id)",
                        type=str,
                        default="report_id")

    parser.add_argument("--result_field",
                        dest='result_field',
                        help="the field to save pefinder (chapman) result to, not saved unless --no-remap is specified.",
                        type=str,
                        default="pe_result")

    parser.add_argument("--delim",
                        dest='delim',
                        help="the delimiter separating the input reports data. Default is tab (\\t)",
                        type=str,
                        default="\t")

    parser.add_argument("--output",
                        dest='output',
                        help="Desired output file (.tsv)",
                        type=str,
                        required=True)

    parser.add_argument('--no-remap',
                        dest='remapping',
                        help="don't remap multilabel PEFinder result to Stanford labels",
                        default=True,
                        action='store_false')

    parser.add_argument('--run',
                        dest="actions",
                        help="mark (mark), or classify (classify) reports.",
                        choices=['classify','mark'],
                        default='classify')

    return parser

def main():
    parser = get_parser()

    try:
        args = parser.parse_args()
    except:
        sys.exit(0)

    # Tell the user what is going to be used, in case is incorrect
    logger.info("\n***STARTING PE-FINDER CONTAINER****")
    logger.info("Will use column %s as report text.",args.report_field)
    logger.info("Will use column %s as report id.",args.id_field)

    # Load the reports
    reports = load_reports(reports_path=args.reports,
                           report_field=args.report_field,
                           id_field=args.id_field,
                           delim=args.delim)

    # What actions does the user want to run?
    if "analyze" == args.actions:
        reports = analyze_reports(reports,result_field=args.result_field)

        # Remap to Stanford labels (default True)
        if args.remapping == True:
            reports = label_remapping(reports=reports,
                                      result_field=args.result_field,
                                      drop_result=True)

    elif "mark" == args.actions:
        reports = mark_reports(reports)

    # Parse result in some format, provide visualization?
    reports.to_csv(args.output,sep="\t",index=False)
    logger.info("Result for %s saved to %s",args.actions,args.output)

if __name__ == '__main__':
    main()
<commit_msg>wrong argument, should have been classify (not analyze)<commit_after>#!/usr/bin/env python3

import argparse
from glob import glob
from logman import logger
import pickle
import os
import sys
from utils import load_reports

from pefinder import (
    mark_reports,
    analyze_reports,
    label_remapping
)

def get_parser():

    parser = argparse.ArgumentParser(description="generate predictions for PE for a set of reports (impressions)")

    # Name of the docker image, required
    parser.add_argument("--reports",
                        dest='reports',
                        help="Path to folder of reports, or tab separated text file",
                        type=str,
                        required=True)

    parser.add_argument("--report_field",
                        dest='report_field',
                        help="the header column that contains the text of interest (default is report_text)",
                        type=str,
                        default="report_text")

    parser.add_argument("--id_field",
                        dest='id_field',
                        help="the header column that contains the id of the report (default is report_id)",
                        type=str,
                        default="report_id")

    parser.add_argument("--result_field",
                        dest='result_field',
                        help="the field to save pefinder (chapman) result to, not saved unless --no-remap is specified.",
                        type=str,
                        default="pe_result")

    parser.add_argument("--delim",
                        dest='delim',
                        help="the delimiter separating the input reports data. Default is tab (\\t)",
                        type=str,
                        default="\t")

    parser.add_argument("--output",
                        dest='output',
                        help="Desired output file (.tsv)",
                        type=str,
                        required=True)

    parser.add_argument('--no-remap',
                        dest='remapping',
                        help="don't remap multilabel PEFinder result to Stanford labels",
                        default=True,
                        action='store_false')

    parser.add_argument('--run',
                        dest="actions",
                        help="mark (mark), or classify (classify) reports.",
                        choices=['classify','mark'],
                        default='classify')

    return parser

def main():
    parser = get_parser()

    try:
        args = parser.parse_args()
    except:
        sys.exit(0)

    # Tell the user what is going to be used, in case is incorrect
    logger.info("\n***STARTING PE-FINDER CONTAINER****")
    logger.info("Will use column %s as report text.",args.report_field)
    logger.info("Will use column %s as report id.",args.id_field)

    # Load the reports
    reports = load_reports(reports_path=args.reports,
                           report_field=args.report_field,
                           id_field=args.id_field,
                           delim=args.delim)

    # What actions does the user want to run?
    if "classify" == args.actions:
        reports = analyze_reports(reports,result_field=args.result_field)

        # Remap to Stanford labels (default True)
        if args.remapping == True:
            reports = label_remapping(reports=reports,
                                      result_field=args.result_field,
                                      drop_result=True)

    elif "mark" == args.actions:
        reports = mark_reports(reports)

    # Parse result in some format, provide visualization?
    reports.to_csv(args.output,sep="\t",index=False)
    logger.info("Result for %s saved to %s",args.actions,args.output)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright 2014 Open Source Robotics Foundation

#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import os
import subprocess
import shutil
import tempfile
import time
import unittest

class TempDirLockfile():
    def __enter__(self):
        self.dir = tempfile.mkdtemp(prefix=os.path.expanduser('~')+'/')
        return os.path.join(self.dir, 'lockfile')

    def __exit__(self, type, value, traceback):
        shutil.rmtree(self.dir)

class TestWithlock(unittest.TestCase):

    def test_basic(self):
        """ Check for basic functionality """
        with TempDirLockfile() as lf:
            cmd = "./withlock %s echo hello world" % lf
            self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_version(self):
        """ Check for not crashing """
        cmd = "./withlock --version"
        self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_help(self):
        """ Check for not crashing """
        cmd = "./withlock --help"
        self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_group_writeable(self):
        """ Test a group writable location for protection """
        cmd = "./withlock /tmp/lockfile echo hello world"
        self.assertEqual(subprocess.call(cmd.split()), 3)

    def test_trylock(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            cmd = "./withlock %s echo hello world" % lf
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 1)

    def test_trylock_quiet(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            cmd = "./withlock -q %s echo hello world" % lf
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 0)

    def test_timeout(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            cmd = "./withlock -w 2 %s echo hello world" % lf
            start_time = time.time()
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 1)
            delta = time.time() - start_time
            self.assertTrue(2.0 <= delta)
            self.assertTrue(delta < 3.0)
<commit_msg>wait to give blocking lock time to start before test lock<commit_after>#!/usr/bin/env python

# Copyright 2014 Open Source Robotics Foundation

#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

import os
import subprocess
import shutil
import tempfile
import time
import unittest

class TempDirLockfile():
    def __enter__(self):
        self.dir = tempfile.mkdtemp(prefix=os.path.expanduser('~')+'/')
        return os.path.join(self.dir, 'lockfile')

    def __exit__(self, type, value, traceback):
        shutil.rmtree(self.dir)

class TestWithlock(unittest.TestCase):

    def test_basic(self):
        """ Check for basic functionality """
        with TempDirLockfile() as lf:
            cmd = "./withlock %s echo hello world" % lf
            self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_version(self):
        """ Check for not crashing """
        cmd = "./withlock --version"
        self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_help(self):
        """ Check for not crashing """
        cmd = "./withlock --help"
        self.assertEqual(subprocess.call(cmd.split()), 0)

    def test_group_writeable(self):
        """ Test a group writable location for protection """
        cmd = "./withlock /tmp/lockfile echo hello world"
        self.assertEqual(subprocess.call(cmd.split()), 3)

    def test_trylock(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            # make sure the above has started
            time.sleep(0.1)
            cmd = "./withlock %s echo hello world" % lf
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 1)

    def test_trylock_quiet(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            # make sure the above has started
            time.sleep(0.1)
            cmd = "./withlock -q %s echo hello world" % lf
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 0)

    def test_timeout(self):
        with TempDirLockfile() as lf:
            cmd = "./withlock %s sleep 5" % lf
            p = subprocess.Popen(cmd.split())
            # make sure the above has started
            time.sleep(0.1)
            cmd = "./withlock -w 2 %s echo hello world" % lf
            start_time = time.time()
            retval = subprocess.call(cmd.split())
            p.kill()
            self.assertEqual(retval, 1)
            delta = time.time() - start_time
            self.assertTrue(2.0 <= delta)
            self.assertTrue(delta < 3.0)
<|endoftext|>


<commit_before>from graphserver.core import Graph, TripBoard, HeadwayBoard, HeadwayAlight, Crossing, TripAlight, Timezone, Street, Link, ElapseTime
from optparse import OptionParser
from graphserver.graphdb import GraphDatabase
from graphserver.ext.gtfs.gtfsdb import GTFSDatabase, parse_gtfs_date
import sys
import pytz
from tools import service_calendar_from_timezone
import datetime

def cons(ary):
    for i in range(len(ary)-1):
        yield (ary[i], ary[i+1])

class GTFSGraphCompiler:
    def __init__(self, gtfsdb, agency_namespace, agency_id=None, reporter=None):
        self.gtfsdb = gtfsdb
        self.agency_namespace = agency_namespace
        self.reporter = reporter

        # get graphserver.core.Timezone and graphserver.core.ServiceCalendars from gtfsdb for agency with given agency_id
        timezone_name = gtfsdb.agency_timezone_name(agency_id)
        self.tz = Timezone.generate( timezone_name )
        if reporter: reporter.write( "constructing service calendar for timezone '%s'\n"%timezone_name )
        self.sc = service_calendar_from_timezone(gtfsdb, timezone_name )

    def bundle_to_boardalight_edges(self, bundle, service_id):
        """takes a bundle and yields a bunch of edges"""

        stop_time_bundles = bundle.stop_time_bundles(service_id)

        n_trips = len(bundle.trip_ids)

        # If there's less than two stations on this trip bundle, the trip bundle doesn't actually span two places
        if len(stop_time_bundles)<2:
            return

        # If there are no stop_times in a bundle on this service day, there is nothing to load
        if n_trips==0:
            return

        if self.reporter: self.reporter.write( "inserting %d trips with %d stop_time bundles on service_id '%s'\n"%(len(stop_time_bundles[0]),len(stop_time_bundles),service_id) )

        #add board edges
        for i, stop_time_bundle in enumerate(stop_time_bundles[:-1]):

            trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled = stop_time_bundle[0]

            if arrival_time != departure_time:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s-depart"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

                # construct the board/alight/dwell triangle for this patternstop
                patternstop_arrival_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

                dwell_crossing = Crossing()
                for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                    dwell_crossing.add_crossing_time( trip_id, departure_time-arrival_time )

                yield (patternstop_arrival_vx_name,
                           patternstop_vx_name,
                           dwell_crossing)

            else:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

            b = TripBoard(service_id, self.sc, self.tz, 0)
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                b.add_boarding( trip_id, departure_time, stop_sequence )

            yield ( "sta-%s"%stop_id, patternstop_vx_name, b )

        #add alight edges
        for i, stop_time_bundle in enumerate(stop_time_bundles[1:]):

            trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled = stop_time_bundle[0]

            if arrival_time != departure_time:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)
            else:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)

            al = TripAlight(service_id, self.sc, self.tz, 0)
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                al.add_alighting( trip_id.encode('ascii'), arrival_time, stop_sequence )

            yield ( patternstop_vx_name, "sta-%s"%stop_id, al )

        # add crossing edges
        for i, (from_stop_time_bundle, to_stop_time_bundle) in enumerate(cons(stop_time_bundles)):

            trip_id, from_arrival_time, from_departure_time, stop_id, stop_sequence, stop_dist_traveled = from_stop_time_bundle[0]
            trip_id, to_arrival_time, to_departure_time, stop_id, stop_sequence, stop_dist_traveled = to_stop_time_bundle[0]

            if from_arrival_time!=from_departure_time:
                from_patternstop_vx_name = "psv-%s-%03d-%03d-%s-depart"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)
            else:
                from_patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

            if to_arrival_time!=to_departure_time:
                to_patternstop_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)
            else:
                to_patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)

            crossing = Crossing()
            for i in range( len( from_stop_time_bundle ) ):
                trip_id, from_arrival_time, from_departure_time, stop_id, stop_sequence, stop_dist_traveled = from_stop_time_bundle[i]
                trip_id, to_arrival_time, to_departure_time, stop_id, stop_sequence, stop_dist_traveled = to_stop_time_bundle[i]
                crossing.add_crossing_time( trip_id, (to_arrival_time-from_departure_time) )

            yield ( from_patternstop_vx_name,
                    to_patternstop_vx_name,
                    crossing )

    def gtfsdb_to_scheduled_edges(self, maxtrips=None, service_ids=None):

        # compile trip bundles from gtfsdb
        if self.reporter: self.reporter.write( "Compiling trip bundles...\n" )
        bundles = self.gtfsdb.compile_trip_bundles(maxtrips=maxtrips, reporter=self.reporter)

        # load bundles to graph
        if self.reporter: self.reporter.write( "Loading trip bundles into graph...\n" )
        n_bundles = len(bundles)
        for i, bundle in enumerate(bundles):
            if self.reporter: self.reporter.write( "%d/%d loading %s\n"%(i+1, n_bundles, bundle) )

            for service_id in [x.encode("ascii") for x in self.gtfsdb.service_ids()]:
                if service_ids is not None and service_id not in service_ids:
		    continue

                for fromv_label, tov_label, edge in self.bundle_to_boardalight_edges(bundle, service_id):
                    yield fromv_label, tov_label, edge

    def gtfsdb_to_headway_edges( self, maxtrips=None ):

        # load headways
        if self.reporter: self.reporter.write( "Loading headways trips to graph...\n" )
        for trip_id, start_time, end_time, headway_secs in self.gtfsdb.execute( "SELECT * FROM frequencies" ):
            service_id = list(self.gtfsdb.execute( "SELECT service_id FROM trips WHERE trip_id=?", (trip_id,) ))[0][0]
            service_id = service_id.encode('utf-8')

            hb = HeadwayBoard( service_id, self.sc, self.tz, 0, trip_id.encode('utf-8'), start_time, end_time, headway_secs )
            ha = HeadwayAlight( service_id, self.sc, self.tz, 0, trip_id.encode('utf-8'), start_time, end_time, headway_secs )

            stoptimes = list(self.gtfsdb.execute( "SELECT * FROM stop_times WHERE trip_id=? ORDER BY stop_sequence", (trip_id,)) )

            #add board edges
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stoptimes[:-1]:
                yield ( "sta-%s"%stop_id, "hwv-%s-%s-%s"%(self.agency_namespace,stop_id, trip_id), hb )

            #add alight edges
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stoptimes[1:]:
                yield ( "hwv-%s-%s-%s"%(self.agency_namespace,stop_id, trip_id), "sta-%s"%stop_id, ha )

            #add crossing edges
            for (trip_id1, arrival_time1, departure_time1, stop_id1, stop_sequence1, stop_dist_traveled1), (trip_id2, arrival_time2, departure_time2, stop_id2, stop_sequence2,stop_dist_traveled2) in cons(stoptimes):
                cr = Crossing()
                cr.add_crossing_time( trip_id1, (arrival_time2-departure_time1) )
                yield ( "hwv-%s-%s-%s"%(self.agency_namespace,stop_id1, trip_id1), "hwv-%s-%s-%s"%(self.agency_namespace,stop_id2, trip_id2), cr )

    def gtfsdb_to_transfer_edges( self ):

        # load transfers
        if self.reporter: self.reporter.write( "Loading transfers to graph...\n" )

        # keep track to avoid redundancies
        # this assumes that transfer relationships are bi-directional.
        # TODO this implementation is also incomplete - it's theoretically possible that
        # a transfers.txt table could contain "A,A,3,", which would mean you can't transfer
        # at A.
        seen = set([])
        for stop_id1, stop_id2, conn_type, min_transfer_time in self.gtfsdb.execute( "SELECT * FROM transfers" ):
            s1 = "sta-%s"%stop_id1
            s2 = "sta-%s"%stop_id2

            # TODO - what is the semantics of this? see note above
            if s1 == s2:
                continue

            key = ".".join(sorted([s1,s2]))
            if key not in seen:
                seen.add(key)
            else:
                continue

            assert conn_type == None or type(conn_type) == int
            if conn_type in (0, None): # This is a recommended transfer point between two routes
                if min_transfer_time in ("", None):
                    yield (s1, s2, Link())
                    yield (s2, s1, Link())
                else:
                    yield (s1, s2, ElapseTime(int(min_transfer_time)))
                    yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 1: # This is a timed transfer point between two routes
                yield (s1, s2, Link())
                yield (s2, s1, Link())
            elif conn_type == 2: # This transfer requires a minimum amount of time
                yield (s1, s2, ElapseTime(int(min_transfer_time)))
                yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 3: # Transfers are not possible between routes at this location.
                print "WARNING: Support for no-transfer (transfers.txt transfer_type=3) not implemented."

    def gtfsdb_to_edges( self, maxtrips=None, service_ids=None ):
        for edge_tuple in self.gtfsdb_to_scheduled_edges(maxtrips, service_ids=service_ids):
            yield edge_tuple

        for edge_tuple in self.gtfsdb_to_headway_edges(maxtrips):
            yield edge_tuple

        for edge_tuple in self.gtfsdb_to_transfer_edges():
            yield edge_tuple

def gdb_load_gtfsdb(gdb, agency_namespace, gtfsdb, cursor, agency_id=None, maxtrips=None, sample_date=None, reporter=sys.stdout):

    # determine which service periods run on the given day, if a day is given
    if sample_date is not None:
        sample_date = datetime.date( *parse_gtfs_date( sample_date ) )
	acceptable_service_ids = gtfsdb.service_periods( sample_date )
	print "Importing only service periods operating on %s: %s"%(sample_date, acceptable_service_ids)
    else:
        acceptable_service_ids = None

    compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter )
    c = gdb.get_cursor()
    v_added = set([])
    for fromv_label, tov_label, edge in compiler.gtfsdb_to_edges( maxtrips, service_ids=acceptable_service_ids ):
        if fromv_label not in v_added:
            gdb.add_vertex( fromv_label, c )
            v_added.add(fromv_label)
        if tov_label not in v_added:
            gdb.add_vertex( tov_label, c )
            v_added.add(tov_label)
        gdb.add_edge( fromv_label, tov_label, edge, c )
    gdb.commit()

def graph_load_gtfsdb( agency_namespace, gtfsdb, agency_id=None, maxtrips=None, reporter=sys.stdout ):
    compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter )

    gg = Graph()

    for fromv_label, tov_label, edge in compiler.gtfsdb_to_edges( maxtrips ):
        gg.add_vertex( fromv_label )
        gg.add_vertex( tov_label )
        gg.add_edge( fromv_label, tov_label, edge )

    return gg

def main():
    usage = """usage: python gdb_import_gtfs.py [options] <graphdb_filename> <gtfsdb_filename> [<agency_id>]"""
    parser = OptionParser(usage=usage)
    parser.add_option("-n", "--namespace", dest="namespace", default="0",
                      help="agency namespace")
    parser.add_option("-m", "--maxtrips", dest="maxtrips", default=None, help="maximum number of trips to load")
    parser.add_option("-d", "--date", dest="sample_date", default=None, help="only load transit running on a given day. YYYYMMDD" )

    (options, args) = parser.parse_args()

    if len(args) != 2:
        parser.print_help()
        exit(-1)

    graphdb_filename = args[0]
    gtfsdb_filename  = args[1]
    agency_id        = args[2] if len(args)==3 else None

    print "importing from gtfsdb '%s' into graphdb '%s'"%(gtfsdb_filename, graphdb_filename)

    gtfsdb = GTFSDatabase( gtfsdb_filename )
    gdb = GraphDatabase( graphdb_filename, overwrite=False )

    maxtrips = int(options.maxtrips) if options.maxtrips else None
    gdb_load_gtfsdb( gdb, options.namespace, gtfsdb, gdb.get_cursor(), agency_id, maxtrips=maxtrips, sample_date=options.sample_date)
    gdb.commit()

    print "done"

if __name__ == '__main__':
    main()
<commit_msg>having two cursors open at the same time does some bizarre things to graph compilation. this fixes that.<commit_after>from graphserver.core import Graph, TripBoard, HeadwayBoard, HeadwayAlight, Crossing, TripAlight, Timezone, Street, Link, ElapseTime
from optparse import OptionParser
from graphserver.graphdb import GraphDatabase
from graphserver.ext.gtfs.gtfsdb import GTFSDatabase, parse_gtfs_date
import sys
import pytz
from tools import service_calendar_from_timezone
import datetime

def cons(ary):
    for i in range(len(ary)-1):
        yield (ary[i], ary[i+1])

class GTFSGraphCompiler:
    def __init__(self, gtfsdb, agency_namespace, agency_id=None, reporter=None):
        self.gtfsdb = gtfsdb
        self.agency_namespace = agency_namespace
        self.reporter = reporter

        # get graphserver.core.Timezone and graphserver.core.ServiceCalendars from gtfsdb for agency with given agency_id
        timezone_name = gtfsdb.agency_timezone_name(agency_id)
        self.tz = Timezone.generate( timezone_name )
        if reporter: reporter.write( "constructing service calendar for timezone '%s'\n"%timezone_name )
        self.sc = service_calendar_from_timezone(gtfsdb, timezone_name )

    def bundle_to_boardalight_edges(self, bundle, service_id):
        """takes a bundle and yields a bunch of edges"""

        stop_time_bundles = bundle.stop_time_bundles(service_id)

        n_trips = len(bundle.trip_ids)

        # If there's less than two stations on this trip bundle, the trip bundle doesn't actually span two places
        if len(stop_time_bundles)<2:
            return

        # If there are no stop_times in a bundle on this service day, there is nothing to load
        if n_trips==0:
            return

        if self.reporter: self.reporter.write( "inserting %d trips with %d stop_time bundles on service_id '%s'\n"%(len(stop_time_bundles[0]),len(stop_time_bundles),service_id) )

        #add board edges
        for i, stop_time_bundle in enumerate(stop_time_bundles[:-1]):

            trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled = stop_time_bundle[0]

            if arrival_time != departure_time:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s-depart"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

                # construct the board/alight/dwell triangle for this patternstop
                patternstop_arrival_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

                dwell_crossing = Crossing()
                for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                    dwell_crossing.add_crossing_time( trip_id, departure_time-arrival_time )

                yield (patternstop_arrival_vx_name,
                           patternstop_vx_name,
                           dwell_crossing)

            else:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

            b = TripBoard(service_id, self.sc, self.tz, 0)
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                b.add_boarding( trip_id, departure_time, stop_sequence )

            yield ( "sta-%s"%stop_id, patternstop_vx_name, b )

        #add alight edges
        for i, stop_time_bundle in enumerate(stop_time_bundles[1:]):

            trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled = stop_time_bundle[0]

            if arrival_time != departure_time:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)
            else:
                patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)

            al = TripAlight(service_id, self.sc, self.tz, 0)
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stop_time_bundle:
                al.add_alighting( trip_id.encode('ascii'), arrival_time, stop_sequence )

            yield ( patternstop_vx_name, "sta-%s"%stop_id, al )

        # add crossing edges
        for i, (from_stop_time_bundle, to_stop_time_bundle) in enumerate(cons(stop_time_bundles)):

            trip_id, from_arrival_time, from_departure_time, stop_id, stop_sequence, stop_dist_traveled = from_stop_time_bundle[0]
            trip_id, to_arrival_time, to_departure_time, stop_id, stop_sequence, stop_dist_traveled = to_stop_time_bundle[0]

            if from_arrival_time!=from_departure_time:
                from_patternstop_vx_name = "psv-%s-%03d-%03d-%s-depart"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)
            else:
                from_patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i,service_id)

            if to_arrival_time!=to_departure_time:
                to_patternstop_vx_name = "psv-%s-%03d-%03d-%s-arrive"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)
            else:
                to_patternstop_vx_name = "psv-%s-%03d-%03d-%s"%(self.agency_namespace,bundle.pattern.pattern_id,i+1,service_id)

            crossing = Crossing()
            for i in range( len( from_stop_time_bundle ) ):
                trip_id, from_arrival_time, from_departure_time, stop_id, stop_sequence, stop_dist_traveled = from_stop_time_bundle[i]
                trip_id, to_arrival_time, to_departure_time, stop_id, stop_sequence, stop_dist_traveled = to_stop_time_bundle[i]
                crossing.add_crossing_time( trip_id, (to_arrival_time-from_departure_time) )

            yield ( from_patternstop_vx_name,
                    to_patternstop_vx_name,
                    crossing )

    def gtfsdb_to_scheduled_edges(self, maxtrips=None, service_ids=None):

        # compile trip bundles from gtfsdb
        if self.reporter: self.reporter.write( "Compiling trip bundles...\n" )
        bundles = self.gtfsdb.compile_trip_bundles(maxtrips=maxtrips, reporter=self.reporter)

        # load bundles to graph
        if self.reporter: self.reporter.write( "Loading trip bundles into graph...\n" )
        n_bundles = len(bundles)
        for i, bundle in enumerate(bundles):
            if self.reporter: self.reporter.write( "%d/%d loading %s\n"%(i+1, n_bundles, bundle) )

            for service_id in [x.encode("ascii") for x in self.gtfsdb.service_ids()]:
                if service_ids is not None and service_id not in service_ids:
		    continue

                for fromv_label, tov_label, edge in self.bundle_to_boardalight_edges(bundle, service_id):
                    yield fromv_label, tov_label, edge

    def gtfsdb_to_headway_edges( self, maxtrips=None ):

        # load headways
        if self.reporter: self.reporter.write( "Loading headways trips to graph...\n" )
        for trip_id, start_time, end_time, headway_secs in self.gtfsdb.execute( "SELECT * FROM frequencies" ):
            service_id = list(self.gtfsdb.execute( "SELECT service_id FROM trips WHERE trip_id=?", (trip_id,) ))[0][0]
            service_id = service_id.encode('utf-8')

            hb = HeadwayBoard( service_id, self.sc, self.tz, 0, trip_id.encode('utf-8'), start_time, end_time, headway_secs )
            ha = HeadwayAlight( service_id, self.sc, self.tz, 0, trip_id.encode('utf-8'), start_time, end_time, headway_secs )

            stoptimes = list(self.gtfsdb.execute( "SELECT * FROM stop_times WHERE trip_id=? ORDER BY stop_sequence", (trip_id,)) )

            #add board edges
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stoptimes[:-1]:
                yield ( "sta-%s"%stop_id, "hwv-%s-%s-%s"%(self.agency_namespace,stop_id, trip_id), hb )

            #add alight edges
            for trip_id, arrival_time, departure_time, stop_id, stop_sequence, stop_dist_traveled in stoptimes[1:]:
                yield ( "hwv-%s-%s-%s"%(self.agency_namespace,stop_id, trip_id), "sta-%s"%stop_id, ha )

            #add crossing edges
            for (trip_id1, arrival_time1, departure_time1, stop_id1, stop_sequence1, stop_dist_traveled1), (trip_id2, arrival_time2, departure_time2, stop_id2, stop_sequence2,stop_dist_traveled2) in cons(stoptimes):
                cr = Crossing()
                cr.add_crossing_time( trip_id1, (arrival_time2-departure_time1) )
                yield ( "hwv-%s-%s-%s"%(self.agency_namespace,stop_id1, trip_id1), "hwv-%s-%s-%s"%(self.agency_namespace,stop_id2, trip_id2), cr )

    def gtfsdb_to_transfer_edges( self ):

        # load transfers
        if self.reporter: self.reporter.write( "Loading transfers to graph...\n" )

        # keep track to avoid redundancies
        # this assumes that transfer relationships are bi-directional.
        # TODO this implementation is also incomplete - it's theoretically possible that
        # a transfers.txt table could contain "A,A,3,", which would mean you can't transfer
        # at A.
        seen = set([])
        for stop_id1, stop_id2, conn_type, min_transfer_time in self.gtfsdb.execute( "SELECT * FROM transfers" ):
            s1 = "sta-%s"%stop_id1
            s2 = "sta-%s"%stop_id2

            # TODO - what is the semantics of this? see note above
            if s1 == s2:
                continue

            key = ".".join(sorted([s1,s2]))
            if key not in seen:
                seen.add(key)
            else:
                continue

            assert conn_type == None or type(conn_type) == int
            if conn_type in (0, None): # This is a recommended transfer point between two routes
                if min_transfer_time in ("", None):
                    yield (s1, s2, Link())
                    yield (s2, s1, Link())
                else:
                    yield (s1, s2, ElapseTime(int(min_transfer_time)))
                    yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 1: # This is a timed transfer point between two routes
                yield (s1, s2, Link())
                yield (s2, s1, Link())
            elif conn_type == 2: # This transfer requires a minimum amount of time
                yield (s1, s2, ElapseTime(int(min_transfer_time)))
                yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 3: # Transfers are not possible between routes at this location.
                print "WARNING: Support for no-transfer (transfers.txt transfer_type=3) not implemented."

    def gtfsdb_to_edges( self, maxtrips=None, service_ids=None ):
        for edge_tuple in self.gtfsdb_to_scheduled_edges(maxtrips, service_ids=service_ids):
            yield edge_tuple

        for edge_tuple in self.gtfsdb_to_headway_edges(maxtrips):
            yield edge_tuple

        for edge_tuple in self.gtfsdb_to_transfer_edges():
            yield edge_tuple

def gdb_load_gtfsdb(gdb, agency_namespace, gtfsdb, cursor, agency_id=None, maxtrips=None, sample_date=None, reporter=sys.stdout):

    # determine which service periods run on the given day, if a day is given
    if sample_date is not None:
        sample_date = datetime.date( *parse_gtfs_date( sample_date ) )
	acceptable_service_ids = gtfsdb.service_periods( sample_date )
	print "Importing only service periods operating on %s: %s"%(sample_date, acceptable_service_ids)
    else:
        acceptable_service_ids = None

    compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter )
    v_added = set([])
    for fromv_label, tov_label, edge in compiler.gtfsdb_to_edges( maxtrips, service_ids=acceptable_service_ids ):
        if fromv_label not in v_added:
            gdb.add_vertex( fromv_label, cursor )
            v_added.add(fromv_label)
        if tov_label not in v_added:
            gdb.add_vertex( tov_label, cursor )
            v_added.add(tov_label)
        gdb.add_edge( fromv_label, tov_label, edge, cursor )

def graph_load_gtfsdb( agency_namespace, gtfsdb, agency_id=None, maxtrips=None, reporter=sys.stdout ):
    compiler = GTFSGraphCompiler( gtfsdb, agency_namespace, agency_id, reporter )

    gg = Graph()

    for fromv_label, tov_label, edge in compiler.gtfsdb_to_edges( maxtrips ):
        gg.add_vertex( fromv_label )
        gg.add_vertex( tov_label )
        gg.add_edge( fromv_label, tov_label, edge )

    return gg

def main():
    usage = """usage: python gdb_import_gtfs.py [options] <graphdb_filename> <gtfsdb_filename> [<agency_id>]"""
    parser = OptionParser(usage=usage)
    parser.add_option("-n", "--namespace", dest="namespace", default="0",
                      help="agency namespace")
    parser.add_option("-m", "--maxtrips", dest="maxtrips", default=None, help="maximum number of trips to load")
    parser.add_option("-d", "--date", dest="sample_date", default=None, help="only load transit running on a given day. YYYYMMDD" )

    (options, args) = parser.parse_args()

    if len(args) != 2:
        parser.print_help()
        exit(-1)

    graphdb_filename = args[0]
    gtfsdb_filename  = args[1]
    agency_id        = args[2] if len(args)==3 else None

    print "importing from gtfsdb '%s' into graphdb '%s'"%(gtfsdb_filename, graphdb_filename)

    gtfsdb = GTFSDatabase( gtfsdb_filename )
    gdb = GraphDatabase( graphdb_filename, overwrite=False )

    maxtrips = int(options.maxtrips) if options.maxtrips else None
    gdb_load_gtfsdb( gdb, options.namespace, gtfsdb, gdb.get_cursor(), agency_id, maxtrips=maxtrips, sample_date=options.sample_date)
    gdb.commit()

    print "done"

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
API harvester for ClinicalTrials.gov for the SHARE Notification Service
"""

#!/usr/bin/env python
from __future__ import unicode_literals

import time
import logging
import datetime

from lxml import etree

from dateutil.parser import *

from scrapi.base import XMLHarvester
from scrapi.base.schemas import default_name_parser
from scrapi.linter.document import RawDocument

logger = logging.getLogger(__name__)

class ClinicalTrialsHarvester(XMLHarvester):

    DEFAULT_ENCODING = 'UTF-8'
    record_encoding = None

    @property
    def name(self):
        return 'clinicaltrials'

    @property
    def namespaces(self):
        return None

    def copy_to_unicode(self, element):
        encoding = self.record_encoding or self.DEFAULT_ENCODING
        element = ''.join(element)
        if isinstance(element, unicode):
            return element
        else:
            return unicode(element, encoding=encoding)

    def harvest(self, days_back=1):
        """ First, get a list of all recently updated study urls,
        then get the xml one by one and save it into a list
        of docs including other information """

        today = datetime.date.today()
        start_date = today - datetime.timedelta(days_back)

        month = today.strftime('%m')
        day = today.strftime('%d')
        year = today.strftime('%Y')

        y_month = start_date.strftime('%m')
        y_day = start_date.strftime('%d')
        y_year = start_date.strftime('%Y')

        base_url = 'http://clinicaltrials.gov/ct2/results?lup_s='
        url_end = '{}%2F{}%2F{}%2F&lup_e={}%2F{}%2F{}&displayxml=true'.\
            format(y_month, y_day, y_year, month, day, year)

        url = base_url + url_end

        # grab the total number of studies
        initial_request = requests.get(url)
        record_encoding = initial_request.encoding
        initial_request_xml = etree.XML(initial_request.content)
        count = int(initial_request_xml.xpath('//search_results/@count')[0])
        xml_list = []
        if int(count) > 0:
            # get a new url with all results in it
            url = url + '&count=' + str(count)
            total_requests = requests.get(url)
            initial_doc = etree.XML(total_requests.content)

            # make a list of urls from that full list of studies
            study_urls = []
            for study in initial_doc.xpath('//clinical_study'):
                study_urls.append(study.xpath('url/node()')[0] + '?displayxml=true')

            # grab each of those urls for full content
            logger.info("There are {} urls to harvest - be patient...".format(len(study_urls)))
            count = 0
            official_count = 0
            for study_url in study_urls:
                try:
                    content = requests.get(study_url)
                except requests.exceptions.ConnectionError as e:
                    logger.info('Connection error: {}, wait a bit...'.format(e))
                    time.sleep(30)
                    continue
                doc = etree.XML(content.content)
                record = etree.tostring(doc, encoding=record_encoding)
                doc_id = doc.xpath('//nct_id/node()')[0]
                xml_list.append(RawDocument({
                    'doc': record,
                    'source': self.name,
                    'docID': self.copy_to_unicode(doc_id),
                    'filetype': 'xml',
                }))
                official_count += 1
                count += 1
                if count % 100 == 0:
                    logger.info("You've requested {} studies, keep going!".format(official_count))
                    count = 0
                time.sleep(1)

        return xml_list

    @property
    def schema(self):
        return {
            "contributors": ('//overall_official/last_name/node()', default_name_parser),
            "id": {
                "url": "//required_header/url/node()",
                "serviceID": "//nct_id/node()"
            },
            "tags": ("//keyword/node()", lambda tags: [unicode(tag.lower()) for tag in tags]),
            "dateUpdated": ("lastchanged_date/node()", lambda x: parse(x).isoformat()),
            "title": ('//official_title/node()', '//brief_title/node()', lambda x, y: x or y or ''),
            "description": ('//brief_summary/textblock/node()', '//brief_summary/textblock/node()', lambda x, y: x or y or ''),
            "properties": {
                'oversightAuthority': '//oversight_info/authority/node()',
                'studyDesign': '//study_design/node()',
                'numberOfArms': '//number_of_arms/node()',
                'source': '//source/node()',
                'verificationDate': '//verification_date/node()',
                'lastChanged': '//lastchanged_date/node()',
                'condition': '//condition/node()',
                'verificationDate': '//verification_date/node()',
                'lastChanged': '//lastchanged_date/node()',
                'status': '//status/node()',
                'locationCountries': '//location_countries/country/node()',
                'isFDARegulated': '//is_fda_regulated/node()',
                'isSection801': '//is_section_801/node()',
                'hasExpandedAccess': '//has_expanded_access/node()',
                'sponsors': {
                    'agency': '//lead_sponsor/agency/node()',
                    'agencyClass': '//lead_sponsor/agency_class/node()'
                },
                'primaryOutcome': {
                    'measure': '//primary_outcome/measure/node()',
                    'timeFrame': '//primary_outcome/time_frame/node()',
                    'safetyIssue': '//primary_outcome/safety_issue/node()'
                },
                'secondaryOutcomes': '//secondary_outcome/node()',
                'enrollment': '//enrollment/node()',
                'armGroup': '//arm_group/node()',
                'intervention': '//intervention/node()',
                'eligibility': '//elligibility/node()',
                'link': '//link/node()',
                'responsible_party': '//responsible_party'
            }
        }

h = ClinicalTrialsHarvester()

harvest = h.harvest
normalize = h.normalize
<commit_msg>Fix import<commit_after>"""
API harvester for ClinicalTrials.gov for the SHARE Notification Service
"""

#!/usr/bin/env python
from __future__ import unicode_literals

import time
import logging
import datetime

from lxml import etree

from dateutil.parser import *

from scrapi import requests
from scrapi.base import XMLHarvester
from scrapi.linter.document import RawDocument
from scrapi.base.schemas import default_name_parser

logger = logging.getLogger(__name__)

class ClinicalTrialsHarvester(XMLHarvester):

    DEFAULT_ENCODING = 'UTF-8'
    record_encoding = None

    @property
    def name(self):
        return 'clinicaltrials'

    @property
    def namespaces(self):
        return None

    def copy_to_unicode(self, element):
        encoding = self.record_encoding or self.DEFAULT_ENCODING
        element = ''.join(element)
        if isinstance(element, unicode):
            return element
        else:
            return unicode(element, encoding=encoding)

    def harvest(self, days_back=1):
        """ First, get a list of all recently updated study urls,
        then get the xml one by one and save it into a list
        of docs including other information """

        today = datetime.date.today()
        start_date = today - datetime.timedelta(days_back)

        month = today.strftime('%m')
        day = today.strftime('%d')
        year = today.strftime('%Y')

        y_month = start_date.strftime('%m')
        y_day = start_date.strftime('%d')
        y_year = start_date.strftime('%Y')

        base_url = 'http://clinicaltrials.gov/ct2/results?lup_s='
        url_end = '{}%2F{}%2F{}%2F&lup_e={}%2F{}%2F{}&displayxml=true'.\
            format(y_month, y_day, y_year, month, day, year)

        url = base_url + url_end

        # grab the total number of studies
        initial_request = requests.get(url)
        record_encoding = initial_request.encoding
        initial_request_xml = etree.XML(initial_request.content)
        count = int(initial_request_xml.xpath('//search_results/@count')[0])
        xml_list = []
        if int(count) > 0:
            # get a new url with all results in it
            url = url + '&count=' + str(count)
            total_requests = requests.get(url)
            initial_doc = etree.XML(total_requests.content)

            # make a list of urls from that full list of studies
            study_urls = []
            for study in initial_doc.xpath('//clinical_study'):
                study_urls.append(study.xpath('url/node()')[0] + '?displayxml=true')

            # grab each of those urls for full content
            logger.info("There are {} urls to harvest - be patient...".format(len(study_urls)))
            count = 0
            official_count = 0
            for study_url in study_urls:
                try:
                    content = requests.get(study_url)
                except requests.exceptions.ConnectionError as e:
                    logger.info('Connection error: {}, wait a bit...'.format(e))
                    time.sleep(30)
                    continue
                doc = etree.XML(content.content)
                record = etree.tostring(doc, encoding=record_encoding)
                doc_id = doc.xpath('//nct_id/node()')[0]
                xml_list.append(RawDocument({
                    'doc': record,
                    'source': self.name,
                    'docID': self.copy_to_unicode(doc_id),
                    'filetype': 'xml',
                }))
                official_count += 1
                count += 1
                if count % 100 == 0:
                    logger.info("You've requested {} studies, keep going!".format(official_count))
                    count = 0
                time.sleep(1)

        return xml_list

    @property
    def schema(self):
        return {
            "contributors": ('//overall_official/last_name/node()', default_name_parser),
            "id": {
                "url": "//required_header/url/node()",
                "serviceID": "//nct_id/node()"
            },
            "tags": ("//keyword/node()", lambda tags: [unicode(tag.lower()) for tag in tags]),
            "dateUpdated": ("lastchanged_date/node()", lambda x: parse(x).isoformat()),
            "title": ('//official_title/node()', '//brief_title/node()', lambda x, y: x or y or ''),
            "description": ('//brief_summary/textblock/node()', '//brief_summary/textblock/node()', lambda x, y: x or y or ''),
            "properties": {
                'oversightAuthority': '//oversight_info/authority/node()',
                'studyDesign': '//study_design/node()',
                'numberOfArms': '//number_of_arms/node()',
                'source': '//source/node()',
                'verificationDate': '//verification_date/node()',
                'lastChanged': '//lastchanged_date/node()',
                'condition': '//condition/node()',
                'verificationDate': '//verification_date/node()',
                'lastChanged': '//lastchanged_date/node()',
                'status': '//status/node()',
                'locationCountries': '//location_countries/country/node()',
                'isFDARegulated': '//is_fda_regulated/node()',
                'isSection801': '//is_section_801/node()',
                'hasExpandedAccess': '//has_expanded_access/node()',
                'sponsors': {
                    'agency': '//lead_sponsor/agency/node()',
                    'agencyClass': '//lead_sponsor/agency_class/node()'
                },
                'primaryOutcome': {
                    'measure': '//primary_outcome/measure/node()',
                    'timeFrame': '//primary_outcome/time_frame/node()',
                    'safetyIssue': '//primary_outcome/safety_issue/node()'
                },
                'secondaryOutcomes': '//secondary_outcome/node()',
                'enrollment': '//enrollment/node()',
                'armGroup': '//arm_group/node()',
                'intervention': '//intervention/node()',
                'eligibility': '//elligibility/node()',
                'link': '//link/node()',
                'responsible_party': '//responsible_party'
            }
        }

h = ClinicalTrialsHarvester()

harvest = h.harvest
normalize = h.normalize
<|endoftext|>