<commit_before>import logging

from pylons import g, request
from tg import flash, redirect, session, config
from openid.consumer import consumer

from pyforge import model as M

log = logging.getLogger(__name__)

def verify_oid(oid_url, failure_redirect=None, return_to=None,
                  **kw):
    '''Step 1 of OID verification -- redirect to provider site'''
    log.info('Trying to login via %s', oid_url)
    realm = config.get('openid.realm', 'http://localhost:8080/')
    return_to = realm + 'auth/' + return_to
    oidconsumer = consumer.Consumer(g.oid_session(), g.oid_store)
    try:
        req = oidconsumer.begin(oid_url)
    except consumer.DiscoveryFailure, ex:
        log.exception('Error in openid login')
        flash(str(ex[0]), 'error')
        redirect(failure_redirect)
    if req is None: # pragma no cover
        flash('No openid services found for %s' % oid_url,
              'error')
        redirect(failure_redirect)
    if req.shouldSendRedirect():
        redirect_url = req.redirectURL(
            realm, return_to, False)
        log.info('Redirecting to %r', redirect_url)
        session.save()
        redirect(redirect_url)
    else:
        return dict(kw, form=req.formMarkup(realm, return_to=return_to))

def process_oid(failure_redirect=None):
    oidconsumer = consumer.Consumer(g.oid_session(), g.oid_store)
    info = oidconsumer.complete(request.params, request.url)
    display_identifier = info.getDisplayIdentifier() or info.identity_url
    if info.status == consumer.FAILURE and display_identifier:
        # In the case of failure, if info is non-None, it is the
        # URL that we were verifying. We include it in the error
        # message to help the user figure out what happened.
        fmt = "Verification of %s failed: %s"
        flash(fmt % (display_identifier, info.message), 'error')
        redirect(failure_redirect)
    elif info.status == consumer.SUCCESS:
        # Success means that the transaction completed without
        # error. If info is None, it means that the user cancelled
        # the verification.
        css_class = 'alert'

        # This is a successful verification attempt. If this
        # was a real application, we would do our login,
        # comment posting, etc. here.
        fmt = "You have successfully verified %s as your identity."
        message = fmt % display_identifier
        if info.endpoint.canonicalID:
            # You should authorize i-name users by their canonicalID,
            # rather than their more human-friendly identifiers.  That
            # way their account with you is not compromised if their
            # i-name registration expires and is bought by someone else.
            message += ("  This is an i-name, and its persistent ID is %s"
                        % info.endpoint.canonicalID )
        flash(message, 'info')
    elif info.status == consumer.CANCEL:
        # cancelled
        message = 'Verification cancelled'
        flash(message, 'error')
        redirect(failure_redirect)
    elif info.status == consumer.SETUP_NEEDED:
        if info.setup_url:
            message = 'Setup needed' % info.setup_url
        else:
            # This means auth didn't succeed, but you're welcome to try
            # non-immediate mode.
            message = 'Setup needed'
        flash(message, 'error')
        redirect(failure_redirect)
    else:
        # Either we don't understand the code or there is no
        # openid_url included with the error. Give a generic
        # failure message. The library should supply debug
        # information in a log.
        message = 'Verification failed.'
        flash(message, 'error')
        redirect(failure_redirect)
    session.save()
    oid_obj = M.OpenId.upsert(info.identity_url, display_identifier=display_identifier)
    return oid_obj
<commit_msg>Test cleanups: add logging hook for openid.oidutil<commit_after>import logging

from pylons import g, request
from tg import flash, redirect, session, config
from openid.consumer import consumer
from pyforge import model as M

log = logging.getLogger(__name__)

# openid.oidutil outputs all logging to STDERR unless otherwise configured.
# We follow the openid.oidutil instructions to install our own logging hook.
from openid import oidutil
oidutil.log = log.info

def verify_oid(oid_url, failure_redirect=None, return_to=None,
                  **kw):
    '''Step 1 of OID verification -- redirect to provider site'''
    log.info('Trying to login via %s', oid_url)
    realm = config.get('openid.realm', 'http://localhost:8080/')
    return_to = realm + 'auth/' + return_to
    oidconsumer = consumer.Consumer(g.oid_session(), g.oid_store)
    try:
        req = oidconsumer.begin(oid_url)
    except consumer.DiscoveryFailure, ex:
        log.exception('Error in openid login')
        flash(str(ex[0]), 'error')
        redirect(failure_redirect)
    if req is None: # pragma no cover
        flash('No openid services found for %s' % oid_url,
              'error')
        redirect(failure_redirect)
    if req.shouldSendRedirect():
        redirect_url = req.redirectURL(
            realm, return_to, False)
        log.info('Redirecting to %r', redirect_url)
        session.save()
        redirect(redirect_url)
    else:
        return dict(kw, form=req.formMarkup(realm, return_to=return_to))

def process_oid(failure_redirect=None):
    oidconsumer = consumer.Consumer(g.oid_session(), g.oid_store)
    info = oidconsumer.complete(request.params, request.url)
    display_identifier = info.getDisplayIdentifier() or info.identity_url
    if info.status == consumer.FAILURE and display_identifier:
        # In the case of failure, if info is non-None, it is the
        # URL that we were verifying. We include it in the error
        # message to help the user figure out what happened.
        fmt = "Verification of %s failed: %s"
        flash(fmt % (display_identifier, info.message), 'error')
        redirect(failure_redirect)
    elif info.status == consumer.SUCCESS:
        # Success means that the transaction completed without
        # error. If info is None, it means that the user cancelled
        # the verification.
        css_class = 'alert'

        # This is a successful verification attempt. If this
        # was a real application, we would do our login,
        # comment posting, etc. here.
        fmt = "You have successfully verified %s as your identity."
        message = fmt % display_identifier
        if info.endpoint.canonicalID:
            # You should authorize i-name users by their canonicalID,
            # rather than their more human-friendly identifiers.  That
            # way their account with you is not compromised if their
            # i-name registration expires and is bought by someone else.
            message += ("  This is an i-name, and its persistent ID is %s"
                        % info.endpoint.canonicalID )
        flash(message, 'info')
    elif info.status == consumer.CANCEL:
        # cancelled
        message = 'Verification cancelled'
        flash(message, 'error')
        redirect(failure_redirect)
    elif info.status == consumer.SETUP_NEEDED:
        if info.setup_url:
            message = 'Setup needed' % info.setup_url
        else:
            # This means auth didn't succeed, but you're welcome to try
            # non-immediate mode.
            message = 'Setup needed'
        flash(message, 'error')
        redirect(failure_redirect)
    else:
        # Either we don't understand the code or there is no
        # openid_url included with the error. Give a generic
        # failure message. The library should supply debug
        # information in a log.
        message = 'Verification failed.'
        flash(message, 'error')
        redirect(failure_redirect)
    session.save()
    oid_obj = M.OpenId.upsert(info.identity_url, display_identifier=display_identifier)
    return oid_obj
<|endoftext|>


<commit_before>

#machine_engine.py

from sqlalchemy import create_engine
from machine_config import DATABASE_URI

engine = create_engine( DATABASE_URI,
    client_encoding = 'utf8', # uses utf8
    echo=True) # prints postgres activity to stdout
<commit_msg>updated engine for new config keys<commit_after>

#machine_engine.py

from sqlalchemy import create_engine
from demo_config import DBASE_URI

engine = create_engine( DBASE_URI,
    client_encoding = 'utf8', # uses utf8
    echo=True) # prints postgres activity to stdout
<|endoftext|>


<commit_before># I'M BUSY! Plugin by Curs3d #
##############################
# Concept by CookieManors :D #
# http://bit.ly/1GnNPW8      #
##############################
# This plugin permits users to
# send a command that renders
# them "busy", not letting them
# to get tpa requests or direct
# messages, except from console.
# On restart, all busy data will
# be cleared.

from helpers import *
from basecommands import simplecommand
import org.bukkit.command.Command as Command
from traceback import format_exc as trace

busy_players = []

def unclear(sender):
    msg(sender, "Umm, what? Sorry, directions unclear, got head stuck in washing machine")

@hook.command("busy",
    aliases = ["focus"],
    usage = "/<command> <on|off|status>",
    description = "Sets busy mode on, you cannot recieve tpas and MSGs"
    )
def on_busy_command(sender, cmd, label, args):

    if not is_player(sender):
       msg(sender, "Sorry, Console cannot be busy")
       return True

    plugin_header(recipient = sender, name = "I'M BUSY!")

    if not sender.hasPermission("utils.busy.allowed"):
        noperm(sender)
        return True

    if len(args) == 0:
        msg(sender, "This plugin allows being busy, and when turned on you will not receive any direct messages or tpa requests.")
        msg(sender, "\nCommands:")
        msg(sender, "/busy on: turns on busy mode")
        msg(sender, "/busy off: turns off busy mode")
        msg(sender, "/busy status [player]: shows your or [player]'s current busy status.")

    elif len(args) == 1:
        if args[0] == "on":
            if sender.getName() in busy_players:
                msg(sender, "You cannot be even more focused than this without being a jedi!")
            else:
                busy_players.append(sender.getName())
                broadcast(None, "&c[&2Busy&c] &fNow busy: %s&r, don't even TRY bothering them!" % sender.getDisplayName())

        elif args[0] == "off":
            try:
                busy_players.remove(sender.getName())
                msg(sender, "Master has sent /busy command, %s&r is freeee of bothering!" % sender.getDisplayName())
            except ValueError:
                msg(sender, "You are not busy! You cannot be even less busy! Are you perhaps bored?")

        elif args[0] == "status":
            if sender.getName() in busy_players:
                msg(sender, "You are super-duper busy and concentrated right now. Think, think, think!")
            else:
                msg(sender, "You are completely unable to focus right now.")

        else:
            unclear(sender)
            return False

    elif len(args) == 2 and args[0] == "status":
        target = server.getPlayer(args[1])
        if target is None:
            msg(sender, "That player is not online, I doubt they are busy.")
        elif target.getName() in busy_players:
            msg(sender, "Yes, %s&r is busy. Shhh..." % target.getDisplayName())
        else:
            msg(sender, "No, you're good. Feel free to chat with %s&r!" % target.getDisplayName())

    else:
        unclear(sender)
        return False
    return True

@hook.event("player.PlayerQuitEvent", "lowest")
def on_player_leave(event):
    try:
        busy_players.remove(event.getPlayer().getName())
    except:
        pass

#---- Dicode for catching any bothering of busy people ----

reply_targets = {}
override_perm = "utils.imbusy.override"

def whisper(sender, target_name):
    target = server.getPlayer(target_name)

    if target is not None:
        if target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
            msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
            return False

        reply_targets[sender.getName()] = target.getName()

        # allow the target to reply regardless of sender being busy
        if target.getName() in reply_targets:
            del reply_targets[target.getName()]
    return True

def reply(sender):
    if sender.getName() in reply_targets:
        target = server.getPlayer(reply_targets[sender.getName()])
        if target is not None:
            if target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
                msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
                return False

            # allow the target to reply regardless of sender being busy
            if target.getName() in reply_targets:
                del reply_targets[target.getName()]
    return True

class CommandWrapper(Command):

    def __init__(self, wrapped, checker):
        Command.__init__(self, wrapped.getName())
        self.setDescription(wrapped.getDescription())
        self.setPermission(wrapped.getPermission())
        self.setUsage(wrapped.getUsage())
        self.setAliases(wrapped.getAliases())
        self.wrapped = wrapped
        self.checker = checker

    def execute(self, sender, label, args):
        info("/" + self.getLabel() + " executed by " + sender.getName())
        try:
            if self.checker(sender, args):
                return self.wrapped.execute(sender, label, args)
        except:
            error(trace())
        return True

    def tabComplete(self, sender, alias, args):
        return self.wrapped.tabComplete(sender, alias, args)

def msg_command_checker(sender, args):
    return len(args) <= 1 or whisper(sender, args[0])

def reply_command_checker(sender, args):
    return len(args) == 0 or reply(sender)

def tpa_command_checker(sender, args):
    if len(args) == 0:
        return True
    target = server.getPlayer(args[0])
    if target is not None and target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
        msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
        return False
    return True

def tpahere_command_checker(sender, args):
    return tpa_command_checker(sender, args)

@hook.event("player.PlayerCommandPreprocessEvent", "monitor")
def on_player_command_preprocess(event):
    message = event.getMessage().split(" ")
    if len(message) > 1 and message[0].lower() in ("/tell", "/minecraft:tell") and not whisper(event.getPlayer(), message[1]):
        event.setCancelled(True)

@hook.enable
def replace_ess_commands():

    try:
        info("[imbusy] Wrapping ess commands")
        map_field = server.getPluginManager().getClass().getDeclaredField("commandMap")
        map_field.setAccessible(True)
        command_map = map_field.get(server.getPluginManager())

        commands_field = command_map.getClass().getDeclaredField("knownCommands")
        commands_field.setAccessible(True)
        map = commands_field.get(command_map)

        ess_msg_cmd = map.get("essentials:msg")
        ess_reply_cmd = map.get("essentials:reply")
        ess_tpa_cmd = map.get("essentials:tpa")
        ess_tpahere_cmd = map.get("essentials:tpahere")

        msg_cmd_wrapper = CommandWrapper(ess_msg_cmd, msg_command_checker)
        reply_cmd_wrapper = CommandWrapper(ess_reply_cmd, reply_command_checker)
        tpa_cmd_wrapper = CommandWrapper(ess_tpa_cmd, tpa_command_checker)
        tpahere_cmd_wrapper = CommandWrapper(ess_tpahere_cmd, tpahere_command_checker)

        iterator = map.entrySet().iterator()
        while iterator.hasNext():
            entry = iterator.next()
            value = entry.getValue()
            if value is ess_msg_cmd:
                entry.setValue(msg_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_reply_cmd:
                entry.setValue(reply_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_tpa_cmd:
                entry.setValue(tpa_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_tpahere_cmd:
                entry.setValue(tpahere_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())

    except:
        error("[Imbusy] Failed to wrap essentials commands")
        error(trace())
<commit_msg>Changed broadcast message, removed two debugs<commit_after># I'M BUSY! Plugin by Curs3d #
##############################
# Concept by CookieManors :D #
# http://bit.ly/1GnNPW8      #
##############################
# This plugin permits users to
# send a command that renders
# them "busy", not letting them
# to get tpa requests or direct
# messages, except from console.
# On restart, all busy data will
# be cleared.

from helpers import *
from basecommands import simplecommand
import org.bukkit.command.Command as Command
from traceback import format_exc as trace

busy_players = []

def unclear(sender):
    msg(sender, "Umm, what? Sorry, directions unclear, got head stuck in washing machine")

@hook.command("busy",
    aliases = ["focus"],
    usage = "/<command> <on|off|status>",
    description = "Sets busy mode on, you cannot recieve tpas and MSGs"
    )
def on_busy_command(sender, cmd, label, args):

    if not is_player(sender):
       msg(sender, "Sorry, Console cannot be busy")
       return True

    plugin_header(recipient = sender, name = "I'M BUSY!")

    if not sender.hasPermission("utils.busy.allowed"):
        noperm(sender)
        return True

    if len(args) == 0:
        msg(sender, "This plugin allows being busy, and when turned on you will not receive any direct messages or tpa requests.")
        msg(sender, "\nCommands:")
        msg(sender, "/busy on: turns on busy mode")
        msg(sender, "/busy off: turns off busy mode")
        msg(sender, "/busy status [player]: shows your or [player]'s current busy status.")

    elif len(args) == 1:
        if args[0] == "on":
            if sender.getName() in busy_players:
                msg(sender, "You cannot be even more focused than this without being a jedi!")
            else:
                busy_players.append(sender.getName())
                broadcast(None, "&c[&2Busy&c] %s&r is now busy, don't even TRY bothering them!" % sender.getDisplayName())

        elif args[0] == "off":
            try:
                busy_players.remove(sender.getName())
                msg(sender, "Master has sent /busy command, %s&r is freeee of bothering!" % sender.getDisplayName())
            except ValueError:
                msg(sender, "You are not busy! You cannot be even less busy! Are you perhaps bored?")

        elif args[0] == "status":
            if sender.getName() in busy_players:
                msg(sender, "You are super-duper busy and concentrated right now. Think, think, think!")
            else:
                msg(sender, "You are completely unable to focus right now.")

        else:
            unclear(sender)
            return False

    elif len(args) == 2 and args[0] == "status":
        target = server.getPlayer(args[1])
        if target is None:
            msg(sender, "That player is not online, I doubt they are busy.")
        elif target.getName() in busy_players:
            msg(sender, "Yes, %s&r is busy. Shhh..." % target.getDisplayName())
        else:
            msg(sender, "No, you're good. Feel free to chat with %s&r!" % target.getDisplayName())

    else:
        unclear(sender)
        return False
    return True

@hook.event("player.PlayerQuitEvent", "lowest")
def on_player_leave(event):
    try:
        busy_players.remove(event.getPlayer().getName())
    except:
        pass

#---- Dicode for catching any bothering of busy people ----

reply_targets = {}
override_perm = "utils.imbusy.override"

def whisper(sender, target_name):
    target = server.getPlayer(target_name)

    if target is not None:
        if target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
            msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
            return False

        reply_targets[sender.getName()] = target.getName()

        # allow the target to reply regardless of sender being busy
        if target.getName() in reply_targets:
            del reply_targets[target.getName()]
    return True

def reply(sender):
    if sender.getName() in reply_targets:
        target = server.getPlayer(reply_targets[sender.getName()])
        if target is not None:
            if target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
                msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
                return False

            # allow the target to reply regardless of sender being busy
            if target.getName() in reply_targets:
                del reply_targets[target.getName()]
    return True

class CommandWrapper(Command):

    def __init__(self, wrapped, checker):
        Command.__init__(self, wrapped.getName())
        self.setDescription(wrapped.getDescription())
        self.setPermission(wrapped.getPermission())
        self.setUsage(wrapped.getUsage())
        self.setAliases(wrapped.getAliases())
        self.wrapped = wrapped
        self.checker = checker

    def execute(self, sender, label, args):
        try:
            if self.checker(sender, args):
                return self.wrapped.execute(sender, label, args)
        except:
            error(trace())
        return True

    def tabComplete(self, sender, alias, args):
        return self.wrapped.tabComplete(sender, alias, args)

def msg_command_checker(sender, args):
    return len(args) <= 1 or whisper(sender, args[0])

def reply_command_checker(sender, args):
    return len(args) == 0 or reply(sender)

def tpa_command_checker(sender, args):
    if len(args) == 0:
        return True
    target = server.getPlayer(args[0])
    if target is not None and target is not sender and not sender.hasPermission(override_perm) and target.getName() in busy_players:
        msg(sender, "&c[&fBUSY&c] %s&r is busy!" % target.getDisplayName())
        return False
    return True

def tpahere_command_checker(sender, args):
    return tpa_command_checker(sender, args)

@hook.event("player.PlayerCommandPreprocessEvent", "monitor")
def on_player_command_preprocess(event):
    message = event.getMessage().split(" ")
    if len(message) > 1 and message[0].lower() in ("/tell", "/minecraft:tell") and not whisper(event.getPlayer(), message[1]):
        event.setCancelled(True)

@hook.enable
def replace_ess_commands():

    try:
        map_field = server.getPluginManager().getClass().getDeclaredField("commandMap")
        map_field.setAccessible(True)
        command_map = map_field.get(server.getPluginManager())

        commands_field = command_map.getClass().getDeclaredField("knownCommands")
        commands_field.setAccessible(True)
        map = commands_field.get(command_map)

        ess_msg_cmd = map.get("essentials:msg")
        ess_reply_cmd = map.get("essentials:reply")
        ess_tpa_cmd = map.get("essentials:tpa")
        ess_tpahere_cmd = map.get("essentials:tpahere")

        msg_cmd_wrapper = CommandWrapper(ess_msg_cmd, msg_command_checker)
        reply_cmd_wrapper = CommandWrapper(ess_reply_cmd, reply_command_checker)
        tpa_cmd_wrapper = CommandWrapper(ess_tpa_cmd, tpa_command_checker)
        tpahere_cmd_wrapper = CommandWrapper(ess_tpahere_cmd, tpahere_command_checker)

        iterator = map.entrySet().iterator()
        while iterator.hasNext():
            entry = iterator.next()
            value = entry.getValue()
            if value is ess_msg_cmd:
                entry.setValue(msg_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_reply_cmd:
                entry.setValue(reply_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_tpa_cmd:
                entry.setValue(tpa_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())
            elif value is ess_tpahere_cmd:
                entry.setValue(tpahere_cmd_wrapper)
                info("[imbusy] wrapped /" + entry.getKey())

    except:
        error("[Imbusy] Failed to wrap essentials commands")
        error(trace())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers._stan_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This file contains the names of functions for Stan used by
    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.

    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

KEYWORDS = (   'else',
    'for',
    'if',
    'in',
    'increment_log_prob',
    'integrate_ode',
    'lp__',
    'print',
    'reject',
    'return',
    'while')

TYPES = (   'cholesky_factor_corr',
    'cholesky_factor_cov',
    'corr_matrix',
    'cov_matrix',
    'int',
    'matrix',
    'ordered',
    'positive_ordered',
    'real',
    'row_vector',
    'row_vectormatrix',
    'simplex',
    'unit_vector',
    'vector',
    'void')

FUNCTIONS = (   'Phi',
    'Phi_approx',
    'abs',
    'acos',
    'acosh',
    'append_col',
    'append_row',
    'asin',
    'asinh',
    'atan',
    'atan2',
    'atanh',
    'bernoulli_ccdf_log',
    'bernoulli_cdf',
    'bernoulli_cdf_log',
    'bernoulli_log',
    'bernoulli_logit_log',
    'bernoulli_rng',
    'bessel_first_kind',
    'bessel_second_kind',
    'beta_binomial_ccdf_log',
    'beta_binomial_cdf',
    'beta_binomial_cdf_log',
    'beta_binomial_log',
    'beta_binomial_rng',
    'beta_ccdf_log',
    'beta_cdf',
    'beta_cdf_log',
    'beta_log',
    'beta_rng',
    'binary_log_loss',
    'binomial_ccdf_log',
    'binomial_cdf',
    'binomial_cdf_log',
    'binomial_coefficient_log',
    'binomial_log',
    'binomial_logit_log',
    'binomial_rng',
    'block',
    'categorical_log',
    'categorical_logit_log',
    'categorical_rng',
    'cauchy_ccdf_log',
    'cauchy_cdf',
    'cauchy_cdf_log',
    'cauchy_log',
    'cauchy_rng',
    'cbrt',
    'ceil',
    'chi_square_ccdf_log',
    'chi_square_cdf',
    'chi_square_cdf_log',
    'chi_square_log',
    'chi_square_rng',
    'cholesky_decompose',
    'col',
    'cols',
    'columns_dot_product',
    'columns_dot_self',
    'cos',
    'cosh',
    'crossprod',
    'csr_extract_u',
    'csr_extract_v',
    'csr_extract_w',
    'csr_matrix_times_vector',
    'csr_to_dense_matrix',
    'cumulative_sum',
    'determinant',
    'diag_matrix',
    'diag_post_multiply',
    'diag_pre_multiply',
    'diagonal',
    'digamma',
    'dims',
    'dirichlet_log',
    'dirichlet_rng',
    'distance',
    'dot_product',
    'dot_self',
    'double_exponential_ccdf_log',
    'double_exponential_cdf',
    'double_exponential_cdf_log',
    'double_exponential_log',
    'double_exponential_rng',
    'e',
    'eigenvalues_sym',
    'eigenvectors_sym',
    'erf',
    'erfc',
    'exp',
    'exp2',
    'exp_mod_normal_ccdf_log',
    'exp_mod_normal_cdf',
    'exp_mod_normal_cdf_log',
    'exp_mod_normal_log',
    'exp_mod_normal_rng',
    'expm1',
    'exponential_ccdf_log',
    'exponential_cdf',
    'exponential_cdf_log',
    'exponential_log',
    'exponential_rng',
    'fabs',
    'falling_factorial',
    'fdim',
    'floor',
    'fma',
    'fmax',
    'fmin',
    'fmod',
    'frechet_ccdf_log',
    'frechet_cdf',
    'frechet_cdf_log',
    'frechet_log',
    'frechet_rng',
    'gamma_ccdf_log',
    'gamma_cdf',
    'gamma_cdf_log',
    'gamma_log',
    'gamma_p',
    'gamma_q',
    'gamma_rng',
    'gaussian_dlm_obs_log',
    'get_lp',
    'gumbel_ccdf_log',
    'gumbel_cdf',
    'gumbel_cdf_log',
    'gumbel_log',
    'gumbel_rng',
    'head',
    'hypergeometric_log',
    'hypergeometric_rng',
    'hypot',
    'if_else',
    'int_step',
    'inv',
    'inv_chi_square_ccdf_log',
    'inv_chi_square_cdf',
    'inv_chi_square_cdf_log',
    'inv_chi_square_log',
    'inv_chi_square_rng',
    'inv_cloglog',
    'inv_gamma_ccdf_log',
    'inv_gamma_cdf',
    'inv_gamma_cdf_log',
    'inv_gamma_log',
    'inv_gamma_rng',
    'inv_logit',
    'inv_phi',
    'inv_sqrt',
    'inv_square',
    'inv_wishart_log',
    'inv_wishart_rng',
    'inverse',
    'inverse_spd',
    'is_inf',
    'is_nan',
    'lbeta',
    'lgamma',
    'lkj_corr_cholesky_log',
    'lkj_corr_cholesky_rng',
    'lkj_corr_log',
    'lkj_corr_rng',
    'lmgamma',
    'log',
    'log10',
    'log1m',
    'log1m_exp',
    'log1m_inv_logit',
    'log1p',
    'log1p_exp',
    'log2',
    'log_determinant',
    'log_diff_exp',
    'log_falling_factorial',
    'log_inv_logit',
    'log_mix',
    'log_rising_factorial',
    'log_softmax',
    'log_sum_exp',
    'logistic_ccdf_log',
    'logistic_cdf',
    'logistic_cdf_log',
    'logistic_log',
    'logistic_rng',
    'logit',
    'lognormal_ccdf_log',
    'lognormal_cdf',
    'lognormal_cdf_log',
    'lognormal_log',
    'lognormal_rng',
    'machine_precision',
    'max',
    'mdivide_left_tri_low',
    'mdivide_right_tri_low',
    'mean',
    'min',
    'modified_bessel_first_kind',
    'modified_bessel_second_kind',
    'multi_gp_cholesky_log',
    'multi_gp_log',
    'multi_normal_cholesky_log',
    'multi_normal_cholesky_rng',
    'multi_normal_log',
    'multi_normal_prec_log',
    'multi_normal_rng',
    'multi_student_t_log',
    'multi_student_t_rng',
    'multinomial_log',
    'multinomial_rng',
    'multiply_log',
    'multiply_lower_tri_self_transpose',
    'neg_binomial_2_ccdf_log',
    'neg_binomial_2_cdf',
    'neg_binomial_2_cdf_log',
    'neg_binomial_2_log',
    'neg_binomial_2_log_log',
    'neg_binomial_2_log_rng',
    'neg_binomial_2_rng',
    'neg_binomial_ccdf_log',
    'neg_binomial_cdf',
    'neg_binomial_cdf_log',
    'neg_binomial_log',
    'neg_binomial_rng',
    'negative_infinity',
    'normal_ccdf_log',
    'normal_cdf',
    'normal_cdf_log',
    'normal_log',
    'normal_rng',
    'not_a_number',
    'num_elements',
    'ordered_logistic_log',
    'ordered_logistic_rng',
    'owens_t',
    'pareto_ccdf_log',
    'pareto_cdf',
    'pareto_cdf_log',
    'pareto_log',
    'pareto_rng',
    'pareto_type_2_ccdf_log',
    'pareto_type_2_cdf',
    'pareto_type_2_cdf_log',
    'pareto_type_2_log',
    'pareto_type_2_rng',
    'pi',
    'poisson_ccdf_log',
    'poisson_cdf',
    'poisson_cdf_log',
    'poisson_log',
    'poisson_log_log',
    'poisson_log_rng',
    'poisson_rng',
    'positive_infinity',
    'pow',
    'prod',
    'qr_Q',
    'qr_R',
    'quad_form',
    'quad_form_diag',
    'quad_form_sym',
    'rank',
    'rayleigh_ccdf_log',
    'rayleigh_cdf',
    'rayleigh_cdf_log',
    'rayleigh_log',
    'rayleigh_rng',
    'rep_array',
    'rep_matrix',
    'rep_row_vector',
    'rep_vector',
    'rising_factorial',
    'round',
    'row',
    'rows',
    'rows_dot_product',
    'rows_dot_self',
    'scaled_inv_chi_square_ccdf_log',
    'scaled_inv_chi_square_cdf',
    'scaled_inv_chi_square_cdf_log',
    'scaled_inv_chi_square_log',
    'scaled_inv_chi_square_rng',
    'sd',
    'segment',
    'sin',
    'singular_values',
    'sinh',
    'size',
    'skew_normal_ccdf_log',
    'skew_normal_cdf',
    'skew_normal_cdf_log',
    'skew_normal_log',
    'skew_normal_rng',
    'softmax',
    'sort_asc',
    'sort_desc',
    'sort_indices_asc',
    'sort_indices_desc',
    'sqrt',
    'sqrt2',
    'square',
    'squared_distance',
    'step',
    'student_t_ccdf_log',
    'student_t_cdf',
    'student_t_cdf_log',
    'student_t_log',
    'student_t_rng',
    'sub_col',
    'sub_row',
    'sum',
    'tail',
    'tan',
    'tanh',
    'tcrossprod',
    'tgamma',
    'to_array_1d',
    'to_array_2d',
    'to_matrix',
    'to_row_vector',
    'to_vector',
    'trace',
    'trace_gen_quad_form',
    'trace_quad_form',
    'trigamma',
    'trunc',
    'uniform_ccdf_log',
    'uniform_cdf',
    'uniform_cdf_log',
    'uniform_log',
    'uniform_rng',
    'variance',
    'von_mises_log',
    'von_mises_rng',
    'weibull_ccdf_log',
    'weibull_cdf',
    'weibull_cdf_log',
    'weibull_log',
    'weibull_rng',
    'wiener_log',
    'wishart_log',
    'wishart_rng')

DISTRIBUTIONS = (   'bernoulli',
    'bernoulli_logit',
    'beta',
    'beta_binomial',
    'binomial',
    'binomial_logit',
    'categorical',
    'categorical_logit',
    'cauchy',
    'chi_square',
    'dirichlet',
    'double_exponential',
    'exp_mod_normal',
    'exponential',
    'frechet',
    'gamma',
    'gaussian_dlm_obs',
    'gumbel',
    'hypergeometric',
    'inv_chi_square',
    'inv_gamma',
    'inv_wishart',
    'lkj_corr',
    'lkj_corr_cholesky',
    'logistic',
    'lognormal',
    'multi_gp',
    'multi_gp_cholesky',
    'multi_normal',
    'multi_normal_cholesky',
    'multi_normal_prec',
    'multi_student_t',
    'multinomial',
    'neg_binomial',
    'neg_binomial_2',
    'neg_binomial_2_log',
    'normal',
    'ordered_logistic',
    'pareto',
    'pareto_type_2',
    'poisson',
    'poisson_log',
    'rayleigh',
    'scaled_inv_chi_square',
    'skew_normal',
    'student_t',
    'uniform',
    'von_mises',
    'weibull',
    'wiener',
    'wishart')

RESERVED = (   'alignas',
    'alignof',
    'and',
    'and_eq',
    'asm',
    'auto',
    'bitand',
    'bitor',
    'bool',
    'break',
    'case',
    'catch',
    'char',
    'char16_t',
    'char32_t',
    'class',
    'compl',
    'const',
    'const_cast',
    'constexpr',
    'continue',
    'decltype',
    'default',
    'delete',
    'do',
    'double',
    'dynamic_cast',
    'enum',
    'explicit',
    'export',
    'extern',
    'false',
    'false',
    'float',
    'friend',
    'fvar',
    'goto',
    'inline',
    'int',
    'long',
    'mutable',
    'namespace',
    'new',
    'noexcept',
    'not',
    'not_eq',
    'nullptr',
    'operator',
    'or',
    'or_eq',
    'private',
    'protected',
    'public',
    'register',
    'reinterpret_cast',
    'repeat',
    'short',
    'signed',
    'sizeof',
    'static',
    'static_assert',
    'static_cast',
    'struct',
    'switch',
    'template',
    'then',
    'this',
    'thread_local',
    'throw',
    'true',
    'true',
    'try',
    'typedef',
    'typeid',
    'typename',
    'union',
    'unsigned',
    'until',
    'using',
    'var',
    'virtual',
    'void',
    'volatile',
    'wchar_t',
    'xor',
    'xor_eq')

<commit_msg>Fix spacing<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers._stan_builtins
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

    This file contains the names of functions for Stan used by
    ``pygments.lexers.math.StanLexer. This is for Stan language version 2.8.0.

    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

KEYWORDS = (
    'else',
    'for',
    'if',
    'in',
    'increment_log_prob',
    'integrate_ode',
    'lp__',
    'print',
    'reject',
    'return',
    'while')

TYPES = (
    'cholesky_factor_corr',
    'cholesky_factor_cov',
    'corr_matrix',
    'cov_matrix',
    'int',
    'matrix',
    'ordered',
    'positive_ordered',
    'real',
    'row_vector',
    'row_vectormatrix',
    'simplex',
    'unit_vector',
    'vector',
    'void')

FUNCTIONS = (
    'Phi',
    'Phi_approx',
    'abs',
    'acos',
    'acosh',
    'append_col',
    'append_row',
    'asin',
    'asinh',
    'atan',
    'atan2',
    'atanh',
    'bernoulli_ccdf_log',
    'bernoulli_cdf',
    'bernoulli_cdf_log',
    'bernoulli_log',
    'bernoulli_logit_log',
    'bernoulli_rng',
    'bessel_first_kind',
    'bessel_second_kind',
    'beta_binomial_ccdf_log',
    'beta_binomial_cdf',
    'beta_binomial_cdf_log',
    'beta_binomial_log',
    'beta_binomial_rng',
    'beta_ccdf_log',
    'beta_cdf',
    'beta_cdf_log',
    'beta_log',
    'beta_rng',
    'binary_log_loss',
    'binomial_ccdf_log',
    'binomial_cdf',
    'binomial_cdf_log',
    'binomial_coefficient_log',
    'binomial_log',
    'binomial_logit_log',
    'binomial_rng',
    'block',
    'categorical_log',
    'categorical_logit_log',
    'categorical_rng',
    'cauchy_ccdf_log',
    'cauchy_cdf',
    'cauchy_cdf_log',
    'cauchy_log',
    'cauchy_rng',
    'cbrt',
    'ceil',
    'chi_square_ccdf_log',
    'chi_square_cdf',
    'chi_square_cdf_log',
    'chi_square_log',
    'chi_square_rng',
    'cholesky_decompose',
    'col',
    'cols',
    'columns_dot_product',
    'columns_dot_self',
    'cos',
    'cosh',
    'crossprod',
    'csr_extract_u',
    'csr_extract_v',
    'csr_extract_w',
    'csr_matrix_times_vector',
    'csr_to_dense_matrix',
    'cumulative_sum',
    'determinant',
    'diag_matrix',
    'diag_post_multiply',
    'diag_pre_multiply',
    'diagonal',
    'digamma',
    'dims',
    'dirichlet_log',
    'dirichlet_rng',
    'distance',
    'dot_product',
    'dot_self',
    'double_exponential_ccdf_log',
    'double_exponential_cdf',
    'double_exponential_cdf_log',
    'double_exponential_log',
    'double_exponential_rng',
    'e',
    'eigenvalues_sym',
    'eigenvectors_sym',
    'erf',
    'erfc',
    'exp',
    'exp2',
    'exp_mod_normal_ccdf_log',
    'exp_mod_normal_cdf',
    'exp_mod_normal_cdf_log',
    'exp_mod_normal_log',
    'exp_mod_normal_rng',
    'expm1',
    'exponential_ccdf_log',
    'exponential_cdf',
    'exponential_cdf_log',
    'exponential_log',
    'exponential_rng',
    'fabs',
    'falling_factorial',
    'fdim',
    'floor',
    'fma',
    'fmax',
    'fmin',
    'fmod',
    'frechet_ccdf_log',
    'frechet_cdf',
    'frechet_cdf_log',
    'frechet_log',
    'frechet_rng',
    'gamma_ccdf_log',
    'gamma_cdf',
    'gamma_cdf_log',
    'gamma_log',
    'gamma_p',
    'gamma_q',
    'gamma_rng',
    'gaussian_dlm_obs_log',
    'get_lp',
    'gumbel_ccdf_log',
    'gumbel_cdf',
    'gumbel_cdf_log',
    'gumbel_log',
    'gumbel_rng',
    'head',
    'hypergeometric_log',
    'hypergeometric_rng',
    'hypot',
    'if_else',
    'int_step',
    'inv',
    'inv_chi_square_ccdf_log',
    'inv_chi_square_cdf',
    'inv_chi_square_cdf_log',
    'inv_chi_square_log',
    'inv_chi_square_rng',
    'inv_cloglog',
    'inv_gamma_ccdf_log',
    'inv_gamma_cdf',
    'inv_gamma_cdf_log',
    'inv_gamma_log',
    'inv_gamma_rng',
    'inv_logit',
    'inv_phi',
    'inv_sqrt',
    'inv_square',
    'inv_wishart_log',
    'inv_wishart_rng',
    'inverse',
    'inverse_spd',
    'is_inf',
    'is_nan',
    'lbeta',
    'lgamma',
    'lkj_corr_cholesky_log',
    'lkj_corr_cholesky_rng',
    'lkj_corr_log',
    'lkj_corr_rng',
    'lmgamma',
    'log',
    'log10',
    'log1m',
    'log1m_exp',
    'log1m_inv_logit',
    'log1p',
    'log1p_exp',
    'log2',
    'log_determinant',
    'log_diff_exp',
    'log_falling_factorial',
    'log_inv_logit',
    'log_mix',
    'log_rising_factorial',
    'log_softmax',
    'log_sum_exp',
    'logistic_ccdf_log',
    'logistic_cdf',
    'logistic_cdf_log',
    'logistic_log',
    'logistic_rng',
    'logit',
    'lognormal_ccdf_log',
    'lognormal_cdf',
    'lognormal_cdf_log',
    'lognormal_log',
    'lognormal_rng',
    'machine_precision',
    'max',
    'mdivide_left_tri_low',
    'mdivide_right_tri_low',
    'mean',
    'min',
    'modified_bessel_first_kind',
    'modified_bessel_second_kind',
    'multi_gp_cholesky_log',
    'multi_gp_log',
    'multi_normal_cholesky_log',
    'multi_normal_cholesky_rng',
    'multi_normal_log',
    'multi_normal_prec_log',
    'multi_normal_rng',
    'multi_student_t_log',
    'multi_student_t_rng',
    'multinomial_log',
    'multinomial_rng',
    'multiply_log',
    'multiply_lower_tri_self_transpose',
    'neg_binomial_2_ccdf_log',
    'neg_binomial_2_cdf',
    'neg_binomial_2_cdf_log',
    'neg_binomial_2_log',
    'neg_binomial_2_log_log',
    'neg_binomial_2_log_rng',
    'neg_binomial_2_rng',
    'neg_binomial_ccdf_log',
    'neg_binomial_cdf',
    'neg_binomial_cdf_log',
    'neg_binomial_log',
    'neg_binomial_rng',
    'negative_infinity',
    'normal_ccdf_log',
    'normal_cdf',
    'normal_cdf_log',
    'normal_log',
    'normal_rng',
    'not_a_number',
    'num_elements',
    'ordered_logistic_log',
    'ordered_logistic_rng',
    'owens_t',
    'pareto_ccdf_log',
    'pareto_cdf',
    'pareto_cdf_log',
    'pareto_log',
    'pareto_rng',
    'pareto_type_2_ccdf_log',
    'pareto_type_2_cdf',
    'pareto_type_2_cdf_log',
    'pareto_type_2_log',
    'pareto_type_2_rng',
    'pi',
    'poisson_ccdf_log',
    'poisson_cdf',
    'poisson_cdf_log',
    'poisson_log',
    'poisson_log_log',
    'poisson_log_rng',
    'poisson_rng',
    'positive_infinity',
    'pow',
    'prod',
    'qr_Q',
    'qr_R',
    'quad_form',
    'quad_form_diag',
    'quad_form_sym',
    'rank',
    'rayleigh_ccdf_log',
    'rayleigh_cdf',
    'rayleigh_cdf_log',
    'rayleigh_log',
    'rayleigh_rng',
    'rep_array',
    'rep_matrix',
    'rep_row_vector',
    'rep_vector',
    'rising_factorial',
    'round',
    'row',
    'rows',
    'rows_dot_product',
    'rows_dot_self',
    'scaled_inv_chi_square_ccdf_log',
    'scaled_inv_chi_square_cdf',
    'scaled_inv_chi_square_cdf_log',
    'scaled_inv_chi_square_log',
    'scaled_inv_chi_square_rng',
    'sd',
    'segment',
    'sin',
    'singular_values',
    'sinh',
    'size',
    'skew_normal_ccdf_log',
    'skew_normal_cdf',
    'skew_normal_cdf_log',
    'skew_normal_log',
    'skew_normal_rng',
    'softmax',
    'sort_asc',
    'sort_desc',
    'sort_indices_asc',
    'sort_indices_desc',
    'sqrt',
    'sqrt2',
    'square',
    'squared_distance',
    'step',
    'student_t_ccdf_log',
    'student_t_cdf',
    'student_t_cdf_log',
    'student_t_log',
    'student_t_rng',
    'sub_col',
    'sub_row',
    'sum',
    'tail',
    'tan',
    'tanh',
    'tcrossprod',
    'tgamma',
    'to_array_1d',
    'to_array_2d',
    'to_matrix',
    'to_row_vector',
    'to_vector',
    'trace',
    'trace_gen_quad_form',
    'trace_quad_form',
    'trigamma',
    'trunc',
    'uniform_ccdf_log',
    'uniform_cdf',
    'uniform_cdf_log',
    'uniform_log',
    'uniform_rng',
    'variance',
    'von_mises_log',
    'von_mises_rng',
    'weibull_ccdf_log',
    'weibull_cdf',
    'weibull_cdf_log',
    'weibull_log',
    'weibull_rng',
    'wiener_log',
    'wishart_log',
    'wishart_rng')

DISTRIBUTIONS = (   'bernoulli',
    'bernoulli_logit',
    'beta',
    'beta_binomial',
    'binomial',
    'binomial_logit',
    'categorical',
    'categorical_logit',
    'cauchy',
    'chi_square',
    'dirichlet',
    'double_exponential',
    'exp_mod_normal',
    'exponential',
    'frechet',
    'gamma',
    'gaussian_dlm_obs',
    'gumbel',
    'hypergeometric',
    'inv_chi_square',
    'inv_gamma',
    'inv_wishart',
    'lkj_corr',
    'lkj_corr_cholesky',
    'logistic',
    'lognormal',
    'multi_gp',
    'multi_gp_cholesky',
    'multi_normal',
    'multi_normal_cholesky',
    'multi_normal_prec',
    'multi_student_t',
    'multinomial',
    'neg_binomial',
    'neg_binomial_2',
    'neg_binomial_2_log',
    'normal',
    'ordered_logistic',
    'pareto',
    'pareto_type_2',
    'poisson',
    'poisson_log',
    'rayleigh',
    'scaled_inv_chi_square',
    'skew_normal',
    'student_t',
    'uniform',
    'von_mises',
    'weibull',
    'wiener',
    'wishart')

RESERVED = (   'alignas',
    'alignof',
    'and',
    'and_eq',
    'asm',
    'auto',
    'bitand',
    'bitor',
    'bool',
    'break',
    'case',
    'catch',
    'char',
    'char16_t',
    'char32_t',
    'class',
    'compl',
    'const',
    'const_cast',
    'constexpr',
    'continue',
    'decltype',
    'default',
    'delete',
    'do',
    'double',
    'dynamic_cast',
    'enum',
    'explicit',
    'export',
    'extern',
    'false',
    'false',
    'float',
    'friend',
    'fvar',
    'goto',
    'inline',
    'int',
    'long',
    'mutable',
    'namespace',
    'new',
    'noexcept',
    'not',
    'not_eq',
    'nullptr',
    'operator',
    'or',
    'or_eq',
    'private',
    'protected',
    'public',
    'register',
    'reinterpret_cast',
    'repeat',
    'short',
    'signed',
    'sizeof',
    'static',
    'static_assert',
    'static_cast',
    'struct',
    'switch',
    'template',
    'then',
    'this',
    'thread_local',
    'throw',
    'true',
    'true',
    'try',
    'typedef',
    'typeid',
    'typename',
    'union',
    'unsigned',
    'until',
    'using',
    'var',
    'virtual',
    'void',
    'volatile',
    'wchar_t',
    'xor',
    'xor_eq')

<|endoftext|>


<commit_before>
class MajorClust(object):
    def __init__(self, graph):
        self.graph = graph

    def get_majorclust(self):
        reclusters = set()
        terminate = False
        while not terminate:
            terminate = True
            for node in self.graph.nodes_iter(data=True):
                initial_cluster = node[1]['cluster']
                current_cluster = self._sub_majorclust(node, g)
                recluster = (node[0], initial_cluster, current_cluster)

                # if has not checked, recluster again
                if initial_cluster != current_cluster and recluster not in reclusters:
                    reclusters.add(recluster)
                    node[1]['cluster'] = current_cluster
                    terminate = False

    def _sub_majorclust(self, node):
        # reclustering
        visited_neighbor, visited_neigbor_num, neighbor_weights = {}, {}, {}

        # observe neighboring edges and nodes
        for current_node, neighbor_node, weight in self.graph.edges_iter([node[0]], data='weight'):
            neighbor_weights[neighbor_node] = weight
            visited_neighbor[self.graph.node[neighbor_node]['cluster']] = \
                visited_neighbor.get(self.graph.node[neighbor_node]['cluster'], 0.0) + weight['weight']

        # get the weight
        for k, v in visited_neighbor.iteritems():
            visited_neigbor_num.setdefault(v, []).append(k)

        # attach a node to the cluster of majority of neighbor nodes
        current_cluster = visited_neigbor_num[max(visited_neigbor_num)][0] if visited_neigbor_num else \
            node[1]['cluster']

        return current_cluster
<commit_msg>Add method _get_cluster<commit_after>from ClusterUtility import ClusterUtility

class MajorClust(object):
    """The implementation of MajorClust graph clustering algorithm [1]_.

    References
    ----------
    .. [1] B. Stein and O. Niggemann, On the nature of structure and its identification,
           Proceedings of the 25th International Workshop on Graph-Theoretic Concepts in Computer Science,
           pp. 122-134, 1999.
    """
    def __init__(self, graph):
        self.graph = graph
        self.clusters = {}
        self.cluster_property = {}

    def get_majorclust(self):
        self._majorclust()
        self._get_cluster()
        self.cluster_property = ClusterUtility.get_cluster_property(self.graph, self.clusters)

    def _majorclust(self):
        reclusters = set()
        terminate = False
        while not terminate:
            terminate = True
            for node in self.graph.nodes_iter(data=True):
                initial_cluster = node[1]['cluster']
                current_cluster = self._re_majorclust(node)
                recluster = (node[0], initial_cluster, current_cluster)

                # if has not checked, recluster again
                if initial_cluster != current_cluster and recluster not in reclusters:
                    reclusters.add(recluster)
                    node[1]['cluster'] = current_cluster
                    terminate = False

    def _re_majorclust(self, node):
        # reclustering
        visited_neighbor, visited_neigbor_num, neighbor_weights = {}, {}, {}

        # observe neighboring edges and nodes
        for current_node, neighbor_node, weight in self.graph.edges_iter([node[0]], data='weight'):
            neighbor_weights[neighbor_node] = weight
            visited_neighbor[self.graph.node[neighbor_node]['cluster']] = \
                visited_neighbor.get(self.graph.node[neighbor_node]['cluster'], 0.0) + weight['weight']

        # get the weight
        for k, v in visited_neighbor.iteritems():
            visited_neigbor_num.setdefault(v, []).append(k)

        # attach a node to the cluster of majority of neighbor nodes
        current_cluster = visited_neigbor_num[max(visited_neigbor_num)][0] if visited_neigbor_num else \
            node[1]['cluster']

        return current_cluster

    def _get_cluster(self):
        cluster = [n[1]['cluster'] for n in self.graph.nodes_iter(data='cluster')]
        unique_cluster = set(cluster)
        cluster_id = 0
        for uc in unique_cluster:
            nodes = []
            for node in self.graph.nodes_iter(data='True'):
                if node[1]['cluster'] == uc:
                    nodes.append(node[1]['cluster'])
            self.clusters[cluster_id] = nodes
            cluster_id += 1
<|endoftext|>


<commit_before>import sqlite3
import os
try:
    import json
except ImportError:
    import simplejson as json
import sys
import xml.sax
import binascii
from vincenty import vincenty
from struct import pack, unpack
from rtree import Rtree

def cons(ary):
    for i in range(len(ary)-1):
        yield (ary[i], ary[i+1])

def pack_coords(coords):
    return binascii.b2a_base64( "".join([pack( "ff", *coord ) for coord in coords]) )

def unpack_coords(str):
    bin = binascii.a2b_base64( str )
    return [unpack( "ff", bin[i:i+8] ) for i in range(0, len(bin), 8)]

class Node:
    def __init__(self, id, lon, lat):
        self.id = id
        self.lon = lon
        self.lat = lat
        self.tags = {}

    def __repr__(self):
        return "<Node id='%s' (%s, %s) n_tags=%d>"%(self.id, self.lon, self.lat, len(self.tags))

class Way:
    def __init__(self, id):
        self.id = id
        self.nd_ids = []
        self.tags = {}

    def __repr__(self):
        return "<Way id='%s' n_nds=%d n_tags=%d>"%(self.id, len(self.nd_ids), len(self.tags))

class WayRecord:
    def __init__(self, id, tags, nds):
        self.id = id

        if type(tags)==unicode:
            self.tags_str = tags
            self.tags_cache = None
        else:
            self.tags_cache = tags
            self.tags_str = None

        if type(nds)==unicode:
            self.nds_str = nds
            self.nds_cache = None
        else:
            self.nds_cache = nds
            self.nds_str = None

    @property
    def tags(self):
        self.tags_cache = self.tags_cache or json.loads(self.tags_str)
        return self.tags_cache

    @property
    def nds(self):
        self.nds_cache = self.nds_cache or json.loads(self.nds_str)
        return self.nds_cache

    def __repr__(self):
        return "<WayRecord id='%s'>"%self.id

class OSMDB:
    def __init__(self, dbname,overwrite=False,rtree_index=True):
        if overwrite:
            try:
                os.remove( dbname )
            except OSError:
                pass

        self.conn = sqlite3.connect(dbname)

        if rtree_index:
            self.index = Rtree( dbname )
        else:
            self.index = None

        if overwrite:
            self.setup()

    def setup(self):
        c = self.conn.cursor()
        c.execute( "CREATE TABLE nodes (id TEXT, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" )
        c.execute( "CREATE TABLE ways (id TEXT, tags TEXT, nds TEXT)" )
        self.conn.commit()
        c.close()

    def create_indexes(self):
        c = self.conn.cursor()
        c.execute( "CREATE INDEX nodes_id ON nodes (id)" )
        c.execute( "CREATE INDEX nodes_lon ON nodes (lon)" )
        c.execute( "CREATE INDEX nodes_lat ON nodes (lat)" )
        c.execute( "CREATE INDEX ways_id ON ways (id)" )
        self.conn.commit()
        c.close()

    def populate(self, osm_filename, accept=lambda tags: True, reporter=None):
        print "importing osm from XML to sqlite database"

        c = self.conn.cursor()

        self.n_nodes = 0
        self.n_ways = 0

        superself = self

        class OSMHandler(xml.sax.ContentHandler):
            @classmethod
            def setDocumentLocator(self,loc):
                pass

            @classmethod
            def startDocument(self):
                pass

            @classmethod
            def endDocument(self):
                pass

            @classmethod
            def startElement(self, name, attrs):
                if name=='node':
                    self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat']))
                elif name=='way':
                    self.currElem = Way(attrs['id'])
                elif name=='tag':
                    self.currElem.tags[attrs['k']] = attrs['v']
                elif name=='nd':
                    self.currElem.nd_ids.append( attrs['ref'] )

            @classmethod
            def endElement(self,name):
                if name=='node':
                    if superself.n_nodes%5000==0:
                        print "node %d"%superself.n_nodes
                    superself.n_nodes += 1
                    superself.add_node( self.currElem, c )
                elif name=='way':
                    if superself.n_ways%5000==0:
                        print "way %d"%superself.n_ways
                    superself.n_ways += 1
                    superself.add_way( self.currElem, c )

            @classmethod
            def characters(self, chars):
                pass

        xml.sax.parse(osm_filename, OSMHandler)

        self.conn.commit()
        c.close()

        print "indexing primary tables...",
        self.create_indexes()
        print "done"

    def set_endnode_ref_counts( self ):
        """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways"""

        print "counting end-node references to find way split-points"

        c = self.conn.cursor()

        endnode_ref_counts = {}

        c.execute( "SELECT nds from ways" )

        print "...counting"
        for i, (nds_str,) in enumerate(c):
            if i%5000==0:
                print i

            nds = json.loads( nds_str )
            for nd in nds:
                endnode_ref_counts[ nd ] = endnode_ref_counts.get( nd, 0 )+1

        print "...updating nodes table"
        for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()):
            if i%5000==0:
                print i

            if ref_count > 1:
                c.execute( "UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id) )

        self.conn.commit()
        c.close()

    def index_endnodes( self ):
        print "indexing endpoint nodes into rtree"

        c = self.conn.cursor()

        c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1" )

        for id, lat, lon in c:
            self.index.add( int(id), (lon, lat, lon, lat) )

        c.close()

    def create_and_populate_edges_table( self, tolerant=False ):
        self.set_endnode_ref_counts()
        self.index_endnodes()

        print "splitting ways and inserting into edge table"

        c = self.conn.cursor()

        c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" )

        for i, way in enumerate(self.ways()):
            try:
                if i%5000==0:
                    print i

                #split way into several sub-ways
                subways = []
                curr_subway = [ way.nds[0] ]
                for nd in way.nds[1:]:
                    curr_subway.append( nd )
                    if self.node(nd)[4] > 1: # node reference count is greater than zero
                        subways.append( curr_subway )
                        curr_subway = [ nd ]

                #insert into edge table
                for i, subway in enumerate(subways):
                    coords = [(lambda x:(x[3],x[2]))(self.node(nd)) for nd in subway]
                    packt = pack_coords( coords )
                    dist = sum([vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords)])
                    c.execute( "INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s"%(way.id, i),
                                                                               way.id,
                                                                               subway[0],
                                                                               subway[-1],
                                                                               dist,
                                                                               packt) )
            except IndexError:
                if tolerant:
                    continue
                else:
                    raise

        print "indexing edges...",
        c.execute( "CREATE INDEX edges_id ON edges (id)" )
        c.execute( "CREATE INDEX edges_parent_id ON edges (parent_id)" )
        print "done"

        self.conn.commit()
        c.close()

    def edge(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id,) )

        try:
            ret = c.next()
            way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret
            return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords( geom ), json.loads(tags))
        except StopIteration:
            c.close()
            raise IndexError( "Database does not have an edge with id '%s'"%id )

        c.close()
        return ret

    def edges(self):
        c = self.conn.cursor()

        c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" )

        for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c:
            yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags))

        c.close()

    def add_way( self, way, curs=None ):
        if curs is None:
            curs = self.conn.cursor()
            close_cursor = True
        else:
            close_cursor = False

        curs.execute("INSERT INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids) ))

        if close_cursor:
            self.conn.commit()
            curs.close()

    def add_node( self, node, curs=None ):
        if curs is None:
            curs = self.conn.cursor()
            close_cursor = True
        else:
            close_cursor = False

        curs.execute("INSERT INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", ( node.id, json.dumps(node.tags), node.lat, node.lon ) )

        if close_cursor:
            self.conn.commit()
            curs.close()

    def nodes(self):
        c = self.conn.cursor()

        c.execute( "SELECT * FROM nodes" )

        for node_row in c:
            yield node_row

        c.close()

    def node(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT * FROM nodes WHERE id = ?", (id,) )

        try:
            ret = c.next()
        except StopIteration:
            c.close()
            raise IndexError( "Database does not have node with id '%s'"%id )

        c.close()
        return ret

    def nearest_node(self, lat, lon, range=0.005):
        c = self.conn.cursor()

        if self.index:
            print "YOUR'RE USING THE INDEX"
            id = self.index.nearest( (lon, lat), 1 )[0]
            print "THE ID IS %d"%id
            c.execute( "SELECT id, lat, lon FROM nodes WHERE id = ?", (id,) )
        else:
            c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat-range, lat+range, lon-range, lon+range) )

        dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]

        if len(dists)==0:
            return (None, None, None, None)

        return min( dists, key = lambda x:x[3] )

    def nearest_of( self, lat, lon, nodes ):
        c = self.conn.cursor()

        c.execute( "SELECT id, lat, lon FROM nodes WHERE id IN (%s)"%",".join([str(x) for x in nodes]) )

        dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]

        if len(dists)==0:
            return (None, None, None, None)

        return min( dists, key = lambda x:x[3] )

    def way(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT id, tags, nds FROM ways WHERE id = ?", (id,) )

        try:
          id, tags_str, nds_str = c.next()
          ret = WayRecord(id, tags_str, nds_str)
        except StopIteration:
          raise Exception( "OSMDB has no way with id '%s'"%id )
        finally:
          c.close()

        return ret

    def way_nds(self, id):
        c = self.conn.cursor()
        c.execute( "SELECT nds FROM ways WHERE id = ?", (id,) )

        (nds_str,) = c.next()
        c.close()

        return json.loads( nds_str )

    def ways(self):
        c = self.conn.cursor()

        c.execute( "SELECT id, tags, nds FROM ways" )

        for id, tags_str, nds_str in c:
            yield WayRecord( id, tags_str, nds_str )

        c.close()

    def count_ways(self):
        c = self.conn.cursor()

        c.execute( "SELECT count(*) FROM ways" )
        ret = c.next()[0]

        c.close()

        return ret

    def count_edges(self):
        c = self.conn.cursor()

        c.execute( "SELECT count(*) FROM edges" )
        ret = c.next()[0]

        c.close()

        return ret

    def delete_way(self, id):
        c = self.conn.cursor()

        c.execute("DELETE FROM ways WHERE id = ?", (id,))

        c.close()

    def bounds(self):
        c = self.conn.cursor()
        c.execute( "SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes" )

        ret = c.next()
        c.close()
        return ret

    def execute(self,sql,args=None):
        c = self.conn.cursor()
        if args:
            for row in c.execute(sql,args):
                yield row
        else:
            for row in c.execute(sql):
                yield row
        c.close()

    def cursor(self):
        return self.conn.cursor()

def test_wayrecord():
    wr = WayRecord( "1", {'highway':'bumpkis'}, ['1','2','3'] )
    assert wr.id == "1"
    assert wr.tags == {'highway':'bumpkis'}
    assert wr.nds == ['1','2','3']

    wr = WayRecord( "1", "{\"highway\":\"bumpkis\"}", "[\"1\",\"2\",\"3\"]" )
    assert wr.id == "1"
    assert wr.tags == {'highway':'bumpkis'}
    assert wr.nds == ['1','2','3']

def osm_to_osmdb(osm_filename, osmdb_filename, tolerant=False):
    osmdb = OSMDB( osmdb_filename, overwrite=True )
    osmdb.populate( osm_filename, accept=lambda tags: 'highway' in tags, reporter=sys.stdout )
    osmdb.create_and_populate_edges_table(tolerant)

def main():
    from sys import argv

    usage = "python osmdb.py osm_filename osmdb_filename"
    if len(argv) < 3:
        print usage
        exit()

    osm_filename = argv[1]
    osmdb_filename = argv[2]

    tolerant = 'tolerant' in argv

    osm_to_osmdb(osm_filename, osmdb_filename, tolerant)

if __name__=='__main__':
    main()
<commit_msg>previously wasn't inserting some ways into the edge table which were otherwise being admitted into the endnode index, creating a problem where some vertices were gettable from the endnode index but weren't in the graph, creating...problems<commit_after>import sqlite3
import os
try:
    import json
except ImportError:
    import simplejson as json
import sys
import xml.sax
import binascii
from vincenty import vincenty
from struct import pack, unpack
from rtree import Rtree

def cons(ary):
    for i in range(len(ary)-1):
        yield (ary[i], ary[i+1])

def pack_coords(coords):
    return binascii.b2a_base64( "".join([pack( "ff", *coord ) for coord in coords]) )

def unpack_coords(str):
    bin = binascii.a2b_base64( str )
    return [unpack( "ff", bin[i:i+8] ) for i in range(0, len(bin), 8)]

class Node:
    def __init__(self, id, lon, lat):
        self.id = id
        self.lon = lon
        self.lat = lat
        self.tags = {}

    def __repr__(self):
        return "<Node id='%s' (%s, %s) n_tags=%d>"%(self.id, self.lon, self.lat, len(self.tags))

class Way:
    def __init__(self, id):
        self.id = id
        self.nd_ids = []
        self.tags = {}

    def __repr__(self):
        return "<Way id='%s' n_nds=%d n_tags=%d>"%(self.id, len(self.nd_ids), len(self.tags))

class WayRecord:
    def __init__(self, id, tags, nds):
        self.id = id

        if type(tags)==unicode:
            self.tags_str = tags
            self.tags_cache = None
        else:
            self.tags_cache = tags
            self.tags_str = None

        if type(nds)==unicode:
            self.nds_str = nds
            self.nds_cache = None
        else:
            self.nds_cache = nds
            self.nds_str = None

    @property
    def tags(self):
        self.tags_cache = self.tags_cache or json.loads(self.tags_str)
        return self.tags_cache

    @property
    def nds(self):
        self.nds_cache = self.nds_cache or json.loads(self.nds_str)
        return self.nds_cache

    def __repr__(self):
        return "<WayRecord id='%s'>"%self.id

class OSMDB:
    def __init__(self, dbname,overwrite=False,rtree_index=True):
        if overwrite:
            try:
                os.remove( dbname )
            except OSError:
                pass

        self.conn = sqlite3.connect(dbname)

        if rtree_index:
            self.index = Rtree( dbname )
        else:
            self.index = None

        if overwrite:
            self.setup()

    def setup(self):
        c = self.conn.cursor()
        c.execute( "CREATE TABLE nodes (id TEXT, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" )
        c.execute( "CREATE TABLE ways (id TEXT, tags TEXT, nds TEXT)" )
        self.conn.commit()
        c.close()

    def create_indexes(self):
        c = self.conn.cursor()
        c.execute( "CREATE INDEX nodes_id ON nodes (id)" )
        c.execute( "CREATE INDEX nodes_lon ON nodes (lon)" )
        c.execute( "CREATE INDEX nodes_lat ON nodes (lat)" )
        c.execute( "CREATE INDEX ways_id ON ways (id)" )
        self.conn.commit()
        c.close()

    def populate(self, osm_filename, accept=lambda tags: True, reporter=None):
        print "importing osm from XML to sqlite database"

        c = self.conn.cursor()

        self.n_nodes = 0
        self.n_ways = 0

        superself = self

        class OSMHandler(xml.sax.ContentHandler):
            @classmethod
            def setDocumentLocator(self,loc):
                pass

            @classmethod
            def startDocument(self):
                pass

            @classmethod
            def endDocument(self):
                pass

            @classmethod
            def startElement(self, name, attrs):
                if name=='node':
                    self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat']))
                elif name=='way':
                    self.currElem = Way(attrs['id'])
                elif name=='tag':
                    self.currElem.tags[attrs['k']] = attrs['v']
                elif name=='nd':
                    self.currElem.nd_ids.append( attrs['ref'] )

            @classmethod
            def endElement(self,name):
                if name=='node':
                    if superself.n_nodes%5000==0:
                        print "node %d"%superself.n_nodes
                    superself.n_nodes += 1
                    superself.add_node( self.currElem, c )
                elif name=='way':
                    if superself.n_ways%5000==0:
                        print "way %d"%superself.n_ways
                    superself.n_ways += 1
                    superself.add_way( self.currElem, c )

            @classmethod
            def characters(self, chars):
                pass

        xml.sax.parse(osm_filename, OSMHandler)

        self.conn.commit()
        c.close()

        print "indexing primary tables...",
        self.create_indexes()
        print "done"

    def set_endnode_ref_counts( self ):
        """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways"""

        print "counting end-node references to find way split-points"

        c = self.conn.cursor()

        endnode_ref_counts = {}

        c.execute( "SELECT nds from ways" )

        print "...counting"
        for i, (nds_str,) in enumerate(c):
            if i%5000==0:
                print i

            nds = json.loads( nds_str )
            for nd in nds:
                endnode_ref_counts[ nd ] = endnode_ref_counts.get( nd, 0 )+1

        print "...updating nodes table"
        for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()):
            if i%5000==0:
                print i

            if ref_count > 1:
                c.execute( "UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id) )

        self.conn.commit()
        c.close()

    def index_endnodes( self ):
        print "indexing endpoint nodes into rtree"

        c = self.conn.cursor()

        #TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes
        c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1" )

        for id, lat, lon in c:
            self.index.add( int(id), (lon, lat, lon, lat) )

        c.close()

    def create_and_populate_edges_table( self, tolerant=False ):
        self.set_endnode_ref_counts()
        self.index_endnodes()

        print "splitting ways and inserting into edge table"

        c = self.conn.cursor()

        c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" )

        for i, way in enumerate(self.ways()):
            try:
                if i%5000==0:
                    print i

                subways = []
                curr_subway = [ way.nds[0] ] # add first node to the current subway
                for nd in way.nds[1:-1]:     # for every internal node of the way
                    curr_subway.append( nd )
                    if self.node(nd)[4] > 1: # node reference count is greater than one, node is shared by two ways
                        subways.append( curr_subway )
                        curr_subway = [ nd ]
                curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway
                subways.append( curr_subway );

                #insert into edge table
                for i, subway in enumerate(subways):
                    coords = [(lambda x:(x[3],x[2]))(self.node(nd)) for nd in subway]
                    packt = pack_coords( coords )
                    dist = sum([vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords)])
                    c.execute( "INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s"%(way.id, i),
                                                                               way.id,
                                                                               subway[0],
                                                                               subway[-1],
                                                                               dist,
                                                                               packt) )
            except IndexError:
                if tolerant:
                    continue
                else:
                    raise

        print "indexing edges...",
        c.execute( "CREATE INDEX edges_id ON edges (id)" )
        c.execute( "CREATE INDEX edges_parent_id ON edges (parent_id)" )
        print "done"

        self.conn.commit()
        c.close()

    def edge(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id,) )

        try:
            ret = c.next()
            way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret
            return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords( geom ), json.loads(tags))
        except StopIteration:
            c.close()
            raise IndexError( "Database does not have an edge with id '%s'"%id )

        c.close()
        return ret

    def edges(self):
        c = self.conn.cursor()

        c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" )

        for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c:
            yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags))

        c.close()

    def add_way( self, way, curs=None ):
        if curs is None:
            curs = self.conn.cursor()
            close_cursor = True
        else:
            close_cursor = False

        curs.execute("INSERT INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids) ))

        if close_cursor:
            self.conn.commit()
            curs.close()

    def add_node( self, node, curs=None ):
        if curs is None:
            curs = self.conn.cursor()
            close_cursor = True
        else:
            close_cursor = False

        curs.execute("INSERT INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", ( node.id, json.dumps(node.tags), node.lat, node.lon ) )

        if close_cursor:
            self.conn.commit()
            curs.close()

    def nodes(self):
        c = self.conn.cursor()

        c.execute( "SELECT * FROM nodes" )

        for node_row in c:
            yield node_row

        c.close()

    def node(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT * FROM nodes WHERE id = ?", (id,) )

        try:
            ret = c.next()
        except StopIteration:
            c.close()
            raise IndexError( "Database does not have node with id '%s'"%id )

        c.close()
        return ret

    def nearest_node(self, lat, lon, range=0.005):
        c = self.conn.cursor()

        if self.index:
            print "YOUR'RE USING THE INDEX"
            id = self.index.nearest( (lon, lat), 1 )[0]
            print "THE ID IS %d"%id
            c.execute( "SELECT id, lat, lon FROM nodes WHERE id = ?", (id,) )
        else:
            c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat-range, lat+range, lon-range, lon+range) )

        dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]

        if len(dists)==0:
            return (None, None, None, None)

        return min( dists, key = lambda x:x[3] )

    def nearest_of( self, lat, lon, nodes ):
        c = self.conn.cursor()

        c.execute( "SELECT id, lat, lon FROM nodes WHERE id IN (%s)"%",".join([str(x) for x in nodes]) )

        dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]

        if len(dists)==0:
            return (None, None, None, None)

        return min( dists, key = lambda x:x[3] )

    def way(self, id):
        c = self.conn.cursor()

        c.execute( "SELECT id, tags, nds FROM ways WHERE id = ?", (id,) )

        try:
          id, tags_str, nds_str = c.next()
          ret = WayRecord(id, tags_str, nds_str)
        except StopIteration:
          raise Exception( "OSMDB has no way with id '%s'"%id )
        finally:
          c.close()

        return ret

    def way_nds(self, id):
        c = self.conn.cursor()
        c.execute( "SELECT nds FROM ways WHERE id = ?", (id,) )

        (nds_str,) = c.next()
        c.close()

        return json.loads( nds_str )

    def ways(self):
        c = self.conn.cursor()

        c.execute( "SELECT id, tags, nds FROM ways" )

        for id, tags_str, nds_str in c:
            yield WayRecord( id, tags_str, nds_str )

        c.close()

    def count_ways(self):
        c = self.conn.cursor()

        c.execute( "SELECT count(*) FROM ways" )
        ret = c.next()[0]

        c.close()

        return ret

    def count_edges(self):
        c = self.conn.cursor()

        c.execute( "SELECT count(*) FROM edges" )
        ret = c.next()[0]

        c.close()

        return ret

    def delete_way(self, id):
        c = self.conn.cursor()

        c.execute("DELETE FROM ways WHERE id = ?", (id,))

        c.close()

    def bounds(self):
        c = self.conn.cursor()
        c.execute( "SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes" )

        ret = c.next()
        c.close()
        return ret

    def execute(self,sql,args=None):
        c = self.conn.cursor()
        if args:
            for row in c.execute(sql,args):
                yield row
        else:
            for row in c.execute(sql):
                yield row
        c.close()

    def cursor(self):
        return self.conn.cursor()

def test_wayrecord():
    wr = WayRecord( "1", {'highway':'bumpkis'}, ['1','2','3'] )
    assert wr.id == "1"
    assert wr.tags == {'highway':'bumpkis'}
    assert wr.nds == ['1','2','3']

    wr = WayRecord( "1", "{\"highway\":\"bumpkis\"}", "[\"1\",\"2\",\"3\"]" )
    assert wr.id == "1"
    assert wr.tags == {'highway':'bumpkis'}
    assert wr.nds == ['1','2','3']

def osm_to_osmdb(osm_filename, osmdb_filename, tolerant=False):
    osmdb = OSMDB( osmdb_filename, overwrite=True )
    osmdb.populate( osm_filename, accept=lambda tags: 'highway' in tags, reporter=sys.stdout )
    osmdb.create_and_populate_edges_table(tolerant)

def main():
    from sys import argv

    usage = "python osmdb.py osm_filename osmdb_filename"
    if len(argv) < 3:
        print usage
        exit()

    osm_filename = argv[1]
    osmdb_filename = argv[2]

    tolerant = 'tolerant' in argv

    osm_to_osmdb(osm_filename, osmdb_filename, tolerant)

if __name__=='__main__':
    main()
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Show the last run loaded into a repository."""

import sys

from stestr import output
from stestr.repository import util
from stestr import results
from stestr import subunit_trace

def get_cli_help():
    help_str = """Show the last run loaded into a repository.

    Failing tests are shown on the console and a summary of the run is printed
    at the end.

    Without --subunit, the process exit code will be non-zero if the test run
    was not successful. With --subunit, the process exit code is non-zero if
    the subunit stream could not be generated successfully.
    """
    return help_str

def set_cli_opts(parser):
    parser.add_argument(
        "--subunit", action="store_true",
        default=False, help="Show output as a subunit stream.")
    parser.add_argument("--no-subunit-trace", action='store_true',
                        default=False,
                        help="Disable output with the subunit-trace output "
                             "filter")
    parser.add_argument('--color', action='store_true', default=False,
                        help='Enable color output in the subunit-trace output,'
                             ' if subunit-trace output is enabled. (this is '
                             'the default). If subunit-trace is disable this '
                             ' does nothing.')

def run(arguments):
    args = arguments[0]
    pretty_out = not args.no_subunit_trace
    return last(repo_type=args.repo_type, repo_url=args.repo_url,
                subunit_out=args.subunit, pretty_out=pretty_out,
                color=args.color)

def last(repo_type='file', repo_url=None, subunit_out=False, pretty_out=True,
         color=False, stdout=sys.stdout):
    """Show the last run loaded into a a repository

    This function will print the results from the last run in the repository
    to STDOUT. It can optionally print the subunit stream for the last run
    to STDOUT if the ``subunit`` option is set to true.

    Note this function depends on the cwd for the repository if `repo_type` is
    set to file and `repo_url` is not specified it will use the repository
    located at CWD/.stestr

    :param str repo_type: This is the type of repository to use. Valid choices
        are 'file' and 'sql'.
    :param str repo_url: The url of the repository to use.
    :param bool subunit_out: Show output as a subunit stream.
    :param pretty_out: Use the subunit-trace output filter.
    :param color: Enable colorized output with the subunit-trace output filter.
    :param bool subunit: Show output as a subunit stream.
    :param file stdout: The output file to write all output to. By default
         this is sys.stdout

    :return return_code: The exit code for the command. 0 for success and > 0
        for failures.
    :rtype: int
    """
    repo = util.get_repo_open(repo_type, repo_url)
    latest_run = repo.get_latest_run()
    if subunit_out:
        stream = latest_run.get_subunit_stream()
        output.output_stream(stream, output=stdout)
        # Exits 0 if we successfully wrote the stream.
        return 0
    case = latest_run.get_test()
    try:
        if repo_type == 'file':
            previous_run = repo.get_test_run(repo.latest_id() - 1)
        # TODO(mtreinish): add a repository api to get the previous_run to
        # unify this logic
        else:
            previous_run = None
    except KeyError:
        previous_run = None
    failed = False
    if not pretty_out:
        output_result = results.CLITestResult(latest_run.get_id, stdout,
                                              previous_run)
        summary = output_result.get_summary()
        output_result.startTestRun()
        try:
            case.run(output_result)
        finally:
            output_result.stopTestRun()
        failed = not summary.wasSuccessful()
    else:
        stream = latest_run.get_subunit_stream()
        failed = subunit_trace.trace(stream, stdout, post_fails=True,
                                     color=color)
    if failed:
        return 1
    else:
        return 0
<commit_msg>Print just error message when no stestr repo<commit_after># Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Show the last run loaded into a repository."""

import sys

from stestr import output
from stestr.repository import abstract
from stestr.repository import util
from stestr import results
from stestr import subunit_trace

def get_cli_help():
    help_str = """Show the last run loaded into a repository.

    Failing tests are shown on the console and a summary of the run is printed
    at the end.

    Without --subunit, the process exit code will be non-zero if the test run
    was not successful. With --subunit, the process exit code is non-zero if
    the subunit stream could not be generated successfully.
    """
    return help_str

def set_cli_opts(parser):
    parser.add_argument(
        "--subunit", action="store_true",
        default=False, help="Show output as a subunit stream.")
    parser.add_argument("--no-subunit-trace", action='store_true',
                        default=False,
                        help="Disable output with the subunit-trace output "
                             "filter")
    parser.add_argument('--color', action='store_true', default=False,
                        help='Enable color output in the subunit-trace output,'
                             ' if subunit-trace output is enabled. (this is '
                             'the default). If subunit-trace is disable this '
                             ' does nothing.')

def run(arguments):
    args = arguments[0]
    pretty_out = not args.no_subunit_trace
    return last(repo_type=args.repo_type, repo_url=args.repo_url,
                subunit_out=args.subunit, pretty_out=pretty_out,
                color=args.color)

def last(repo_type='file', repo_url=None, subunit_out=False, pretty_out=True,
         color=False, stdout=sys.stdout):
    """Show the last run loaded into a a repository

    This function will print the results from the last run in the repository
    to STDOUT. It can optionally print the subunit stream for the last run
    to STDOUT if the ``subunit`` option is set to true.

    Note this function depends on the cwd for the repository if `repo_type` is
    set to file and `repo_url` is not specified it will use the repository
    located at CWD/.stestr

    :param str repo_type: This is the type of repository to use. Valid choices
        are 'file' and 'sql'.
    :param str repo_url: The url of the repository to use.
    :param bool subunit_out: Show output as a subunit stream.
    :param pretty_out: Use the subunit-trace output filter.
    :param color: Enable colorized output with the subunit-trace output filter.
    :param bool subunit: Show output as a subunit stream.
    :param file stdout: The output file to write all output to. By default
         this is sys.stdout

    :return return_code: The exit code for the command. 0 for success and > 0
        for failures.
    :rtype: int
    """
    try:
        repo = util.get_repo_open(repo_type, repo_url)
    except abstract.RepositoryNotFound as e:
        stdout.write(str(e) + '\n')
        return 1

    try:
        latest_run = repo.get_latest_run()
    except KeyError as e:
        stdout.write(str(e) + '\n')
        return 1

    if subunit_out:
        stream = latest_run.get_subunit_stream()
        output.output_stream(stream, output=stdout)
        # Exits 0 if we successfully wrote the stream.
        return 0
    case = latest_run.get_test()
    try:
        if repo_type == 'file':
            previous_run = repo.get_test_run(repo.latest_id() - 1)
        # TODO(mtreinish): add a repository api to get the previous_run to
        # unify this logic
        else:
            previous_run = None
    except KeyError:
        previous_run = None
    failed = False
    if not pretty_out:
        output_result = results.CLITestResult(latest_run.get_id, stdout,
                                              previous_run)
        summary = output_result.get_summary()
        output_result.startTestRun()
        try:
            case.run(output_result)
        finally:
            output_result.stopTestRun()
        failed = not summary.wasSuccessful()
    else:
        stream = latest_run.get_subunit_stream()
        failed = subunit_trace.trace(stream, stdout, post_fails=True,
                                     color=color)
    if failed:
        return 1
    else:
        return 0
<|endoftext|>