<commit_before>"""
OpenNSA JunOS backend.
"""

# configure snippet:

# opennsa@cph> configure <return>
# Entering configuration mode
#
#[edit]
#
#opennsa@cph# set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/0/5.1780
#
#[edit]
#
#opennsa@cph# set interfaces ge-1/0/5 unit 1780 vlan-id 1780 encapsulation vlan-ccc
#
#[edit]
#opennsa@cph# delete interfaces ge-1/0/5 unit 1780
#
#[edit]
#opennsa@cph# delete protocols connections interface-switch ps-to-netherlight-1780
#
#[edit]
#
# Basically stuff should end with [edit] :-)
#

import StringIO

from twisted.python import log
from twisted.internet import defer, protocol, reactor, endpoints
from twisted.conch import error
from twisted.conch.ssh import common, transport, keys, userauth, connection, channel

from opennsa import state
from opennsa.backends.common import calendar as reservationcalendar

# Stuff to get from config:
JUNOS_HOST              = 'cph.dcn.nordu.net'
JUNOS_HOST_PORT         = 22
JUNOS_HOST_FINGERPRINT  = 'd8:43:73:ee:a4:87:87:36:3f:c4:e5:3e:7c:d9:8b:d7'

USERNAME                = 'opennsa'
PUBLIC_KEY_PATH         = '/home/opennsa/.ssh/id_rsa.pub'
PRIVATE_KEY_PATH        = '/home/opennsa/.ssh/id_rsa'

#> set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/0/5.1780
#> set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/1/9.1780
#> set interfaces ge-1/0/5 unit 1780 vlan-id 1780 encapsulation vlan-ccc
#> set interfaces ge-1/1/9 unit 1780 vlan-id 1780 encapsulation vlan-ccc

# parameterized commands
COMMAND_CONFIGURE           = 'configure'
COMMAND_SET_CONNECTIONS     = 'set protocols connections interface-switch %(name)s interface %(interface)s' # connection name / interface
COMMAND_SET_INTERFACES      = 'set interfaces %(port)s unit %(vlan)s vlan-id %(vlan)s encapsulation vlan-ccc' # port / vlan / vlan
COMMAND_DELETE_INTERFACES   = 'delete interfaces %(port)s unit %(vlan)s' # port / vlan
COMMAND_DELETE_CONNECTIONS  = 'delete protocols connections interface-switch %(name)s' # name

LOG_SYSTEM = 'opennsa.JunOS'

PORT_MAPPING = {
    'ams-80'    : 'ge-1/0/5.1780',
    'ams-81'    : 'ge-1/0/5.1781',
    'ams-82'    : 'ge-1/0/5.1782',
    'ams-83'    : 'ge-1/0/5.1783',
    'ps-80'     : 'ge-1/1/9.1780',
    'ps-81'     : 'ge-1/1/9.1781',
    'ps-82'     : 'ge-1/1/9.1782',
    'ps-83'     : 'ge-1/1/9.1783'
}

NAME_PREFIX = 'ps-to-netherlight-'

def portToInterfaceVLAN(topo_port):

    interface = PORT_MAPPING[topo_port]
    port, vlan = interface.split('.')
    connection_name = NAME_PREFIX + vlan
    return connection_name, interface, port, vlan

def createConfigureCommands(source_port, dest_port):

    s_connection_name, s_interface, s_port, s_vlan = portToInterfaceVLAN(source_port)
    d_connection_name, d_interface, d_port, d_vlan = portToInterfaceVLAN(dest_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    cfg_conn_source = COMMAND_SET_CONNECTIONS % {'name':s_connection_name, 'interface':s_interface }
    cfg_conn_dest   = COMMAND_SET_CONNECTIONS % {'name':d_connection_name, 'interface':d_interface }
    cfg_intf_source = COMMAND_SET_INTERFACES  % {'port':s_port, 'vlan': s_vlan }
    cfg_intf_dest   = COMMAND_SET_INTERFACES  % {'port':d_port, 'vlan': d_vlan }

    commands = [ cfg_conn_source, cfg_conn_dest, cfg_intf_source, cfg_intf_dest ]
    return commands

def createDeleteCommands(source_port, dest_port):

    s_connection_name, s_interface, s_port, s_vlan = portToInterfaceVLAN(source_port)
    d_connection_name, d_interface, d_port, d_vlan = portToInterfaceVLAN(dest_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    del_intf_source = COMMAND_DELETE_INTERFACES  % {'port':s_port, 'vlan': s_vlan }
    del_intf_dest   = COMMAND_DELETE_INTERFACES  % {'port':d_port, 'vlan': d_vlan }
    del_conn        = COMMAND_DELETE_CONNECTIONS % {'name':s_connection_name } # , 'interface':s_interface }
#    del_conn_dest   = COMMAND_DELETE_CONNECTIONS % {'name':d_connection_name, 'interface':d_interface }

    commands = [ del_intf_source, del_intf_dest, del_conn ]
    return commands

# ----

class SSHClientTransport(transport.SSHClientTransport):

    def __init__(self, fingerprints=None):
        self.fingerprints = fingerprints or []
        self.secure_defer = defer.Deferred()

    def verifyHostKey(self, public_key, fingerprint):
        if fingerprint in self.fingerprints:
            return defer.succeed(1)
        else:
            return defer.fail(error.ConchError('Fingerprint not accepted'))

    def connectionSecure(self):
        self.secure_defer.callback(self)

class SSHClientFactory(protocol.ClientFactory):

    protocol = SSHClientTransport

    def __init__(self, fingerprints=None):
        # ClientFactory has no __init__ method, so we don't call it
        self.fingerprints = fingerprints or []

    def buildProtocol(self, addr):

        p = self.protocol(self.fingerprints)
        p.factory = self
        return p

class ClientUserAuth(userauth.SSHUserAuthClient):

    def __init__(self, user, connection, public_key_path, private_key_path):
        userauth.SSHUserAuthClient.__init__(self, user, connection)
        self.public_key_path    = public_key_path
        self.private_key_path   = private_key_path

    def getPassword(self, prompt = None):
        return # this says we won't do password authentication

    def getPublicKey(self):
        return keys.Key.fromFile(self.public_key_path)

    def getPrivateKey(self):
        return defer.succeed( keys.Key.fromFile(self.private_key_path) )

class SSHConnection(connection.SSHConnection):

    def __init__(self):
        connection.SSHConnection.__init__(self)
        self.service_started = defer.Deferred()

    def serviceStarted(self):
        self.service_started.callback(self)

class SSHChannel(channel.SSHChannel):

    name = 'session'

    def __init__(self, localWindow = 0, localMaxPacket = 0, remoteWindow = 0, remoteMaxPacket = 0, conn = None, data=None, avatar = None):
        channel.SSHChannel.__init__(self, localWindow = 0, localMaxPacket = 0, remoteWindow = 0, remoteMaxPacket = 0, conn = None, data=None, avatar = None)
        self.channel_open = defer.Deferred()

        self.line = ''

        self.wait_defer = None
        self.wait_line  = None

    def channelOpen(self, data):
        self.channel_open.callback(self)
        #print "CHANNEL OPEN"

    @defer.inlineCallbacks
    def sendCommands(self, commands):
        log.msg('sendCommands', system=LOG_SYSTEM)
#        set_connection = COMMAND_SET_CONNECTIONS
#        set_interface  = COMMAND_SET_INTERFACES

        try:
            yield self.conn.sendRequest(self, 'shell', '', wantReply=1)

            d = self.waitForLine('[edit]')
            self.write('configure\r\n')
            yield d

            log.msg('Entered configure mode', system=LOG_SYSTEM)

            for cmd in commands:
                log.msg('CMD> %s' % cmd, system=LOG_SYSTEM)
                d = self.waitForLine('[edit]')
                self.write(cmd + '\r\n')
                yield d

            d = self.waitForLine('[edit]')
            self.write('commit check')
            yield d
            log.msg('Commands sent', system=LOG_SYSTEM)

        except Exception, e:
            print "E", e

#        d = defer.Deferred()
#        print "BLOCK"
#        yield d

        self.sendEOF()
        self.closeIt()

        # read data, check for "commit complete"

        #return d

    def request_exit_status(self, data):
        if data and len(data) != 4:
            log.msg('Exit status data: %s' % data, system=LOG_SYSTEM)

    def waitForLine(self, line):
        self.wait_line = line
        self.wait_defer = defer.Deferred()
        return self.wait_defer

    def matchLine(self, line):
        if self.wait_line and self.wait_defer:
            if self.wait_line == line.strip():
#                print "=", line
                d = self.wait_defer
                self.wait_line  = None
                self.wait_defer = None
                d.callback(self)
#            else:
#                print "!", line

    def sendEOF(self, passthru=None):
        self.conn.sendEOF(self)
        return passthru

    def closeIt(self, passthru=None):
        self.loseConnection()
        return passthru

    def dataReceived(self, data):
        if len(data) == 0:
            pass
        else:
            self.line += data
            if '\n' in data:
                lines = [ line.strip() for line in self.line.split('\n') if line.strip() ]
                self.line = ''
                for l in lines:
                    self.matchLine(l)

#    def eofReceived(self):
#        log.msg("EOF received", system=LOG_SYSTEM)

#    def closed(self):
#        log.msg("SSHChannel closed", system=LOG_SYSTEM)

class JunOSCommandSender:

    def __init__(self):
        self.client = None
        self.proto  = None

    def createSSHConnection(self):

        def transportSecure(client, proto):
            return proto

        def gotProtocol(proto):
            proto.secure_defer.addCallback(transportSecure, proto)
            return proto.secure_defer

        # should check if there is an existing protocol in place
        if self.proto:
            log.msg('Reusing SSH connection', system=LOG_SYSTEM)
            return defer.succeed(self.proto)

        factory = SSHClientFactory( [ JUNOS_HOST_FINGERPRINT ] )
        point = endpoints.TCP4ClientEndpoint(reactor, JUNOS_HOST, JUNOS_HOST_PORT)
        d = point.connect(factory)
        d.addCallback(gotProtocol)
        return d

    def getSSHChannel(self):

        def serviceStarted(connection):
            channel = SSHChannel(conn = connection)
            connection.openChannel(channel)
            return channel.channel_open

        def gotProtocol(proto):
            ssh_connection = SSHConnection()
            proto.requestService(ClientUserAuth(USERNAME, ssh_connection, PUBLIC_KEY_PATH, PRIVATE_KEY_PATH))
            ssh_connection.service_started.addCallback(serviceStarted)
            return ssh_connection.service_started

        d = self.createSSHConnection()
        d.addCallback(gotProtocol)
        return d

    def provision(self, source_port, dest_port, start_time, end_time):

        commands = createConfigureCommands(source_port, dest_port)

        def gotChannel(channel):
            d = channel.sendCommands(commands)
            return d

        d = self.getSSHChannel()
        d.addCallback(gotChannel)
        return d

    def release(self, source_port, dest_port, start_time, end_time):

        commands = createDeleteCommands(source_port, dest_port)

        def gotChannel(channel):
            d = channel.sendCommands(commands)
            return d

        d = self.getSSHChannel()
        d.addCallback(gotChannel)
        return d

# --------

class JunOSBackend:

    def __init__(self, network_name):
        self.network_name = network_name
        self.command_sender = JunOSCommandSender()
        self.calendar = reservationcalendar.ReservationCalendar()

    def createConnection(self, source_port, dest_port, service_parameters):

        # probably need a short hand for this
        self.calendar.checkReservation(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.checkReservation(dest_port  , service_parameters.start_time, service_parameters.end_time)

        self.calendar.addConnection(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.addConnection(dest_port  , service_parameters.start_time, service_parameters.end_time)

        c = JunOSConnection(source_port, dest_port, service_parameters, self.command_sender, self.calendar)
        return c

class JunOSConnection:

    def __init__(self, source_port, dest_port, service_parameters, command_sender, calendar):
        self.source_port = source_port
        self.dest_port = dest_port
        self.service_parameters = service_parameters
        self.command_sender = command_sender
        self.calendar = calendar

        self.state = state.ConnectionState()

    def stps(self):
        return self.service_parameters.source_stp, self.service_parameters.dest_stp

    def reserve(self):
        # resource availability has already been checked in the backend during connection creation
        # since we assume no other NRM for this backend (OpenNSA have exclusive rights to the HW) there is nothing to do here
        self.state.switchState(state.RESERVING)

        self.state.switchState(state.RESERVED)

        # state transition to SCHEDULED should be added sometime
        return defer.succeed(self)

    def provision(self):

        def provisioned(_):
            self.state.switchState(state.PROVISIONED)
            return self

        # start the hardware, we'll worry about scheduling later
        self.state.switchState(state.PROVISIONING)
        d = self.command_sender.provision(self.source_port, self.dest_port, self.service_parameters.start_time, self.service_parameters.end_time)
        d.addCallback(provisioned)
        return d

    def release(self):

        def released(_):
            self.state.switchState(state.RESERVED)
            return self

        self.state.switchState(state.RELEASING)

        d = self.command_sender.release(self.source_port, self.dest_port, self.service_parameters.start_time, self.service_parameters.end_time)
        d.addCallback(released)
        return d

    def terminate(self):

        self.state.switchState(state.TERMINATING)

        self.calendar.removeConnection(self.source_port, self.service_parameters.start_time, self.service_parameters.end_time)
        self.calendar.removeConnection(self.dest_port  , self.service_parameters.start_time, self.service_parameters.end_time)

        self.state.switchState(state.TERMINATED)

        return defer.succeed(self)

<commit_msg>more junos updates - can create and tear down connection now, still needs more checking<commit_after>"""
OpenNSA JunOS backend.
"""

# configure snippet:

# opennsa@cph> configure <return>
# Entering configuration mode
#
#[edit]
#
#opennsa@cph# set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/0/5.1780
#
#[edit]
#
#opennsa@cph# set interfaces ge-1/0/5 unit 1780 vlan-id 1780 encapsulation vlan-ccc
#
#[edit]
#opennsa@cph# delete interfaces ge-1/0/5 unit 1780
#
#[edit]
#opennsa@cph# delete protocols connections interface-switch ps-to-netherlight-1780
#
#[edit]
#
# Basically stuff should end with [edit] :-)
#

import StringIO

from twisted.python import log
from twisted.internet import defer, protocol, reactor, endpoints
from twisted.conch import error
from twisted.conch.ssh import common, transport, keys, userauth, connection, channel

from opennsa import state
from opennsa.backends.common import calendar as reservationcalendar

# Stuff to get from config:
JUNOS_HOST              = 'cph.dcn.nordu.net'
JUNOS_HOST_PORT         = 22
JUNOS_HOST_FINGERPRINT  = 'd8:43:73:ee:a4:87:87:36:3f:c4:e5:3e:7c:d9:8b:d7'

USERNAME                = 'opennsa'
PUBLIC_KEY_PATH         = '/home/opennsa/.ssh/id_rsa.pub'
PRIVATE_KEY_PATH        = '/home/opennsa/.ssh/id_rsa'

#> set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/0/5.1780
#> set protocols connections interface-switch ps-to-netherlight-1780 interface ge-1/1/9.1780
#> set interfaces ge-1/0/5 unit 1780 vlan-id 1780 encapsulation vlan-ccc
#> set interfaces ge-1/1/9 unit 1780 vlan-id 1780 encapsulation vlan-ccc

# parameterized commands
COMMAND_CONFIGURE           = 'configure'
COMMAND_SET_CONNECTIONS     = 'set protocols connections interface-switch %(name)s interface %(interface)s' # connection name / interface
COMMAND_SET_INTERFACES      = 'set interfaces %(port)s unit %(vlan)s vlan-id %(vlan)s encapsulation vlan-ccc' # port / vlan / vlan
COMMAND_DELETE_INTERFACES   = 'delete interfaces %(port)s unit %(vlan)s' # port / vlan
COMMAND_DELETE_CONNECTIONS  = 'delete protocols connections interface-switch %(name)s' # name

LOG_SYSTEM = 'opennsa.JunOS'

PORT_MAPPING = {
    'ams-80'    : 'ge-1/0/5.1780',
    'ams-81'    : 'ge-1/0/5.1781',
    'ams-82'    : 'ge-1/0/5.1782',
    'ams-83'    : 'ge-1/0/5.1783',
    'ps-80'     : 'ge-1/1/9.1780',
    'ps-81'     : 'ge-1/1/9.1781',
    'ps-82'     : 'ge-1/1/9.1782',
    'ps-83'     : 'ge-1/1/9.1783'
}

NAME_PREFIX = 'ps-to-netherlight-'

def portToInterfaceVLAN(topo_port):

    interface = PORT_MAPPING[topo_port]
    port, vlan = interface.split('.')
    connection_name = NAME_PREFIX + vlan
    return connection_name, interface, port, vlan

def createConfigureCommands(source_port, dest_port):

    s_connection_name, s_interface, s_port, s_vlan = portToInterfaceVLAN(source_port)
    d_connection_name, d_interface, d_port, d_vlan = portToInterfaceVLAN(dest_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    cfg_conn_source = COMMAND_SET_CONNECTIONS % {'name':s_connection_name, 'interface':s_interface }
    cfg_conn_dest   = COMMAND_SET_CONNECTIONS % {'name':d_connection_name, 'interface':d_interface }
    cfg_intf_source = COMMAND_SET_INTERFACES  % {'port':s_port, 'vlan': s_vlan }
    cfg_intf_dest   = COMMAND_SET_INTERFACES  % {'port':d_port, 'vlan': d_vlan }

    commands = [ cfg_conn_source, cfg_conn_dest, cfg_intf_source, cfg_intf_dest ]
    return commands

def createDeleteCommands(source_port, dest_port):

    s_connection_name, s_interface, s_port, s_vlan = portToInterfaceVLAN(source_port)
    d_connection_name, d_interface, d_port, d_vlan = portToInterfaceVLAN(dest_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    del_intf_source = COMMAND_DELETE_INTERFACES  % {'port':s_port, 'vlan': s_vlan }
    del_intf_dest   = COMMAND_DELETE_INTERFACES  % {'port':d_port, 'vlan': d_vlan }
    del_conn        = COMMAND_DELETE_CONNECTIONS % {'name':s_connection_name }

    commands = [ del_intf_source, del_intf_dest, del_conn ]
    return commands

# ----

class SSHClientTransport(transport.SSHClientTransport):

    def __init__(self, fingerprints=None):
        self.fingerprints = fingerprints or []
        self.secure_defer = defer.Deferred()

    def verifyHostKey(self, public_key, fingerprint):
        if fingerprint in self.fingerprints:
            return defer.succeed(1)
        else:
            return defer.fail(error.ConchError('Fingerprint not accepted'))

    def connectionSecure(self):
        self.secure_defer.callback(self)

class SSHClientFactory(protocol.ClientFactory):

    protocol = SSHClientTransport

    def __init__(self, fingerprints=None):
        # ClientFactory has no __init__ method, so we don't call it
        self.fingerprints = fingerprints or []

    def buildProtocol(self, addr):

        p = self.protocol(self.fingerprints)
        p.factory = self
        return p

class ClientUserAuth(userauth.SSHUserAuthClient):

    def __init__(self, user, connection, public_key_path, private_key_path):
        userauth.SSHUserAuthClient.__init__(self, user, connection)
        self.public_key_path    = public_key_path
        self.private_key_path   = private_key_path

    def getPassword(self, prompt = None):
        return # this says we won't do password authentication

    def getPublicKey(self):
        return keys.Key.fromFile(self.public_key_path)

    def getPrivateKey(self):
        return defer.succeed( keys.Key.fromFile(self.private_key_path) )

class SSHConnection(connection.SSHConnection):

    def __init__(self):
        connection.SSHConnection.__init__(self)
        self.service_started = defer.Deferred()

    def serviceStarted(self):
        self.service_started.callback(self)

class SSHChannel(channel.SSHChannel):

    name = 'session'

    def __init__(self, localWindow = 0, localMaxPacket = 0, remoteWindow = 0, remoteMaxPacket = 0, conn = None, data=None, avatar = None):
        channel.SSHChannel.__init__(self, localWindow = 0, localMaxPacket = 0, remoteWindow = 0, remoteMaxPacket = 0, conn = None, data=None, avatar = None)
        self.channel_open = defer.Deferred()

        self.line = ''

        self.wait_defer = None
        self.wait_line  = None

    def channelOpen(self, data):
        self.channel_open.callback(self)
        #print "CHANNEL OPEN"

    @defer.inlineCallbacks
    def sendCommands(self, commands):
        LT = '\r' # line termination
        log.msg('sendCommands', system=LOG_SYSTEM)

        try:
            yield self.conn.sendRequest(self, 'shell', '', wantReply=1)

            d = self.waitForLine('[edit]')
            self.write('configure' + LT)
            yield d

            log.msg('Entered configure mode', system=LOG_SYSTEM)

            for cmd in commands:
                log.msg('CMD> %s' % cmd, system=LOG_SYSTEM)
                d = self.waitForLine('[edit]')
                self.write(cmd + LT)
                yield d

            # commit commands, check for 'commit complete' as success
            # not quite sure how to handle failure here

#            # test stuff
#            d = self.waitForLine('[edit]')
#            self.write('commit check' + LT)

            d = self.waitForLine('commit complete')
            self.write('commit' + LT)
            yield d

        except Exception, e:
            log.msg('Error sending commands: %s' % str(e))
            raise e

        log.msg('Commands successfully sent', system=LOG_SYSTEM)
        self.sendEOF()
        self.closeIt()

    def request_exit_status(self, data):
        if data and len(data) != 4:
            log.msg('Exit status data: %s' % data, system=LOG_SYSTEM)

    def waitForLine(self, line):
        self.wait_line = line
        self.wait_defer = defer.Deferred()
        return self.wait_defer

    def matchLine(self, line):
        if self.wait_line and self.wait_defer:
            if self.wait_line == line.strip():
                d = self.wait_defer
                self.wait_line  = None
                self.wait_defer = None
                d.callback(self)
            else:
                pass

    def sendEOF(self, passthru=None):
        self.conn.sendEOF(self)
        return passthru

    def closeIt(self, passthru=None):
        self.loseConnection()
        return passthru

    def dataReceived(self, data):
        if len(data) == 0:
            pass
        else:
            self.line += data
            if '\n' in data:
                lines = [ line.strip() for line in self.line.split('\n') if line.strip() ]
                self.line = ''
                for l in lines:
                    self.matchLine(l)

class JunOSCommandSender:

    def __init__(self):
        self.client = None
        self.proto  = None

    def createSSHConnection(self):

        def transportSecure(client, proto):
            return proto

        def gotProtocol(proto):
            proto.secure_defer.addCallback(transportSecure, proto)
            return proto.secure_defer

        # should check if there is an existing protocol in place
        if self.proto:
            log.msg('Reusing SSH connection', system=LOG_SYSTEM)
            return defer.succeed(self.proto)

        factory = SSHClientFactory( [ JUNOS_HOST_FINGERPRINT ] )
        point = endpoints.TCP4ClientEndpoint(reactor, JUNOS_HOST, JUNOS_HOST_PORT)
        d = point.connect(factory)
        d.addCallback(gotProtocol)
        return d

    def getSSHChannel(self):

        def serviceStarted(connection):
            channel = SSHChannel(conn = connection)
            connection.openChannel(channel)
            return channel.channel_open

        def gotProtocol(proto):
            ssh_connection = SSHConnection()
            proto.requestService(ClientUserAuth(USERNAME, ssh_connection, PUBLIC_KEY_PATH, PRIVATE_KEY_PATH))
            ssh_connection.service_started.addCallback(serviceStarted)
            return ssh_connection.service_started

        d = self.createSSHConnection()
        d.addCallback(gotProtocol)
        return d

    def provision(self, source_port, dest_port, start_time, end_time):

        commands = createConfigureCommands(source_port, dest_port)

        def gotChannel(channel):
            d = channel.sendCommands(commands)
            return d

        d = self.getSSHChannel()
        d.addCallback(gotChannel)
        return d

    def release(self, source_port, dest_port, start_time, end_time):

        commands = createDeleteCommands(source_port, dest_port)

        def gotChannel(channel):
            d = channel.sendCommands(commands)
            return d

        d = self.getSSHChannel()
        d.addCallback(gotChannel)
        return d

# --------

class JunOSBackend:

    def __init__(self, network_name):
        self.network_name = network_name
        self.command_sender = JunOSCommandSender()
        self.calendar = reservationcalendar.ReservationCalendar()

    def createConnection(self, source_port, dest_port, service_parameters):

        # probably need a short hand for this
        self.calendar.checkReservation(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.checkReservation(dest_port  , service_parameters.start_time, service_parameters.end_time)

        self.calendar.addConnection(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.addConnection(dest_port  , service_parameters.start_time, service_parameters.end_time)

        c = JunOSConnection(source_port, dest_port, service_parameters, self.command_sender, self.calendar)
        return c

class JunOSConnection:

    def __init__(self, source_port, dest_port, service_parameters, command_sender, calendar):
        self.source_port = source_port
        self.dest_port = dest_port
        self.service_parameters = service_parameters
        self.command_sender = command_sender
        self.calendar = calendar

        self.state = state.ConnectionState()

    def stps(self):
        return self.service_parameters.source_stp, self.service_parameters.dest_stp

    def reserve(self):
        # resource availability has already been checked in the backend during connection creation
        # since we assume no other NRM for this backend (OpenNSA have exclusive rights to the HW) there is nothing to do here
        self.state.switchState(state.RESERVING)

        self.state.switchState(state.RESERVED)

        # state transition to SCHEDULED should be added sometime
        return defer.succeed(self)

    def provision(self):

        def provisioned(_):
            self.state.switchState(state.PROVISIONED)
            return self

        # start the hardware, we'll worry about scheduling later
        self.state.switchState(state.PROVISIONING)
        d = self.command_sender.provision(self.source_port, self.dest_port, self.service_parameters.start_time, self.service_parameters.end_time)
        d.addCallback(provisioned)
        return d

    def release(self):

        def released(_):
            self.state.switchState(state.RESERVED)
            return self

        self.state.switchState(state.RELEASING)

        d = self.command_sender.release(self.source_port, self.dest_port, self.service_parameters.start_time, self.service_parameters.end_time)
        d.addCallback(released)
        return d

    def terminate(self):

        self.state.switchState(state.TERMINATING)

        self.calendar.removeConnection(self.source_port, self.service_parameters.start_time, self.service_parameters.end_time)
        self.calendar.removeConnection(self.dest_port  , self.service_parameters.start_time, self.service_parameters.end_time)

        self.state.switchState(state.TERMINATED)

        return defer.succeed(self)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import logging
import atexit
import tornado.auth
import tornado.autoreload
import tornado.escape
import tornado.ioloop
import tornado.options
import tornado.web
import os
import matplotlib
matplotlib.use('Agg')

from tornado.options import define, options

import cleanup

# Import all of our custom routes

# Upload routes
from handlers.upload import UploadHandler
from handlers.uploadVideo import UploadVideoHandler
from handlers.configHomography import ConfigHomographyHandler

# Configure routes
from handlers.config import ConfigHandler
from handlers.testConfig import TestConfigHandler
from handlers.defaultConfig import DefaultConfigHandler

# Analysis routes
from handlers.analysis import AnalysisHandler
from handlers.objectTracking import ObjectTrackingHandler
from handlers.safetyAnalysis import SafetyAnalysisHandler

#Status Routes
from handlers.status import StatusHandler

# Results routes
from handlers.createHighlightVideo import CreateHighlightVideoHandler
from handlers.makeReport import MakeReportHandler
from handlers.roadUserCounts import RoadUserCountsHandler
from handlers.createSpeedDistribution import CreateSpeedDistributionHandler
from handlers.retrieveResults import RetrieveResultsHandler
from handlers.createHighlightVideo import CreateHighlightVideoHandler

define("port", default=8888, help="run on the given port", type=int)
define("max_body_size", default=100*1024*1024, help="max size of content body", type=int)
define("max_buffer_size", default=100*1024*1024, help="max size loaded into memory", type=int)

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/upload", UploadHandler),
            (r"/uploadVideo", UploadVideoHandler),
            (r"/configHomography", ConfigHomographyHandler),
            (r"/config", ConfigHandler),
            (r"/testConfig", TestConfigHandler),
            (r"/defaultConfig", DefaultConfigHandler),
            (r"/analysis", AnalysisHandler),
            (r"/objectTracking", ObjectTrackingHandler),
            (r"/safetyAnalysis", SafetyAnalysisHandler),
            (r"/status", StatusHandler),
            (r"/highlightVideo", CreateHighlightVideoHandler),
            (r"/makeReport", MakeReportHandler),
            (r"/roadUserCounts", RoadUserCountsHandler),
            (r"/speedDistribution", CreateSpeedDistributionHandler),
            (r"/retrieveResults", RetrieveResultsHandler)

        ]
        settings = dict(
            cookie_secret=os.environ.get('SANTOSCLOUD_SECRET_KEY'),
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies=False,
        )
        tornado.web.Application.__init__(self, handlers, **settings)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("index.html")

def main():
    keys = ['SANTOSCLOUD_SECRET_KEY', 'SANTOSCLOUD_EMAIL', 'SANTOSCLOUD_EMAIL_PASSWORD']
    for key in keys:
        if os.environ.get(key) == None:
            print("Set the "+key+" environment variable")
            return

    if os.environ.get('SANTOSCLOUD_SECRET_KEY') == "DefaultSecretKey":
        print('WARNING: You are using the default secret key. This is insecure! Please create a secret key and set it on the SANTOSCLOUD_SECRET_KEY environment variable.')
    if os.environ.get('SANTOSCLOUD_EMAIL') == '' or os.environ.get('SANTOSCLOUD_EMAIL_PASSWORD') == '':
        print("WARNING: Running without email capabilities. Users won't be emailed when their processing completes. To fix this, set the SANTOSCLOUD_EMAIL and SANTOSCLOUD_EMAIL_PASSWORD environment variables.")

    if not os.path.exists(os.path.join(os.path.dirname(__file__),'..','.temp')):
        os.mkdir(os.path.join(os.path.dirname(__file__),'..','.temp'))
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port,\
               max_body_size = options.max_body_size,\
               max_buffer_size = options.max_buffer_size)
    print('Listening on port '+str(options.port))
    print('Max Body Size {} MB'.format(options.max_body_size/(1024*1024)))
    print('Max Buffer Size {} MB'.format(options.max_buffer_size/(1024*1024)))
    ioloop = tornado.ioloop.IOLoop().instance()
    tornado.autoreload.start(ioloop)
    ioloop.start()

if __name__ == "__main__":
    # If we're just starting the program, there shouldn't be anything already
    # running, so we should clean up, just in case something went terribly
    # wrong last time
    cleanup.cleanup_func()

    # Register for cleanup_func to be called after program exit
    atexit.register(cleanup.cleanup_func)

    main()

<commit_msg>Update app.py<commit_after>#!/usr/bin/env python

import logging
import atexit
import tornado.auth
import tornado.autoreload
import tornado.escape
import tornado.ioloop
import tornado.options
import tornado.web
import os
import matplotlib
matplotlib.use('Agg')

from tornado.options import define, options

import cleanup

# Import all of our custom routes

# Upload routes
from handlers.upload import UploadHandler
from handlers.uploadVideo import UploadVideoHandler
from handlers.configHomography import ConfigHomographyHandler

# Configure routes
from handlers.config import ConfigHandler
from handlers.testConfig import TestConfigHandler
from handlers.defaultConfig import DefaultConfigHandler

# Analysis routes
from handlers.analysis import AnalysisHandler
from handlers.objectTracking import ObjectTrackingHandler
from handlers.safetyAnalysis import SafetyAnalysisHandler

#Status Routes
from handlers.status import StatusHandler

# Results routes
from handlers.createHighlightVideo import CreateHighlightVideoHandler
from handlers.makeReport import MakeReportHandler
from handlers.roadUserCounts import RoadUserCountsHandler
from handlers.createSpeedDistribution import CreateSpeedDistributionHandler
from handlers.retrieveResults import RetrieveResultsHandler
from handlers.createHighlightVideo import CreateHighlightVideoHandler

MB = 1024*1024
define("port", default=8888, help="run on the given port", type=int)
define("max_body_size", default=100*MB, help="max size of content body", type=int)
define("max_buffer_size", default=100*MB, help="max size loaded into memory", type=int)

class Application(tornado.web.Application):
    def __init__(self):
        handlers = [
            (r"/", MainHandler),
            (r"/upload", UploadHandler),
            (r"/uploadVideo", UploadVideoHandler),
            (r"/configHomography", ConfigHomographyHandler),
            (r"/config", ConfigHandler),
            (r"/testConfig", TestConfigHandler),
            (r"/defaultConfig", DefaultConfigHandler),
            (r"/analysis", AnalysisHandler),
            (r"/objectTracking", ObjectTrackingHandler),
            (r"/safetyAnalysis", SafetyAnalysisHandler),
            (r"/status", StatusHandler),
            (r"/highlightVideo", CreateHighlightVideoHandler),
            (r"/makeReport", MakeReportHandler),
            (r"/roadUserCounts", RoadUserCountsHandler),
            (r"/speedDistribution", CreateSpeedDistributionHandler),
            (r"/retrieveResults", RetrieveResultsHandler)

        ]
        settings = dict(
            cookie_secret=os.environ.get('SANTOSCLOUD_SECRET_KEY'),
            template_path=os.path.join(os.path.dirname(__file__), "templates"),
            static_path=os.path.join(os.path.dirname(__file__), "static"),
            xsrf_cookies=False,
        )
        tornado.web.Application.__init__(self, handlers, **settings)

class MainHandler(tornado.web.RequestHandler):
    def get(self):
        self.render("index.html")

def main():
    keys = ['SANTOSCLOUD_SECRET_KEY', 'SANTOSCLOUD_EMAIL', 'SANTOSCLOUD_EMAIL_PASSWORD']
    for key in keys:
        if os.environ.get(key) == None:
            print("Set the "+key+" environment variable")
            return

    if os.environ.get('SANTOSCLOUD_SECRET_KEY') == "DefaultSecretKey":
        print('WARNING: You are using the default secret key. This is insecure! Please create a secret key and set it on the SANTOSCLOUD_SECRET_KEY environment variable.')
    if os.environ.get('SANTOSCLOUD_EMAIL') == '' or os.environ.get('SANTOSCLOUD_EMAIL_PASSWORD') == '':
        print("WARNING: Running without email capabilities. Users won't be emailed when their processing completes. To fix this, set the SANTOSCLOUD_EMAIL and SANTOSCLOUD_EMAIL_PASSWORD environment variables.")

    if not os.path.exists(os.path.join(os.path.dirname(__file__),'..','.temp')):
        os.mkdir(os.path.join(os.path.dirname(__file__),'..','.temp'))
    tornado.options.parse_command_line()
    app = Application()
    app.listen(options.port,\
               max_body_size = options.max_body_size,\
               max_buffer_size = options.max_buffer_size)
    print('Listening on port '+str(options.port))
    print('Max Body Size {} MB'.format(options.max_body_size/(1024*1024)))
    print('Max Buffer Size {} MB'.format(options.max_buffer_size/(1024*1024)))
    ioloop = tornado.ioloop.IOLoop().instance()
    tornado.autoreload.start(ioloop)
    ioloop.start()

if __name__ == "__main__":
    # If we're just starting the program, there shouldn't be anything already
    # running, so we should clean up, just in case something went terribly
    # wrong last time
    cleanup.cleanup_func()

    # Register for cleanup_func to be called after program exit
    atexit.register(cleanup.cleanup_func)

    main()

<|endoftext|>


<commit_before>"""Grow translation importers."""

import io
import collections
import copy
import csv
import errno
import logging
import os
import shutil
import tempfile
import zipfile
from babel.messages import catalog
from babel.messages import pofile

default_external_to_babel_locales = collections.defaultdict(list)
builtin_locales = {
    'en-GB': 'en_GB',
    'es-419': 'es_419',
    'fr-CA': 'fr_CA',
    'iw': 'he_IL',
    'no': 'nb_NO',
    'pt-BR': 'pt_BR',
    'pt-PT': 'pt_PT',
    'zh-CN': 'zh_Hans_CN',
    'zh-HK': 'zh_Hant_HK',
    'zh-TW': 'zh_Hant_TW',
}
for key, value in list(builtin_locales.items()):
    default_external_to_babel_locales[key].append(value)

def _mkdir(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            return
        raise

class Error(Exception):
    pass

class Importer(object):

    def __init__(self, pod, include_obsolete=True, untranslated=False):
        self.pod = pod
        self.include_obsolete = include_obsolete
        self.untranslated = untranslated

    def _validate_path(self, path):
        if not os.path.isfile(path):
            raise Error('Not found: {}'.format(path))

    def import_path(self, path, locale=None):
        if path.endswith('.zip'):
            self._validate_path(path)
            return self.import_zip_file(path)
        elif path.endswith('.po'):
            self._validate_path(path)
            return self.import_file(locale, path)
        elif os.path.isdir(path):
            return self.import_dir(path)
        elif path.endswith('.csv'):
            return self.import_csv_file(path)
        else:
            raise Error('Must import a .zip, .csv, .po file, or directory.')

    def import_csv_file(self, path):
        """Imports a CSV file formatted with locales in the header row and
        translations in the body rows."""
        default_locale = self.pod.podspec.localization.get('default_locale', 'en')
        locales_to_catalogs = {}
        with open(path) as fp:
            reader = csv.DictReader(fp)
            for row in reader:
                if default_locale not in row:
                    text = 'Locale {} not found in {}'.format(default_locale, path)
                    raise Error(text)
                msgid = row[default_locale]
                msgid = msgid
                for locale, translation in list(row.items()):
                    if locale == default_locale:
                        continue
                    translation = translation
                    message = catalog.Message(msgid, translation)
                    if locale not in locales_to_catalogs:
                        locales_to_catalogs[locale] = catalog.Catalog()
                    locales_to_catalogs[locale][msgid] = message
        for locale, catalog_obj in list(locales_to_catalogs.items()):
            fp = io.BytesIO()
            pofile.write_po(fp, catalog_obj)
            fp.seek(0)
            content = fp.read()
            self.import_content(locale, content)
            fp.close()

    def import_zip_file(self, zip_path):
        try:
            temp_dir_path = tempfile.mkdtemp()
            with zipfile.ZipFile(zip_path, 'r') as zip_file:
                zip_file.extractall(temp_dir_path)
            return self.import_dir(temp_dir_path)
        finally:
            shutil.rmtree(temp_dir_path)

    def import_dir(self, dir_path):
        # Track if the imported content is actually changing the translations.
        has_changed_content = False

        # Detect whether importing from another pod.
        podspec_path = os.path.join(dir_path, 'podspec.yaml')
        if os.path.exists(podspec_path):
            text = 'Importing from pod -> {}'.format(podspec_path)
            self.pod.logger.info(text)
            translations_dir = os.path.join(dir_path, 'translations')
            for locale in os.listdir(translations_dir):
                po_path = os.path.join(
                        translations_dir, locale, 'LC_MESSAGES', 'messages.po')
                if os.path.exists(po_path):
                    if self.import_file(locale, po_path):
                        has_changed_content = True
            return has_changed_content

        # TODO(jeremydw): Allow a custom syntax for translation importers.
        # Currently, assume one directory per locale.
        for locale in os.listdir(dir_path):
            locale_dir = os.path.join(dir_path, locale)
            if locale.startswith('.') or os.path.isfile(locale_dir):
                continue
            for basename in os.listdir(locale_dir):
                po_path = os.path.join(locale_dir, basename)
                if basename.endswith('.po'):
                    if self.import_file(locale, po_path):
                        has_changed_content = True
                else:
                    self.pod.logger.warning('Skipping: {}'.format(po_path))

        return has_changed_content

    def import_file(self, locale, po_path):
        if not os.path.exists(po_path):
            raise Error('Couldn\'t find PO file: {}'.format(po_path))
        content = open(po_path).read()
        return self.import_content(locale, content)

    def import_content(self, locale, content):
        if locale is None:
            raise Error('Must specify locale.')

        # Track if the imported content is actually changing the translations.
        has_changed_content = False

        # Leverage user-defined locale identifiers when importing translations.
        external_to_babel_locales = copy.deepcopy(
            default_external_to_babel_locales)
        if self.pod.podspec.localization:
            if 'import_as' in self.pod.podspec.localization:
                import_as = self.pod.podspec.localization['import_as']
                for external_locale, babel_locales in list(import_as.items()):
                    for babel_locale in babel_locales:
                        external_to_babel_locales[external_locale].append(
                            babel_locale)

        babel_locales = external_to_babel_locales.get(locale, [locale])
        for babel_locale in babel_locales:
            pod_translations_dir = os.path.join(
                'translations', babel_locale, 'LC_MESSAGES')
            pod_po_path = os.path.join(pod_translations_dir, 'messages.po')
            if self.pod.file_exists(pod_po_path):
                try:
                    existing_po_file = self.pod.open_file(pod_po_path)
                    existing_catalog = pofile.read_po(
                        existing_po_file, babel_locale)
                    po_file_to_merge = cStringIO.StringIO()
                    po_file_to_merge.write(content)
                    po_file_to_merge.seek(0)
                    catalog_to_merge = pofile.read_po(
                        po_file_to_merge, babel_locale)
                except Exception as e:
                    text = 'Error merging catalogs for locale -> {}'
                    logging.error(text.format(locale))
                    raise
                num_imported = 0
                for message in catalog_to_merge:
                    if message.id not in existing_catalog:
                        # Skip messages that don't exist if we are not
                        # including obsolete messages.
                        if not self.include_obsolete:
                            continue
                        existing_catalog[message.id] = message
                        if message.id:  # Only count the non-header messages.
                            num_imported += 1
                    elif (message.string
                          and existing_catalog[message.id].string != message.string):
                        # Skip messages that have translations in the catalog
                        # if we are only importing untranslated strings.
                        if self.untranslated and existing_catalog[message.id].string:
                            continue
                        # Avoid overwriting with empty/identical strings.
                        existing_catalog[message.id].string = message.string
                        num_imported += 1

                if num_imported > 0:
                    has_changed_content = True
                    existing_po_file = self.pod.open_file(pod_po_path, mode='wb')
                    pofile.write_po(existing_po_file, existing_catalog, width=80,
                                    sort_output=True, sort_by_file=True)
                    text = 'Updated {} of {} translations: {}'
                    message = text.format(num_imported, len(
                        catalog_to_merge), babel_locale)
                    self.pod.logger.info(message)
                else:
                    text = 'No translations updated: {}'
                    message = text.format(babel_locale)
                    self.pod.logger.info(message)
            else:
                # Skip new catalogs if not including obsolete messages.
                if not self.include_obsolete:
                    continue
                has_changed_content = True
                self.pod.write_file(pod_po_path, content)
                message = 'Imported new catalog: {}'.format(babel_locale)
                self.pod.logger.info(message)

        return has_changed_content
<commit_msg>Fixing the po write to decode bytes.<commit_after>"""Grow translation importers."""

import io
import collections
import copy
import csv
import errno
import logging
import os
import shutil
import tempfile
import zipfile
from babel.messages import catalog
from babel.messages import pofile

default_external_to_babel_locales = collections.defaultdict(list)
builtin_locales = {
    'en-GB': 'en_GB',
    'es-419': 'es_419',
    'fr-CA': 'fr_CA',
    'iw': 'he_IL',
    'no': 'nb_NO',
    'pt-BR': 'pt_BR',
    'pt-PT': 'pt_PT',
    'zh-CN': 'zh_Hans_CN',
    'zh-HK': 'zh_Hant_HK',
    'zh-TW': 'zh_Hant_TW',
}
for key, value in list(builtin_locales.items()):
    default_external_to_babel_locales[key].append(value)

def _mkdir(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST and os.path.isdir(path):
            return
        raise

class Error(Exception):
    pass

class Importer(object):

    def __init__(self, pod, include_obsolete=True, untranslated=False):
        self.pod = pod
        self.include_obsolete = include_obsolete
        self.untranslated = untranslated

    def _validate_path(self, path):
        if not os.path.isfile(path):
            raise Error('Not found: {}'.format(path))

    def import_path(self, path, locale=None):
        if path.endswith('.zip'):
            self._validate_path(path)
            return self.import_zip_file(path)
        elif path.endswith('.po'):
            self._validate_path(path)
            return self.import_file(locale, path)
        elif os.path.isdir(path):
            return self.import_dir(path)
        elif path.endswith('.csv'):
            return self.import_csv_file(path)
        else:
            raise Error('Must import a .zip, .csv, .po file, or directory.')

    def import_csv_file(self, path):
        """Imports a CSV file formatted with locales in the header row and
        translations in the body rows."""
        default_locale = self.pod.podspec.localization.get('default_locale', 'en')
        locales_to_catalogs = {}
        with open(path) as fp:
            reader = csv.DictReader(fp)
            for row in reader:
                if default_locale not in row:
                    text = 'Locale {} not found in {}'.format(default_locale, path)
                    raise Error(text)
                msgid = row[default_locale]
                msgid = msgid
                for locale, translation in list(row.items()):
                    if locale == default_locale:
                        continue
                    translation = translation
                    message = catalog.Message(msgid, translation)
                    if locale not in locales_to_catalogs:
                        locales_to_catalogs[locale] = catalog.Catalog()
                    locales_to_catalogs[locale][msgid] = message
        for locale, catalog_obj in list(locales_to_catalogs.items()):
            fp = io.BytesIO()
            pofile.write_po(fp, catalog_obj)
            fp.seek(0)
            content = fp.read()
            self.import_content(locale, content)
            fp.close()

    def import_zip_file(self, zip_path):
        try:
            temp_dir_path = tempfile.mkdtemp()
            with zipfile.ZipFile(zip_path, 'r') as zip_file:
                zip_file.extractall(temp_dir_path)
            return self.import_dir(temp_dir_path)
        finally:
            shutil.rmtree(temp_dir_path)

    def import_dir(self, dir_path):
        # Track if the imported content is actually changing the translations.
        has_changed_content = False

        # Detect whether importing from another pod.
        podspec_path = os.path.join(dir_path, 'podspec.yaml')
        if os.path.exists(podspec_path):
            text = 'Importing from pod -> {}'.format(podspec_path)
            self.pod.logger.info(text)
            translations_dir = os.path.join(dir_path, 'translations')
            for locale in os.listdir(translations_dir):
                po_path = os.path.join(
                        translations_dir, locale, 'LC_MESSAGES', 'messages.po')
                if os.path.exists(po_path):
                    if self.import_file(locale, po_path):
                        has_changed_content = True
            return has_changed_content

        # TODO(jeremydw): Allow a custom syntax for translation importers.
        # Currently, assume one directory per locale.
        for locale in os.listdir(dir_path):
            locale_dir = os.path.join(dir_path, locale)
            if locale.startswith('.') or os.path.isfile(locale_dir):
                continue
            for basename in os.listdir(locale_dir):
                po_path = os.path.join(locale_dir, basename)
                if basename.endswith('.po'):
                    if self.import_file(locale, po_path):
                        has_changed_content = True
                else:
                    self.pod.logger.warning('Skipping: {}'.format(po_path))

        return has_changed_content

    def import_file(self, locale, po_path):
        if not os.path.exists(po_path):
            raise Error('Couldn\'t find PO file: {}'.format(po_path))
        content = open(po_path).read()
        return self.import_content(locale, content)

    def import_content(self, locale, content):
        if locale is None:
            raise Error('Must specify locale.')

        # Track if the imported content is actually changing the translations.
        has_changed_content = False

        # Leverage user-defined locale identifiers when importing translations.
        external_to_babel_locales = copy.deepcopy(
            default_external_to_babel_locales)
        if self.pod.podspec.localization:
            if 'import_as' in self.pod.podspec.localization:
                import_as = self.pod.podspec.localization['import_as']
                for external_locale, babel_locales in list(import_as.items()):
                    for babel_locale in babel_locales:
                        external_to_babel_locales[external_locale].append(
                            babel_locale)

        babel_locales = external_to_babel_locales.get(locale, [locale])
        for babel_locale in babel_locales:
            pod_translations_dir = os.path.join(
                'translations', babel_locale, 'LC_MESSAGES')
            pod_po_path = os.path.join(pod_translations_dir, 'messages.po')
            if self.pod.file_exists(pod_po_path):
                try:
                    existing_po_file = self.pod.open_file(pod_po_path)
                    existing_catalog = pofile.read_po(
                        existing_po_file, babel_locale)
                    po_file_to_merge = io.StringIO()
                    try:
                        po_file_to_merge.write(content)
                    except TypeError:
                        po_file_to_merge.write(content.decode())
                    po_file_to_merge.seek(0)
                    catalog_to_merge = pofile.read_po(
                        po_file_to_merge, babel_locale)
                except Exception as e:
                    text = 'Error merging catalogs for locale -> {}'
                    logging.error(text.format(locale))
                    raise
                num_imported = 0
                for message in catalog_to_merge:
                    if message.id not in existing_catalog:
                        # Skip messages that don't exist if we are not
                        # including obsolete messages.
                        if not self.include_obsolete:
                            continue
                        existing_catalog[message.id] = message
                        if message.id:  # Only count the non-header messages.
                            num_imported += 1
                    elif (message.string
                          and existing_catalog[message.id].string != message.string):
                        # Skip messages that have translations in the catalog
                        # if we are only importing untranslated strings.
                        if self.untranslated and existing_catalog[message.id].string:
                            continue
                        # Avoid overwriting with empty/identical strings.
                        existing_catalog[message.id].string = message.string
                        num_imported += 1

                if num_imported > 0:
                    has_changed_content = True
                    existing_po_file = self.pod.open_file(pod_po_path, mode='wb')
                    pofile.write_po(existing_po_file, existing_catalog, width=80,
                                    sort_output=True, sort_by_file=True)
                    text = 'Updated {} of {} translations: {}'
                    message = text.format(num_imported, len(
                        catalog_to_merge), babel_locale)
                    self.pod.logger.info(message)
                else:
                    text = 'No translations updated: {}'
                    message = text.format(babel_locale)
                    self.pod.logger.info(message)
            else:
                # Skip new catalogs if not including obsolete messages.
                if not self.include_obsolete:
                    continue
                has_changed_content = True
                self.pod.write_file(pod_po_path, content)
                message = 'Imported new catalog: {}'.format(babel_locale)
                self.pod.logger.info(message)

        return has_changed_content
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import imaplib
import logging
import json
import re
import smtplib
import os

def setupLogging():
    if getConfigValue("settings", "loglevel") == "ERROR":
        logging.basicConfig(format="%(asctime)s %(message)s", level=logging.ERROR)
    else:
        logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)

def processVariable(variables, text):
    foundVarInText = re.findall(r"\$([A-Z]|[a-z]|[0-9])+", text)

    for var in foundVarInText:
        if var in variables:
            text = text.replace("$" + var, processVariable(mail, variables[var]))
    return text

def imapCommand(imapmail, command, uid, *args):
    logging.debug("\t%s %s %s" % (command, uid, " ".join(args)))

    # IMAP Command caller with error handling
    if uid:
        code, response = imapmail.uid(command, uid, *args)
    else:
        code, response = imapmail.uid(command, *args)

    if "OK" in code:
        response = response[0]
        if response:
            return response
        return
    else:
        logging.error("Server responded with Code '%s' for '%s %s %s'." % (code, command, uid, args))
        raise OSError.ConnectionError("Server responded with Code '%s' for '%s %s %s'." % (code, command, uid, args))
        return

def getConfigValue(*args):
    with open("config_default.json") as defaultsFile:
        defaultsValue = json.load(defaultsFile)
        if os.path.isfile(configPath):
            with open(configPath) as userFile:
                userValue = json.load(userFile)
                try:
                    for path in args:
                        userValue = userValue[path]
                    return userValue
                except KeyError:
                    pass
        try:
            for path in args:
                defaultsValue = defaultsValue[path]
            return defaultsValue
        except KeyError:
            pass

        logging.error("Tried to access config value at path %s, which doesn't exist." % os.sep.join(args))

def smtpMail(to, what):
    smtpmail = smtplib.SMTP(getConfigValue("login", "smtpmail_server"))
    smtpmail.ehlo()
    smtpmail.starttls()
    smtpmail.login(getConfigValue("login", "mail_address"), getConfigValue("login", "mail_password"))
    smtpmail.sendmail(getConfigValue("login", "mail_address"), to, what)
    smtpmail.quit()

def patch_imaplib():
    # imaplib is missing some essential commands.
    # Since we just need these passed through to the server, patch them in
    imaplib.Commands["MOVE"] = ("SELECTED",)
    imaplib.Commands["IDLE"] = ("AUTH", "SELECTED",)
    imaplib.Commands["DONE"] = ("AUTH", "SELECTED",)

def escapePath(path):
    if os.sep in path:
        logging.error("Found '%s' in '%s', possible attack." % (os.sep, path))
        path.replace(os.sep, "_")

    for pathElement in path.split(os.sep):
        if pathElement[0] == ".":
            logging.error("Found dot at beginning of filename, possible attack.")
            pathElement[0] = "_"
<commit_msg>Replace re.findall with re.sub (fix #10 and commit da7e0fb)<commit_after>from __future__ import unicode_literals

import imaplib
import logging
import json
import re
import smtplib
import os

def setupLogging():
    if getConfigValue("settings", "loglevel") == "ERROR":
        logging.basicConfig(format="%(asctime)s %(message)s", level=logging.ERROR)
    else:
        logging.basicConfig(format="%(asctime)s %(message)s", level=logging.DEBUG)

def processVariable(variables, text):
    return re.sub(r'\$([a-zA-Z0-9_]+)', lambda m: (processVariable(variables,str(variables[m.group(0)]))), text)

def imapCommand(imapmail, command, uid, *args):
    logging.debug("\t%s %s %s" % (command, uid, " ".join(args)))

    # IMAP Command caller with error handling
    if uid:
        code, response = imapmail.uid(command, uid, *args)
    else:
        code, response = imapmail.uid(command, *args)

    if "OK" in code:
        response = response[0]
        if response:
            return response
        return
    else:
        logging.error("Server responded with Code '%s' for '%s %s %s'." % (code, command, uid, args))
        raise OSError.ConnectionError("Server responded with Code '%s' for '%s %s %s'." % (code, command, uid, args))
        return

def getConfigValue(*args):
    with open("config_default.json") as defaultsFile:
        defaultsValue = json.load(defaultsFile)
        if os.path.isfile(configPath):
            with open(configPath) as userFile:
                userValue = json.load(userFile)
                try:
                    for path in args:
                        userValue = userValue[path]
                    return userValue
                except KeyError:
                    pass
        try:
            for path in args:
                defaultsValue = defaultsValue[path]
            return defaultsValue
        except KeyError:
            pass

        logging.error("Tried to access config value at path %s, which doesn't exist." % os.sep.join(args))

def smtpMail(to, what):
    smtpmail = smtplib.SMTP(getConfigValue("login", "smtpmail_server"))
    smtpmail.ehlo()
    smtpmail.starttls()
    smtpmail.login(getConfigValue("login", "mail_address"), getConfigValue("login", "mail_password"))
    smtpmail.sendmail(getConfigValue("login", "mail_address"), to, what)
    smtpmail.quit()

def patch_imaplib():
    # imaplib is missing some essential commands.
    # Since we just need these passed through to the server, patch them in
    imaplib.Commands["MOVE"] = ("SELECTED",)
    imaplib.Commands["IDLE"] = ("AUTH", "SELECTED",)
    imaplib.Commands["DONE"] = ("AUTH", "SELECTED",)

def escapePath(path):
    if os.sep in path:
        logging.error("Found '%s' in '%s', possible attack." % (os.sep, path))
        path.replace(os.sep, "_")

    for pathElement in path.split(os.sep):
        if pathElement[0] == ".":
            logging.error("Found dot at beginning of filename, possible attack.")
            pathElement[0] = "_"
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
description:    WRF part of wrfpy
license:        APACHE 2.0
author:         Ronald van Haren, NLeSC (r.vanharen@esciencecenter.nl)
'''

from config import config
from datetime import datetime
import glob
import os
import f90nml

class run_wrf(config):
  '''
  run_wrf is a subclass of config  # TODO: use better names
  '''
  def __init__(self, datestart, dateend):
    config.__init__(self)
    # TODO: wrf_run_dir should be flexible if running in UPP mode
    self.wrf_run_dir = self.config['filesystem']['wrf_run_dir']
    self.cleanup_previous_wrf_run()
    self.prepare_wrf_config(datestart, dateend)

  def cleanup_previous_wrf_run(self):
    from utils import silentremove
    '''
    cleanup initial/boundary conditions and namelist from previous WRF run
    '''
    # remove initial conditions (wrfinput files)
    for filename in glob.glob(os.path.join(
      self.config['filesystem']['wrf_run_dir'], 'wrfinput_d*')):
      silentremove(filename)
    # remove lateral boundary conditions (wrfbdy_d01)
    silentremove(os.path.join(self.config['filesystem']['wrf_run_dir'],
                              'wrfbdy_d01'))
    silentremove(os.path.join(self.config['filesystem']['wrf_run_dir'],
                              'namelist.input'))

  def prepare_wrf_config(self, datestart, dateend):
    '''
    Copy over default WRF namelist and modify time_control variables
    '''
    from datetime import datetime
    # check if both datestart and dateend are a datetime instance
    if not all([ isinstance(dt, datetime) for dt in [datestart, dateend] ]):
      raise TypeError("datestart and dateend must be an instance of datetime")
    # namelist.input target
    input_namelist = os.path.join(self.config['filesystem']['wrf_run_dir'],
                                  'namelist.input')
    # read WRF namelist in WRF work_dir
    wrf_nml = f90nml.read(os.path.join(self.config['filesystem']['wrf_run_dir'],
                                       'namelist.forecast'))
    # get number of domains
    ndoms = wrf_nml['domains']['max_dom']
    # check if ndoms is an integer and >0
    if not (isinstance(ndoms, int) and ndoms>0):
      raise ValueError("'domains_max_dom' namelist variable should be an " \
                      "integer>0")
    # define dictionary with time control values
    dict = {'time_control:start_year':datestart.year,
            'time_control:start_month':datestart.month,
            'time_control:start_day':datestart.day,
            'time_control:start_hour':datestart.hour,
            'time_control:start_date':datetime.strftime(datestart,
                                                        '%Y-%m-%d_%H:%M:%S')
            'time_control:end_year':dateend.year,
            'time_control:end_month':dateend.month,
            'time_control:end_day':dateend.day,
            'time_control:end_hour':dateend.hour,
            'time_control:end_date':datetime.strftime(dateend,
                                                      '%Y-%m-%d_%H:%M:%S')
            }
    import pdb; pdb.set_trace()
    # loop over dictionary and set start/end date parameters
    for el in dict.keys():
      wrf_nml[el.split(':')[0]][el.split(':')[1]] = dict[el] * ndoms

    # write namelist.input
    wrf_nml.write(os.path.join(
      self.config['filesystem']['wrf_run_dir'], 'namelist.input'))

  def run_real(self):
    '''
    run wrf real.exe
    '''
    # check if slurm_real.exe is defined
    if len(self.config['options_slurm']['slurm_real.exe']):
      # TODO: copy slurm script to wrf run dir (test if needed?)
      real_slurm = self.config['slurm']['slurm_real.exe']
      utils.check_file_exists(real_slurm)
      real_command = 'sbatch ' + real_slurm
    else:  # run locally
      real_command = os.path.join(self.config['filesystem']['wrf_dir'],
                              'main', 'real.exe')
      utils.check_file_exists(real_command)
    try:
      subprocess.check_call(real_command, cwd=self.wrf_rundir,
                            stdout=utils.devnull(), stderr=utils.devnull())
    except CalledProcessError:
      logger.error('real.exe failed %s:' %real_command)
      raise  # re-raise exception

  def run_wrf(self):
    '''
    run wrf.exe
    '''
    # check if slurm_wrf.exe is defined
    if len(self.config['options_slurm']['slurm_wrf.exe']):
      wrf_slurm = self.config['slurm']['slurm_wrf.exe']
      utils.check_file_exists(wrf_slurm)
      wrf_command = 'sbatch ' + wrf_slurm
    else:  # run locally
      wrf_command = os.path.join(self.config['filesystem']['wrf_dir'],
                              'main', 'wrf.exe')
      utils.check_file_exists(wrf_command)
    try:
      subprocess.check_call(wrf_command, cwd=self.wrf_rundir,
                            stdout=utils.devnull(), stderr=utils.devnull())
    except CalledProcessError:
      logger.error('wrf.exe failed %s:' %real_command)
      raise  # re-raise exception

if __name__=="__main__":
  datestart= datetime(2014,07,16,00)
  dateend = datetime(2014,07,17,00)
  wrf = run_wrf(datestart, dateend)
  wrf.run_real()
<commit_msg>add slurm calls for wrf.exe and real.exe<commit_after>#!/usr/bin/env python

'''
description:    WRF part of wrfpy
license:        APACHE 2.0
author:         Ronald van Haren, NLeSC (r.vanharen@esciencecenter.nl)
'''

from config import config
from datetime import datetime
import glob
import os
import f90nml

class run_wrf(config):
  '''
  run_wrf is a subclass of config  # TODO: use better names
  '''
  def __init__(self, datestart, dateend):
    config.__init__(self)
    # TODO: wrf_run_dir should be flexible if running in UPP mode
    self.wrf_run_dir = self.config['filesystem']['wrf_run_dir']
    self.cleanup_previous_wrf_run()
    self.prepare_wrf_config(datestart, dateend)

  def cleanup_previous_wrf_run(self):
    from utils import silentremove
    '''
    cleanup initial/boundary conditions and namelist from previous WRF run
    '''
    # remove initial conditions (wrfinput files)
    for filename in glob.glob(os.path.join(
      self.config['filesystem']['wrf_run_dir'], 'wrfinput_d*')):
      silentremove(filename)
    # remove lateral boundary conditions (wrfbdy_d01)
    silentremove(os.path.join(self.config['filesystem']['wrf_run_dir'],
                              'wrfbdy_d01'))
    silentremove(os.path.join(self.config['filesystem']['wrf_run_dir'],
                              'namelist.input'))

  def prepare_wrf_config(self, datestart, dateend):
    '''
    Copy over default WRF namelist and modify time_control variables
    '''
    from datetime import datetime
    # check if both datestart and dateend are a datetime instance
    if not all([ isinstance(dt, datetime) for dt in [datestart, dateend] ]):
      raise TypeError("datestart and dateend must be an instance of datetime")
    # namelist.input target
    input_namelist = os.path.join(self.config['filesystem']['wrf_run_dir'],
                                  'namelist.input')
    # read WRF namelist in WRF work_dir
    wrf_nml = f90nml.read(os.path.join(self.config['filesystem']['wrf_run_dir'],
                                       'namelist.forecast'))
    # get number of domains
    ndoms = wrf_nml['domains']['max_dom']
    # check if ndoms is an integer and >0
    if not (isinstance(ndoms, int) and ndoms>0):
      raise ValueError("'domains_max_dom' namelist variable should be an " \
                      "integer>0")
    # define dictionary with time control values
    dict = {'time_control:start_year':datestart.year,
            'time_control:start_month':datestart.month,
            'time_control:start_day':datestart.day,
            'time_control:start_hour':datestart.hour,
            'time_control:start_date':datetime.strftime(datestart,
                                                        '%Y-%m-%d_%H:%M:%S')
            'time_control:end_year':dateend.year,
            'time_control:end_month':dateend.month,
            'time_control:end_day':dateend.day,
            'time_control:end_hour':dateend.hour,
            'time_control:end_date':datetime.strftime(dateend,
                                                      '%Y-%m-%d_%H:%M:%S')
            }
    import pdb; pdb.set_trace()
    # loop over dictionary and set start/end date parameters
    for el in dict.keys():
      wrf_nml[el.split(':')[0]][el.split(':')[1]] = dict[el] * ndoms

    # write namelist.input
    wrf_nml.write(os.path.join(
      self.config['filesystem']['wrf_run_dir'], 'namelist.input'))

  def run_real(self, j_id=None):
    '''
    run wrf real.exe
    '''
    # check if slurm_real.exe is defined
    if len(self.config['options_slurm']['slurm_real.exe']):
      if j_id:
        mid = "--dependency=afterok:%d" %j_id
        real_command = ['sbatch', mid, self.config['options_slurm']['slurm_real.exe']]
      else:
        real_command = ['sbatch', self.config['options_slurm']['slurm_real.exe']]
      utils.check_file_exists(real_command[-1])
      utils.silentremove(os.path.join(self.wrf_rundir, 'real.exe'))
      os.symlink(os.path.join(self.config['filesystem']['wrf_dir'],'main','real.exe'),
                 os.path.join(self.wrf_rundir, 'real.exe'))
      try:
        res = subprocess.check_output(real_command, cwd=self.wrf_rundir,
                                      stderr=utils.devnull())
        j_id = int(res.split()[-1])  # slurm job-id
      except subprocess.CalledProcessError:
        logger.error('Real failed %s:' %real_command)
        raise  # re-raise exception
      return j_id  # return slurm job-id
    else:  # run locally
      real_command = os.path.join(self.config['filesystem']['wrf_dir'],
                              'main', 'real.exe')
      utils.check_file_exists(real_command)
      try:
        subprocess.check_call(real_command, cwd=self.wrf_rundir,
                              stdout=utils.devnull(), stderr=utils.devnull())
      except subprocess.CalledProcessError:
        logger.error('real.exe failed %s:' %real_command)
        raise  # re-raise exception

  def run_wrf(self, j_id=None):
    '''
    run wrf.exe
    '''
    # check if slurm_wrf.exe is defined
    if len(self.config['options_slurm']['slurm_wrf.exe']):
      if j_id:
        mid = "--dependency=afterok:%d" %j_id
        wrf_command = ['sbatch', mid, self.config['options_slurm']['slurm_wrf.exe']]
      else:
        wrf_command = ['sbatch', self.config['options_slurm']['slurm_wrf.exe']]
      utils.check_file_exists(wrf_command[-1])
      utils.silentremove(os.path.join(self.wrf_rundir, 'wrf.exe'))
      os.symlink(os.path.join(self.config['filesystem']['wrf_dir'],'main','wrf.exe'),
                 os.path.join(self.wrf_rundir, 'wrf.exe'))
      try:
        res = subprocess.check_output(wrf_command, cwd=self.wrf_rundir,
                                      stderr=utils.devnull())
        j_id = int(res.split()[-1])  # slurm job-id
      except subprocess.CalledProcessError:
        logger.error('Wrf failed %s:' %wrf_command)
        raise  # re-raise exception
      return j_id  # return slurm job-id
    else:  # run locally
      wrf_command = os.path.join(self.config['filesystem']['wrf_dir'],
                              'main', 'wrf.exe')
      utils.check_file_exists(wrf_command)
      try:
        subprocess.check_call(wrf_command, cwd=self.wrf_rundir,
                              stdout=utils.devnull(), stderr=utils.devnull())
      except subprocess.CalledProcessError:
        logger.error('wrf.exe failed %s:' %wrf_command)
        raise  # re-raise exception

if __name__=="__main__":
  datestart= datetime(2014,07,16,00)
  dateend = datetime(2014,07,17,00)
  wrf = run_wrf(datestart, dateend)
  real_jid = wrf.run_real()
<|endoftext|>


<commit_before><commit_msg>Add optional arg to allow displaying nodes without stripping tags<commit_after><|endoftext|>


<commit_before>import os
import sys
import json

import click

default_siteconfig = {
    "version": 2,
    "siteModules": {}
}

@click.command()
@click.option('--dir', default=os.getcwd(), prompt='Enter directory to create web-standard-app module',
              help='Directory where the web-standard-app module should be created')
@click.option('--name', prompt='Enter module name',
              help='The name of the web-standard-app module name')
@click.option('--js/--no-js', default=True,
              prompt='Do you want to add a basic javascript template?',
              help='These options will determine if a templated javascript file ought to be created')
@click.option('--css/--no-css', default=True,
              prompt='Do you want to add a basic css template?',
              help='These options will determine if a templated css file ought to be created')
@click.option('--config-version', default=2, prompt='Config version',
              help="This option sets the version for siteconfig.json")
@click.option('--template', default='primary', prompt='Template type', help='The type of template')
@click.option('--force', is_flag=True, default=False,
              help='This option will force the script to overwrite a currently existing module setup if the directory already exists')
def mkwsa(dir, name, js, css, config_version, template, force):
    """ Command line template for creating a basic web-standard-apps module """
    module_dir = os.path.join(dir, name)

    static_dir = os.path.join(module_dir, "static")
    css_dir = os.path.join(static_dir, "css")
    css_modules = "modules"
    css_modules_dir = os.path.join(css_dir, "modules")
    js_dir = os.path.join(static_dir, "js")
    js_modules = "modules"
    js_modules_dir = os.path.join(js_dir, "modules")

    site_config_dir = os.path.join(js_dir, "siteconfig")
    site_config_module_dir = os.path.join(site_config_dir, "modules")
    site_config_file = os.path.join(site_config_module_dir, "{}-siteconfig.json".format(name))

    template_dir = os.path.join(module_dir, "templates")
    template_module_dir = os.path.join(template_dir, "modules")
    template_module_name_dir = os.path.join(template_module_dir, name)
    template_module_file = os.path.join(template_module_name_dir, "{}-{}".format(name, template))

    click.echo("Creating module named {} in {} ...".format(name, dir))

    if os.path.isdir(module_dir):
        if not force:
            click.echo("Directory already exists, use --force option to overwrite module setup settings")
            sys.exit()
    else:
        # create base module folder
        os.mkdir(module_dir)

    # setup template folder structure and default template
    create_dir(template_dir)
    create_dir(template_module_dir)
    create_dir(template_module_name_dir)

    click.echo("Creating {}.html file in {} ...".format(template_module_file, template_module_name_dir))
    with open("{}.html".format(template_module_file), 'wb') as fp:
        output = '

'.format(name)
        content = bytes(output, 'UTF-8')
        fp.write(content)

    # setup static folder structure and config files
    create_dir(static_dir)
    create_dir(css_dir)
    create_dir(js_dir)

    create_dir(site_config_dir)
    create_dir(site_config_module_dir)

    default_siteconfig['version'] = config_version
    default_siteconfig['siteModules'][name] = {}

    # create js modules directory and templated js file
    if js:
        create_dir(js_modules_dir)
        default_siteconfig['siteModules']['path'] = "{}/{}".format(js_modules, name)

    # create css modules directory and templated css file
    if css:
        create_dir(css_modules_dir)
        default_siteconfig['siteModules']['css'] = "{}/{}".format(css_modules, name)

    click.echo("Creating {}-siteconfig.json file in {} ..." \
               .format(name, site_config_module_dir))
    create_json(site_config_file, default_siteconfig)

def create_dir(directory):
    if os.path.isdir(directory):
        click.echo("Found {}, skipping folder creation".format(directory))
    else:
        click.echo("Creating {} ...".format(directory))
        os.mkdir(directory)

def create_json(fname, data):
    """ Create JSON file

    :param fname: Name of the file to be saved
    :param data: Array or dictionary to be converted to JSON
    """
    with open(fname, 'w') as outfile:
        json.dump(data, outfile, indent=2, separators=(',', ': '))

    return True

if __name__ == '__main__':
    create_module()
<commit_msg>doc updates<commit_after># -*- coding: utf8 -*-
import os
import sys
import json

import click

default_siteconfig = {
    "version": 2,
    "siteModules": {}
}

@click.command()
@click.option('--dir', default=os.getcwd(), prompt='Enter directory to create web-standard-app module',
              help='Directory where the web-standard-app module should be created')
@click.option('--name', prompt='Enter module name',
              help='The name of the web-standard-app module name')
@click.option('--js/--no-js', default=True,
              prompt='Do you want to add a basic javascript template?',
              help='These options will determine if a templated javascript file ought to be created')
@click.option('--css/--no-css', default=True,
              prompt='Do you want to add a basic css template?',
              help='These options will determine if a templated css file ought to be created')
@click.option('--config-version', default=2, prompt='Config version',
              help="This option sets the version for siteconfig.json")
@click.option('--template', default='primary', prompt='Template type', help='The type of template')
@click.option('--force', is_flag=True, default=False,
              help='This option will force the script to overwrite a currently existing module setup if the directory already exists')
def mkwsa(dir, name, js, css, config_version, template, force):
    """ Command line template for creating a basic web-standard-apps module """
    module_dir = os.path.join(dir, name)

    static_dir = os.path.join(module_dir, "static")
    css_dir = os.path.join(static_dir, "css")
    css_modules = "modules"
    css_modules_dir = os.path.join(css_dir, "modules")
    js_dir = os.path.join(static_dir, "js")
    js_modules = "modules"
    js_modules_dir = os.path.join(js_dir, "modules")

    site_config_dir = os.path.join(js_dir, "siteconfig")
    site_config_module_dir = os.path.join(site_config_dir, "modules")
    site_config_file = os.path.join(site_config_module_dir, "{}-siteconfig.json".format(name))

    template_dir = os.path.join(module_dir, "templates")
    template_module_dir = os.path.join(template_dir, "modules")
    template_module_name_dir = os.path.join(template_module_dir, name)
    template_module_file = os.path.join(template_module_name_dir, "{}-{}".format(name, template))

    click.echo("Creating module named {} in {} ...".format(name, dir))

    if os.path.isdir(module_dir):
        if not force:
            click.echo("Directory already exists, use --force option to overwrite module setup settings")
            sys.exit()
    else:
        # create base module folder
        os.mkdir(module_dir)

    # setup template folder structure and default template
    create_dir(template_dir)
    create_dir(template_module_dir)
    create_dir(template_module_name_dir)

    click.echo("Creating {}.html file in {} ...".format(template_module_file, template_module_name_dir))
    with open("{}.html".format(template_module_file), 'wb') as fp:
        output = '

'.format(name)
        content = bytes(output, 'UTF-8')
        fp.write(content)

    # setup static folder structure and config files
    create_dir(static_dir)
    create_dir(css_dir)
    create_dir(js_dir)

    create_dir(site_config_dir)
    create_dir(site_config_module_dir)

    default_siteconfig['version'] = config_version
    default_siteconfig['siteModules'][name] = {}

    # create js modules directory and templated js file
    if js:
        create_dir(js_modules_dir)
        default_siteconfig['siteModules']['path'] = "{}/{}".format(js_modules, name)

    # create css modules directory and templated css file
    if css:
        create_dir(css_modules_dir)
        default_siteconfig['siteModules']['css'] = "{}/{}".format(css_modules, name)

    click.echo("Creating {}-siteconfig.json file in {} ..." \
               .format(name, site_config_module_dir))
    create_json(site_config_file, default_siteconfig)

def create_dir(directory):
    """ Create directory or display message if directory already exists

    :param director: The directory """
    if os.path.isdir(directory):
        click.echo("Found {}, skipping folder creation".format(directory))
    else:
        click.echo("Creating {} ...".format(directory))
        os.mkdir(directory)

def create_json(fname, data):
    """ Create JSON file

    :param fname: Name of the file to be saved
    :param data: Array or dictionary to be converted to JSON
    """
    with open(fname, 'w') as outfile:
        json.dump(data, outfile, indent=2, separators=(',', ': '))

    return True

if __name__ == '__main__':
    create_module()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import collections
import tempfile
import logging
import os.path
import cProfile
import pstats

from openquake.baselib import performance, general, sap, datastore
from openquake.hazardlib import valid
from openquake.commonlib import readinput, oqvalidation, logs
from openquake.calculators import base, views
from openquake.server import dbserver

calc_path = None  # set only when the flag --slowest is given

PStatData = collections.namedtuple(
    'PStatData', 'ncalls tottime percall cumtime percall2 path')

oqvalidation.OqParam.calculation_mode.validator.choices = tuple(
    base.calculators)

def get_pstats(pstatfile, n):
    """
    Return profiling information as an RST table.

    :param pstatfile: path to a .pstat file
    :param n: the maximum number of stats to retrieve
    """
    with tempfile.TemporaryFile(mode='w+') as stream:
        ps = pstats.Stats(pstatfile, stream=stream)
        ps.sort_stats('cumtime')
        ps.print_stats(n)
        stream.seek(0)
        lines = list(stream)
    for i, line in enumerate(lines):
        if line.startswith('   ncalls'):
            break
    data = []
    for line in lines[i + 2:]:
        columns = line.split()
        if len(columns) == 6:
            data.append(PStatData(*columns))
    rows = [(rec.ncalls, rec.cumtime, rec.path) for rec in data]
    # here is an example of the expected output table:
    # ====== ======= ========================================================
    # ncalls cumtime path
    # ====== ======= ========================================================
    # 1      33.502  commands/run.py:77(_run)
    # 1      33.483  calculators/base.py:110(run)
    # 1      25.166  calculators/classical.py:115(execute)
    # 1      25.104  baselib.parallel.py:249(apply_reduce)
    # 1      25.099  calculators/classical.py:41(classical)
    # 1      25.099  hazardlib/calc/hazard_curve.py:164(classical)
    return views.rst_table(rows, header='ncalls cumtime path'.split())

def run2(job_haz, job_risk, calc_id, concurrent_tasks, pdb, loglevel,
         exports, params):
    """
    Run both hazard and risk, one after the other
    """
    hcalc = base.calculators(readinput.get_oqparam(job_haz), calc_id)
    hcalc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
              exports=exports, **params)
    hc_id = hcalc.datastore.calc_id
    rcalc_id = logs.init(level=getattr(logging, loglevel.upper()))
    oq = readinput.get_oqparam(job_risk, hc_id=hc_id)
    rcalc = base.calculators(oq, rcalc_id)
    rcalc.run(pdb=pdb, exports=exports, **params)
    return rcalc

def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
    global calc_path
    assert len(job_inis) in (1, 2), job_inis
    # set the logs first of all
    calc_id = logs.init(level=getattr(logging, loglevel.upper()))
    with performance.Monitor('total runtime', measuremem=True) as monitor:
        if len(job_inis) == 1:  # run hazard or risk
            if hc:
                hc_id = hc[0]
                rlz_ids = hc[1:]
            else:
                hc_id = None
                rlz_ids = ()
            oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
            vars(oqparam).update(params)
            if hc_id and hc_id < 0:  # interpret negative calculation ids
                calc_ids = datastore.get_calc_ids()
                try:
                    hc_id = calc_ids[hc_id]
                except IndexError:
                    raise SystemExit(
                        'There are %d old calculations, cannot '
                        'retrieve the %s' % (len(calc_ids), hc_id))
            calc = base.calculators(oqparam, calc_id)
            calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
                     exports=exports, hazard_calculation_id=hc_id,
                     rlz_ids=rlz_ids)
        else:  # run hazard + risk
            calc = run2(
                job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb,
                loglevel, exports, params)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    print('See the output with silx view %s' % calc.datastore.filename)
    calc_path, _ = os.path.splitext(calc.datastore.filename)  # used below
    return calc

@sap.script
def run(job_ini, slowest=False, hc=None, param='', concurrent_tasks=None,
        exports='', loglevel='info', pdb=None):
    """
    Run a calculation bypassing the database layer
    """
    dbserver.ensure_on()
    if param:
        params = oqvalidation.OqParam.check(
            dict(p.split('=', 1) for p in param.split(',')))
    else:
        params = {}
    if slowest:
        prof = cProfile.Profile()
        stmt = ('_run(job_ini, concurrent_tasks, pdb, loglevel, hc, '
                'exports, params)')
        prof.runctx(stmt, globals(), locals())
        pstat = calc_path + '.pstat'
        prof.dump_stats(pstat)
        print('Saved profiling info in %s' % pstat)
        print(get_pstats(pstat, slowest))
    else:
        return _run(job_ini, concurrent_tasks, pdb, loglevel,
                    hc, exports, params)

run.arg('job_ini', 'calculation configuration file '
        '(or files, space-separated)', nargs='+')
run.opt('slowest', 'profile and show the slowest operations', type=int)
run.opt('hc', 'previous calculation ID', type=valid.hazard_id)
run.opt('param', 'override parameter with the syntax NAME=VALUE,...')
run.opt('concurrent_tasks', 'hint for the number of tasks to spawn',
        type=int)
run.opt('exports', 'export formats as a comma-separated string',
        type=valid.export_formats)
run.opt('loglevel', 'logging level',
        choices='debug info warn error critical'.split())
run.flg('pdb', 'enable post mortem debugging', '-d')
<commit_msg>[skip hazardlib][demos]<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2014-2019 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import unittest.mock
import collections
import tempfile
import logging
import os.path
import cProfile
import pstats

from openquake.baselib import performance, general, sap, datastore
from openquake.hazardlib import valid
from openquake.commonlib import readinput, oqvalidation, logs
from openquake.calculators import base, views
from openquake.server import dbserver

calc_path = None  # set only when the flag --slowest is given

PStatData = collections.namedtuple(
    'PStatData', 'ncalls tottime percall cumtime percall2 path')

oqvalidation.OqParam.calculation_mode.validator.choices = tuple(
    base.calculators)

def get_pstats(pstatfile, n):
    """
    Return profiling information as an RST table.

    :param pstatfile: path to a .pstat file
    :param n: the maximum number of stats to retrieve
    """
    with tempfile.TemporaryFile(mode='w+') as stream:
        ps = pstats.Stats(pstatfile, stream=stream)
        ps.sort_stats('cumtime')
        ps.print_stats(n)
        stream.seek(0)
        lines = list(stream)
    for i, line in enumerate(lines):
        if line.startswith('   ncalls'):
            break
    data = []
    for line in lines[i + 2:]:
        columns = line.split()
        if len(columns) == 6:
            data.append(PStatData(*columns))
    rows = [(rec.ncalls, rec.cumtime, rec.path) for rec in data]
    # here is an example of the expected output table:
    # ====== ======= ========================================================
    # ncalls cumtime path
    # ====== ======= ========================================================
    # 1      33.502  commands/run.py:77(_run)
    # 1      33.483  calculators/base.py:110(run)
    # 1      25.166  calculators/classical.py:115(execute)
    # 1      25.104  baselib.parallel.py:249(apply_reduce)
    # 1      25.099  calculators/classical.py:41(classical)
    # 1      25.099  hazardlib/calc/hazard_curve.py:164(classical)
    return views.rst_table(rows, header='ncalls cumtime path'.split())

def run2(job_haz, job_risk, calc_id, concurrent_tasks, pdb, loglevel,
         exports, params):
    """
    Run both hazard and risk, one after the other
    """
    hcalc = base.calculators(readinput.get_oqparam(job_haz), calc_id)
    hcalc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
              exports=exports, **params)
    hc_id = hcalc.datastore.calc_id
    rcalc_id = logs.init(level=getattr(logging, loglevel.upper()))
    oq = readinput.get_oqparam(job_risk, hc_id=hc_id)
    rcalc = base.calculators(oq, rcalc_id)
    rcalc.run(pdb=pdb, exports=exports, **params)
    return rcalc

def _run(job_inis, concurrent_tasks, pdb, loglevel, hc, exports, params):
    global calc_path
    assert len(job_inis) in (1, 2), job_inis
    # set the logs first of all
    calc_id = logs.init(level=getattr(logging, loglevel.upper()))
    with performance.Monitor('total runtime', measuremem=True) as monitor:
        with unittest.mock.patch.dict(os.environ, OQ_DISTRIBUTE='no'):
            if len(job_inis) == 1:  # run hazard or risk
                if hc:
                    hc_id = hc[0]
                    rlz_ids = hc[1:]
                else:
                    hc_id = None
                    rlz_ids = ()
                oqparam = readinput.get_oqparam(job_inis[0], hc_id=hc_id)
                vars(oqparam).update(params)
                if hc_id and hc_id < 0:  # interpret negative calculation ids
                    calc_ids = datastore.get_calc_ids()
                    try:
                        hc_id = calc_ids[hc_id]
                    except IndexError:
                        raise SystemExit(
                            'There are %d old calculations, cannot '
                            'retrieve the %s' % (len(calc_ids), hc_id))
                calc = base.calculators(oqparam, calc_id)
                calc.run(concurrent_tasks=concurrent_tasks, pdb=pdb,
                         exports=exports, hazard_calculation_id=hc_id,
                         rlz_ids=rlz_ids)
            else:  # run hazard + risk
                calc = run2(
                    job_inis[0], job_inis[1], calc_id, concurrent_tasks, pdb,
                    loglevel, exports, params)

    logging.info('Total time spent: %s s', monitor.duration)
    logging.info('Memory allocated: %s', general.humansize(monitor.mem))
    print('See the output with silx view %s' % calc.datastore.filename)
    calc_path, _ = os.path.splitext(calc.datastore.filename)  # used below
    return calc

@sap.script
def run(job_ini, slowest=False, hc=None, param='', concurrent_tasks=None,
        exports='', loglevel='info', pdb=None):
    """
    Run a calculation bypassing the database layer
    """
    dbserver.ensure_on()
    if param:
        params = oqvalidation.OqParam.check(
            dict(p.split('=', 1) for p in param.split(',')))
    else:
        params = {}
    if slowest:
        prof = cProfile.Profile()
        stmt = ('_run(job_ini, concurrent_tasks, pdb, loglevel, hc, '
                'exports, params)')
        prof.runctx(stmt, globals(), locals())
        pstat = calc_path + '.pstat'
        prof.dump_stats(pstat)
        print('Saved profiling info in %s' % pstat)
        print(get_pstats(pstat, slowest))
    else:
        return _run(job_ini, concurrent_tasks, pdb, loglevel,
                    hc, exports, params)

run.arg('job_ini', 'calculation configuration file '
        '(or files, space-separated)', nargs='+')
run.opt('slowest', 'profile and show the slowest operations', type=int)
run.opt('hc', 'previous calculation ID', type=valid.hazard_id)
run.opt('param', 'override parameter with the syntax NAME=VALUE,...')
run.opt('concurrent_tasks', 'hint for the number of tasks to spawn',
        type=int)
run.opt('exports', 'export formats as a comma-separated string',
        type=valid.export_formats)
run.opt('loglevel', 'logging level',
        choices='debug info warn error critical'.split())
run.flg('pdb', 'enable post mortem debugging', '-d')
<|endoftext|>