<commit_before>from __future__ import print_function
import re, threading, socket, SocketServer
from binascii import hexlify
from ..util import ipaddrs
from ..util.hkdf import HKDF
from ..const import TRANSIT_RELAY

class TransitError(Exception):
    pass

# The beginning of each TCP connection consists of the following handshake
# messages. The sender transmits the same text regardless of whether it is on
# the initiating/connecting end of the TCP connection, or on the
# listening/accepting side. Same for the receiver.
#
#  sender -> receiver: transit sender TXID_HEX ready\n\n
#  receiver -> sender: transit receiver RXID_HEX ready\n\n
#
# Any deviations from this result in the socket being closed. The handshake
# messages are designed to provoke an invalid response from other sorts of
# servers (HTTP, SMTP, echo).
#
# If the sender is satisfied with the handshake, and this is the first socket
# to complete negotiation, the sender does:
#
#  sender -> receiver: go\n
#
# and the next byte on the wire will be from the application.
#
# If this is not the first socket, the sender does:
#
#  sender -> receiver: nevermind\n
#
# and closes the socket.

# So the receiver looks for "transit sender TXID_HEX ready\n\ngo\n" and hangs
# up upon the first wrong byte. The sender lookgs for "transit receiver
# RXID_HEX ready\n\n" and then makes a first/not-first decision about sending
# "go\n" or "nevermind\n"+close().

def build_receiver_handshake(key):
    hexid = HKDF(key, 32, CTXinfo=b"transit_receiver")
    return "transit receiver %s ready\n\n" % hexlify(hexid)

def build_sender_handshake(key):
    hexid = HKDF(key, 32, CTXinfo=b"transit_sender")
    return "transit sender %s ready\n\n" % hexlify(hexid)

def build_relay_handshake(key):
    token = HKDF(key, 32, CTXinfo=b"transit_relay_token")
    return "please relay %s\n" % hexlify(token)

TIMEOUT=15

# 1: sender only transmits, receiver only accepts, both wait forever
# 2: sender also accepts, receiver also transmits
# 3: timeouts / stop when no more progress can be made
# 4: add relay
# 5: accelerate shutdown of losing sockets

class BadHandshake(Exception):
    pass

def force_ascii(s):
    if isinstance(s, type(u"")):
        return s.encode("ascii")
    return s

def send_to(skt, data):
    sent = 0
    while sent < len(data):
        sent += skt.send(data[sent:])

def wait_for(skt, expected, description):
    got = b""
    while len(got) < len(expected):
        got += skt.recv(1)
        if expected[:len(got)] != got:
            raise BadHandshake("got '%r' want '%r' on %s" %
                               (got, expected, description))

# The hint format is: TYPE,VALUE= /^([a-zA-Z0-9]+):(.*)$/ . VALUE depends
# upon TYPE, and it can have more colons in it. For TYPE=tcp (the only one
# currently defined), ADDR,PORT = /^(.*):(\d+)$/ , so ADDR can have colons.
# ADDR can be a hostname, ipv4 dotted-quad, or ipv6 colon-hex. If the hint
# publisher wants anonymity, their only hint's ADDR will end in .onion .

def parse_hint_tcp(hint):
    # return tuple or None for an unparseable hint
    mo = re.search(r'^([a-zA-Z0-9]+):(.*)$', hint)
    if not mo:
        print("unparseable hint '%s'" % (hint,))
        return None
    hint_type = mo.group(1)
    if hint_type != "tcp":
        print("unknown hint type '%s' in '%s'" % (hint_type, hint))
        return None
    hint_value = mo.group(2)
    mo = re.search(r'^(.*):(\d+)$', hint_value)
    if not mo:
        print("unparseable TCP hint '%s'" % (hint,))
        return None
    hint_host = mo.group(1)
    try:
        hint_port = int(mo.group(2))
    except ValueError:
        print("non-numeric port in TCP hint '%s'" % (hint,))
        return None
    return hint_host, hint_port

def connector(owner, hint, description,
              send_handshake, expected_handshake, relay_handshake=None):
    parsed_hint = parse_hint_tcp(hint)
    if not parsed_hint:
        return # unparseable
    addr,port = parsed_hint
    skt = None
    try:
        skt = socket.create_connection((addr,port),
                                       TIMEOUT) # timeout or ECONNREFUSED
        skt.settimeout(TIMEOUT)
        #print("socket(%s) connected" % (description,))
        if relay_handshake:
            send_to(skt, relay_handshake)
            wait_for(skt, "ok\n", description)
            #print("relay ready %r" % (description,))
        send_to(skt, send_handshake)
        wait_for(skt, expected_handshake, description)
        #print("connector ready %r" % (description,))
    except Exception as e:
        try:
            if skt:
                skt.shutdown(socket.SHUT_WR)
        except socket.error:
            pass
        if skt:
            skt.close()
        # ignore socket errors, warn about coding errors
        if not isinstance(e, (socket.error, socket.timeout, BadHandshake)):
            raise
        owner._connector_failed(hint)
        return
    # owner is now responsible for the socket
    owner._negotiation_finished(skt, description) # note thread

def handle(skt, client_address, owner, description,
           send_handshake, expected_handshake):
    try:
        #print("handle %r" %  (skt,))
        skt.settimeout(TIMEOUT)
        send_to(skt, send_handshake)
        got = b""
        # for the receiver, this includes the "go\n"
        while len(got) < len(expected_handshake):
            more = skt.recv(1)
            if not more:
                raise BadHandshake("disconnect after merely '%r'" % got)
            got += more
            if expected_handshake[:len(got)] != got:
                raise BadHandshake("got '%r' want '%r'" %
                                   (got, expected_handshake))
        #print("handler negotiation finished %r" % (client_address,))
    except Exception as e:
        #print("handler failed %r" % (client_address,))
        try:
            # this raises socket.err(EBADF) if the socket was already closed
            skt.shutdown(socket.SHUT_WR)
        except socket.error:
            pass
        skt.close() # this appears to be idempotent
        # ignore socket errors, warn about coding errors
        if not isinstance(e, (socket.error, socket.timeout, BadHandshake)):
            raise
        return
    # owner is now responsible for the socket
    owner._negotiation_finished(skt, description) # note thread

class MyTCPServer(SocketServer.TCPServer):
    allow_reuse_address = True

    def process_request(self, request, client_address):
        description = "<-%s" % (client_address,)
        kc = self.owner._have_transit_key
        kc.acquire()
        while not self.owner._transit_key:
            kc.wait()
        # owner._transit_key is either None or set to a value. We don't
        # modify it from here, so we can release the condition lock before
        # grabbing the key.
        kc.release()

        # Once it is set, we can get handler_(send|receive)_handshake, which
        # is what we actually care about.
        t = threading.Thread(target=handle,
                             args=(request, client_address,
                                   self.owner, description,
                                   self.owner.handler_send_handshake,
                                   self.owner.handler_expected_handshake))
        t.daemon = True
        t.start()

class Common:
    def __init__(self):
        self.winning = threading.Event()
        self._negotiation_check_lock = threading.Lock()
        self._have_transit_key = threading.Condition()
        self._transit_key = None
        self._start_server()

    def _start_server(self):
        server = MyTCPServer(("", 0), None)
        _, port = server.server_address
        self.my_direct_hints = ["tcp:%s:%d" % (addr, port)
                                for addr in ipaddrs.find_addresses()]
        server.owner = self
        server_thread = threading.Thread(target=server.serve_forever)
        server_thread.daemon = True
        server_thread.start()
        self.listener = server

    def get_direct_hints(self):
        return self.my_direct_hints
    def get_relay_hints(self):
        return [TRANSIT_RELAY]

    def add_their_direct_hints(self, hints):
        self._their_direct_hints = [force_ascii(h) for h in hints]
    def add_their_relay_hints(self, hints):
        self._their_relay_hints = [force_ascii(h) for h in hints]

    def _send_this(self):
        if self.is_sender:
            return build_sender_handshake(self._transit_key)
        else:
            return build_receiver_handshake(self._transit_key)

    def _expect_this(self):
        if self.is_sender:
            return build_receiver_handshake(self._transit_key)
        else:
            return build_sender_handshake(self._transit_key) + "go\n"

    def set_transit_key(self, key):
        # This _have_transit_key condition/lock protects us against the race
        # where the sender knows the hints and the key, and connects to the
        # receiver's transit socket before the receiver gets relay message
        # (and thus the key).
        self._have_transit_key.acquire()
        self._transit_key = key
        self.handler_send_handshake = self._send_this() # no "go"
        self.handler_expected_handshake = self._expect_this()
        self._have_transit_key.notify_all()
        self._have_transit_key.release()

    def _start_outbound(self):
        self._active_connectors = set(self._their_direct_hints)
        for hint in self._their_direct_hints:
            self._start_connector(hint)
        if not self._their_direct_hints:
            self._start_relay_connectors()

    def _start_connector(self, hint, is_relay=False):
        description = "->%s" % (hint,)
        if is_relay:
            description = "->relay:%s" % (hint,)
        args = (self, hint, description,
                self._send_this(), self._expect_this())
        if is_relay:
            args = args + (build_relay_handshake(self._transit_key),)
        t = threading.Thread(target=connector, args=args)
        t.daemon = True
        t.start()

    def _start_relay_connectors(self):
        for hint in self._their_relay_hints:
            self._start_connector(hint, is_relay=True)

    def establish_connection(self):
        self.winning_skt = None
        self.winning_skt_description = None
        self._start_outbound()

        # we sit here until one of our inbound or outbound sockets succeeds
        flag = self.winning.wait(TIMEOUT)

        if not flag:
            # timeout: self.winning_skt will not be set. ish. race.
            pass
        if self.listener:
            self.listener.shutdown() # TODO: waits up to 0.5s. push to thread
        if self.winning_skt:
            return self.winning_skt
        raise TransitError

    def describe(self):
        if not self.winning_skt_description:
            return "not yet established"
        return self.winning_skt_description

    def _connector_failed(self, hint):
        self._active_connectors.remove(hint)
        if not self._active_connectors:
            self._start_relay_connectors()

    def _negotiation_finished(self, skt, description):
        # inbound/outbound sockets call this when they finish negotiation.
        # The first one wins and gets a "go". Any subsequent ones lose and
        # get a "nevermind" before being closed.

        with self._negotiation_check_lock:
            if self.winning_skt:
                is_winner = False
            else:
                is_winner = True
                self.winning_skt = skt
                self.winning_skt_description = description

        if is_winner:
            if self.is_sender:
                send_to(skt, "go\n")
            self.winning.set()
        else:
            if self.is_sender:
                send_to(skt, "nevermind\n")
            skt.close()

class TransitSender(Common):
    is_sender = True

class TransitReceiver(Common):
    is_sender = False
<commit_msg>format inbound-hint better<commit_after>from __future__ import print_function
import re, threading, socket, SocketServer
from binascii import hexlify
from ..util import ipaddrs
from ..util.hkdf import HKDF
from ..const import TRANSIT_RELAY

class TransitError(Exception):
    pass

# The beginning of each TCP connection consists of the following handshake
# messages. The sender transmits the same text regardless of whether it is on
# the initiating/connecting end of the TCP connection, or on the
# listening/accepting side. Same for the receiver.
#
#  sender -> receiver: transit sender TXID_HEX ready\n\n
#  receiver -> sender: transit receiver RXID_HEX ready\n\n
#
# Any deviations from this result in the socket being closed. The handshake
# messages are designed to provoke an invalid response from other sorts of
# servers (HTTP, SMTP, echo).
#
# If the sender is satisfied with the handshake, and this is the first socket
# to complete negotiation, the sender does:
#
#  sender -> receiver: go\n
#
# and the next byte on the wire will be from the application.
#
# If this is not the first socket, the sender does:
#
#  sender -> receiver: nevermind\n
#
# and closes the socket.

# So the receiver looks for "transit sender TXID_HEX ready\n\ngo\n" and hangs
# up upon the first wrong byte. The sender lookgs for "transit receiver
# RXID_HEX ready\n\n" and then makes a first/not-first decision about sending
# "go\n" or "nevermind\n"+close().

def build_receiver_handshake(key):
    hexid = HKDF(key, 32, CTXinfo=b"transit_receiver")
    return "transit receiver %s ready\n\n" % hexlify(hexid)

def build_sender_handshake(key):
    hexid = HKDF(key, 32, CTXinfo=b"transit_sender")
    return "transit sender %s ready\n\n" % hexlify(hexid)

def build_relay_handshake(key):
    token = HKDF(key, 32, CTXinfo=b"transit_relay_token")
    return "please relay %s\n" % hexlify(token)

TIMEOUT=15

# 1: sender only transmits, receiver only accepts, both wait forever
# 2: sender also accepts, receiver also transmits
# 3: timeouts / stop when no more progress can be made
# 4: add relay
# 5: accelerate shutdown of losing sockets

class BadHandshake(Exception):
    pass

def force_ascii(s):
    if isinstance(s, type(u"")):
        return s.encode("ascii")
    return s

def send_to(skt, data):
    sent = 0
    while sent < len(data):
        sent += skt.send(data[sent:])

def wait_for(skt, expected, description):
    got = b""
    while len(got) < len(expected):
        got += skt.recv(1)
        if expected[:len(got)] != got:
            raise BadHandshake("got '%r' want '%r' on %s" %
                               (got, expected, description))

# The hint format is: TYPE,VALUE= /^([a-zA-Z0-9]+):(.*)$/ . VALUE depends
# upon TYPE, and it can have more colons in it. For TYPE=tcp (the only one
# currently defined), ADDR,PORT = /^(.*):(\d+)$/ , so ADDR can have colons.
# ADDR can be a hostname, ipv4 dotted-quad, or ipv6 colon-hex. If the hint
# publisher wants anonymity, their only hint's ADDR will end in .onion .

def parse_hint_tcp(hint):
    # return tuple or None for an unparseable hint
    mo = re.search(r'^([a-zA-Z0-9]+):(.*)$', hint)
    if not mo:
        print("unparseable hint '%s'" % (hint,))
        return None
    hint_type = mo.group(1)
    if hint_type != "tcp":
        print("unknown hint type '%s' in '%s'" % (hint_type, hint))
        return None
    hint_value = mo.group(2)
    mo = re.search(r'^(.*):(\d+)$', hint_value)
    if not mo:
        print("unparseable TCP hint '%s'" % (hint,))
        return None
    hint_host = mo.group(1)
    try:
        hint_port = int(mo.group(2))
    except ValueError:
        print("non-numeric port in TCP hint '%s'" % (hint,))
        return None
    return hint_host, hint_port

def connector(owner, hint, description,
              send_handshake, expected_handshake, relay_handshake=None):
    parsed_hint = parse_hint_tcp(hint)
    if not parsed_hint:
        return # unparseable
    addr,port = parsed_hint
    skt = None
    try:
        skt = socket.create_connection((addr,port),
                                       TIMEOUT) # timeout or ECONNREFUSED
        skt.settimeout(TIMEOUT)
        #print("socket(%s) connected" % (description,))
        if relay_handshake:
            send_to(skt, relay_handshake)
            wait_for(skt, "ok\n", description)
            #print("relay ready %r" % (description,))
        send_to(skt, send_handshake)
        wait_for(skt, expected_handshake, description)
        #print("connector ready %r" % (description,))
    except Exception as e:
        try:
            if skt:
                skt.shutdown(socket.SHUT_WR)
        except socket.error:
            pass
        if skt:
            skt.close()
        # ignore socket errors, warn about coding errors
        if not isinstance(e, (socket.error, socket.timeout, BadHandshake)):
            raise
        owner._connector_failed(hint)
        return
    # owner is now responsible for the socket
    owner._negotiation_finished(skt, description) # note thread

def handle(skt, client_address, owner, description,
           send_handshake, expected_handshake):
    try:
        #print("handle %r" %  (skt,))
        skt.settimeout(TIMEOUT)
        send_to(skt, send_handshake)
        got = b""
        # for the receiver, this includes the "go\n"
        while len(got) < len(expected_handshake):
            more = skt.recv(1)
            if not more:
                raise BadHandshake("disconnect after merely '%r'" % got)
            got += more
            if expected_handshake[:len(got)] != got:
                raise BadHandshake("got '%r' want '%r'" %
                                   (got, expected_handshake))
        #print("handler negotiation finished %r" % (client_address,))
    except Exception as e:
        #print("handler failed %r" % (client_address,))
        try:
            # this raises socket.err(EBADF) if the socket was already closed
            skt.shutdown(socket.SHUT_WR)
        except socket.error:
            pass
        skt.close() # this appears to be idempotent
        # ignore socket errors, warn about coding errors
        if not isinstance(e, (socket.error, socket.timeout, BadHandshake)):
            raise
        return
    # owner is now responsible for the socket
    owner._negotiation_finished(skt, description) # note thread

class MyTCPServer(SocketServer.TCPServer):
    allow_reuse_address = True

    def process_request(self, request, client_address):
        description = "<-tcp:%s:%d" % (client_address[0], client_address[1])
        kc = self.owner._have_transit_key
        kc.acquire()
        while not self.owner._transit_key:
            kc.wait()
        # owner._transit_key is either None or set to a value. We don't
        # modify it from here, so we can release the condition lock before
        # grabbing the key.
        kc.release()

        # Once it is set, we can get handler_(send|receive)_handshake, which
        # is what we actually care about.
        t = threading.Thread(target=handle,
                             args=(request, client_address,
                                   self.owner, description,
                                   self.owner.handler_send_handshake,
                                   self.owner.handler_expected_handshake))
        t.daemon = True
        t.start()

class Common:
    def __init__(self):
        self.winning = threading.Event()
        self._negotiation_check_lock = threading.Lock()
        self._have_transit_key = threading.Condition()
        self._transit_key = None
        self._start_server()

    def _start_server(self):
        server = MyTCPServer(("", 0), None)
        _, port = server.server_address
        self.my_direct_hints = ["tcp:%s:%d" % (addr, port)
                                for addr in ipaddrs.find_addresses()]
        server.owner = self
        server_thread = threading.Thread(target=server.serve_forever)
        server_thread.daemon = True
        server_thread.start()
        self.listener = server

    def get_direct_hints(self):
        return self.my_direct_hints
    def get_relay_hints(self):
        return [TRANSIT_RELAY]

    def add_their_direct_hints(self, hints):
        self._their_direct_hints = [force_ascii(h) for h in hints]
    def add_their_relay_hints(self, hints):
        self._their_relay_hints = [force_ascii(h) for h in hints]

    def _send_this(self):
        if self.is_sender:
            return build_sender_handshake(self._transit_key)
        else:
            return build_receiver_handshake(self._transit_key)

    def _expect_this(self):
        if self.is_sender:
            return build_receiver_handshake(self._transit_key)
        else:
            return build_sender_handshake(self._transit_key) + "go\n"

    def set_transit_key(self, key):
        # This _have_transit_key condition/lock protects us against the race
        # where the sender knows the hints and the key, and connects to the
        # receiver's transit socket before the receiver gets relay message
        # (and thus the key).
        self._have_transit_key.acquire()
        self._transit_key = key
        self.handler_send_handshake = self._send_this() # no "go"
        self.handler_expected_handshake = self._expect_this()
        self._have_transit_key.notify_all()
        self._have_transit_key.release()

    def _start_outbound(self):
        self._active_connectors = set(self._their_direct_hints)
        for hint in self._their_direct_hints:
            self._start_connector(hint)
        if not self._their_direct_hints:
            self._start_relay_connectors()

    def _start_connector(self, hint, is_relay=False):
        description = "->%s" % (hint,)
        if is_relay:
            description = "->relay:%s" % (hint,)
        args = (self, hint, description,
                self._send_this(), self._expect_this())
        if is_relay:
            args = args + (build_relay_handshake(self._transit_key),)
        t = threading.Thread(target=connector, args=args)
        t.daemon = True
        t.start()

    def _start_relay_connectors(self):
        for hint in self._their_relay_hints:
            self._start_connector(hint, is_relay=True)

    def establish_connection(self):
        self.winning_skt = None
        self.winning_skt_description = None
        self._start_outbound()

        # we sit here until one of our inbound or outbound sockets succeeds
        flag = self.winning.wait(TIMEOUT)

        if not flag:
            # timeout: self.winning_skt will not be set. ish. race.
            pass
        if self.listener:
            self.listener.shutdown() # TODO: waits up to 0.5s. push to thread
        if self.winning_skt:
            return self.winning_skt
        raise TransitError

    def describe(self):
        if not self.winning_skt_description:
            return "not yet established"
        return self.winning_skt_description

    def _connector_failed(self, hint):
        self._active_connectors.remove(hint)
        if not self._active_connectors:
            self._start_relay_connectors()

    def _negotiation_finished(self, skt, description):
        # inbound/outbound sockets call this when they finish negotiation.
        # The first one wins and gets a "go". Any subsequent ones lose and
        # get a "nevermind" before being closed.

        with self._negotiation_check_lock:
            if self.winning_skt:
                is_winner = False
            else:
                is_winner = True
                self.winning_skt = skt
                self.winning_skt_description = description

        if is_winner:
            if self.is_sender:
                send_to(skt, "go\n")
            self.winning.set()
        else:
            if self.is_sender:
                send_to(skt, "nevermind\n")
            skt.close()

class TransitSender(Common):
    is_sender = True

class TransitReceiver(Common):
    is_sender = False
<|endoftext|>


<commit_before># Send SPS messages to iembot

import sys, StringIO, logging, re, traceback, mx.DateTime
import smtplib
from email.MIMEText import MIMEText

import secret, os
from settings import *
from common import *
from pyIEM.nws import TextProduct
from pyIEM import iemdb, ldmbridge
i = iemdb.iemdb(secret.dbhost)
postgis = i['postgis']

from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor

errors = StringIO.StringIO()
logging.basicConfig(filename='/mesonet/data/logs/%s/sps2bot.log' % (os.getenv("USER"), ), filemode='a')
logger=logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)

ugc_dict = {}
sql = "SELECT name, ugc from nws_ugc WHERE name IS NOT Null"
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
    name = (rs[i]["name"]).replace("\x92"," ")
    ugc_dict[ rs[i]['ugc'] ] = name

def countyText(u):
    countyState = {}
    c = ""
    for k in range(len(u)):
        cnty = u[k]
        stateAB = cnty[:2]
        if (not countyState.has_key(stateAB)):
            countyState[stateAB] = []
        if (not ugc_dict.has_key(cnty)):
            name = "((%s))" % (cnty,)
        else:
            name = ugc_dict[cnty]
        countyState[stateAB].append(name)

    for st in countyState.keys():
       countyState[stateAB].sort()
       c +=" %s [%s] and" %(", ".join(countyState[st]), st)
    return c[:-4]

offsets = {
 'EDT': 4,
 'CDT': 5, 'EST': 5,
 'MDT': 6, 'CST': 6,
 'PDT': 7, 'MST': 7,
 'ADT': 8, 'PST': 8,
 'HDT': 9, 'AST': 9,
           'HST':10,
}

# LDM Ingestor
class myProductIngestor(ldmbridge.LDMProductReceiver):

    def processData(self, buf):
        try:
            real_process(buf)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            logger.error( io.getvalue() )
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),buf.replace("\015\015\012", "\n") ))
            msg['subject'] = 'sps2bot.py Traceback'
            msg['From'] = "ldm@mesonet.agron.iastate.edu"
            msg['To'] = "akrherz@iastate.edu"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()

    def connectionLost(self,reason):
        logger.info("LDM Closed PIPE")

def real_process(raw):
    sqlraw = raw.replace("'", "\\'")
    sqlraw = sqlraw.replace("\015\015\012", "\n")
    prod = TextProduct.TextProduct(raw)

    for seg in prod.segments:
        headline = "[NO HEADLINE FOUND IN SPS]"
        if (len(seg.headlines) > 0):
            headline = (seg.headlines[0]).replace("\n", " ")
        counties = countyText(seg.ugc)
        if (counties.strip() == ""):
            counties = "entire area"
        expire = ""
        if (seg.ugcExpire is not None):
            expire = "till "+ (seg.ugcExpire - mx.DateTime.RelativeDateTime(hours= offsets[prod.z] )).strftime("%-I:%M %p ")+ prod.z

        sql = "INSERT into text_products(product) values ('%s')" % (sqlraw,)
        postgis.query(sql)
        sql = "select last_value from text_products_id_seq"
        rs = postgis.query(sql).dictresult()
        id = rs[0]['last_value']

        mess = "%s: %s issues %s for %s %s http://mesonet.agron.iastate.edu/p.php?id=%s" % (prod.source[1:], \
           prod.source[1:], headline, counties, expire, id)
        htmlmess = "%s issues %s for %s %s" % ( prod.source[1:], id, headline, counties, expire)
        logger.info(mess)

        jabber.sendMessage(mess, htmlmess)

myJid = jid.JID('iembot_ingest@%s/sps2bot_%s' % (secret.chatserver, mx.DateTime.now().ticks() ) )
factory = client.basicClientFactory(myJid, secret.iembot_ingest_password)

jabber = JabberClient(myJid)

factory.addBootstrap('//event/stream/authd',jabber.authd)
factory.addBootstrap("//event/client/basicauth/invaliduser", jabber.debug)
factory.addBootstrap("//event/client/basicauth/authfailed", jabber.debug)
factory.addBootstrap("//event/stream/error", jabber.debug)
factory.addBootstrap(xmlstream.STREAM_END_EVENT, jabber._disconnect )

reactor.connectTCP(secret.connect_chatserver,5222,factory)

ldm = ldmbridge.LDMProductFactory( myProductIngestor() )
reactor.run()
<commit_msg>get rid of settings<commit_after># Send SPS messages to iembot

import sys, StringIO, logging, re, traceback, mx.DateTime
import smtplib
from email.MIMEText import MIMEText

import secret, os
from common import *
from pyIEM.nws import TextProduct
from pyIEM import iemdb, ldmbridge
i = iemdb.iemdb(secret.dbhost)
postgis = i['postgis']

from twisted.words.protocols.jabber import client, jid, xmlstream
from twisted.words.xish import domish
from twisted.internet import reactor

errors = StringIO.StringIO()
logging.basicConfig(filename='/mesonet/data/logs/%s/sps2bot.log' % (os.getenv("USER"), ), filemode='a')
logger=logging.getLogger()
logger.addHandler(logging.StreamHandler())
logger.setLevel(logging.INFO)

ugc_dict = {}
sql = "SELECT name, ugc from nws_ugc WHERE name IS NOT Null"
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
    name = (rs[i]["name"]).replace("\x92"," ")
    ugc_dict[ rs[i]['ugc'] ] = name

def countyText(u):
    countyState = {}
    c = ""
    for k in range(len(u)):
        cnty = u[k]
        stateAB = cnty[:2]
        if (not countyState.has_key(stateAB)):
            countyState[stateAB] = []
        if (not ugc_dict.has_key(cnty)):
            name = "((%s))" % (cnty,)
        else:
            name = ugc_dict[cnty]
        countyState[stateAB].append(name)

    for st in countyState.keys():
       countyState[stateAB].sort()
       c +=" %s [%s] and" %(", ".join(countyState[st]), st)
    return c[:-4]

offsets = {
 'EDT': 4,
 'CDT': 5, 'EST': 5,
 'MDT': 6, 'CST': 6,
 'PDT': 7, 'MST': 7,
 'ADT': 8, 'PST': 8,
 'HDT': 9, 'AST': 9,
           'HST':10,
}

# LDM Ingestor
class myProductIngestor(ldmbridge.LDMProductReceiver):

    def processData(self, buf):
        try:
            real_process(buf)
        except:
            io = StringIO.StringIO()
            traceback.print_exc(file=io)
            logger.error( io.getvalue() )
            msg = MIMEText("%s\n\n>RAW DATA\n\n%s"%(io.getvalue(),buf.replace("\015\015\012", "\n") ))
            msg['subject'] = 'sps2bot.py Traceback'
            msg['From'] = "ldm@mesonet.agron.iastate.edu"
            msg['To'] = "akrherz@iastate.edu"

            s = smtplib.SMTP()
            s.connect()
            s.sendmail(msg["From"], msg["To"], msg.as_string())
            s.close()

    def connectionLost(self,reason):
        logger.info("LDM Closed PIPE")

def real_process(raw):
    sqlraw = raw.replace("'", "\\'")
    sqlraw = sqlraw.replace("\015\015\012", "\n")
    prod = TextProduct.TextProduct(raw)

    for seg in prod.segments:
        headline = "[NO HEADLINE FOUND IN SPS]"
        if (len(seg.headlines) > 0):
            headline = (seg.headlines[0]).replace("\n", " ")
        counties = countyText(seg.ugc)
        if (counties.strip() == ""):
            counties = "entire area"
        expire = ""
        if (seg.ugcExpire is not None):
            expire = "till "+ (seg.ugcExpire - mx.DateTime.RelativeDateTime(hours= offsets[prod.z] )).strftime("%-I:%M %p ")+ prod.z

        sql = "INSERT into text_products(product) values ('%s')" % (sqlraw,)
        postgis.query(sql)
        sql = "select last_value from text_products_id_seq"
        rs = postgis.query(sql).dictresult()
        id = rs[0]['last_value']

        mess = "%s: %s issues %s for %s %s http://mesonet.agron.iastate.edu/p.php?id=%s" % (prod.source[1:], \
           prod.source[1:], headline, counties, expire, id)
        htmlmess = "%s issues %s for %s %s" % ( prod.source[1:], id, headline, counties, expire)
        logger.info(mess)

        jabber.sendMessage(mess, htmlmess)

myJid = jid.JID('iembot_ingest@%s/sps2bot_%s' % (secret.chatserver, mx.DateTime.now().ticks() ) )
factory = client.basicClientFactory(myJid, secret.iembot_ingest_password)

jabber = JabberClient(myJid)

factory.addBootstrap('//event/stream/authd',jabber.authd)
factory.addBootstrap("//event/client/basicauth/invaliduser", jabber.debug)
factory.addBootstrap("//event/client/basicauth/authfailed", jabber.debug)
factory.addBootstrap("//event/stream/error", jabber.debug)
factory.addBootstrap(xmlstream.STREAM_END_EVENT, jabber._disconnect )

reactor.connectTCP(secret.connect_chatserver,5222,factory)

ldm = ldmbridge.LDMProductFactory( myProductIngestor() )
reactor.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
__all__ = ['ixigua_download', 'ixigua_download_playlist']
from .toutiao import download as toutiao_download
from .toutiao import download_playlist as toutiao_download_playlist

def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    return toutiao_download(url.replace('ixigua', '365yg'))

site_info = "ixigua.com"
download = ixigua_download
download_playlist = toutiao_download_playlist<commit_msg>[ixigua] remove undefined name (#2599)<commit_after>#!/usr/bin/env python
__all__ = ['ixigua_download']

from .toutiao import download as toutiao_download
from .toutiao import download_playlist as toutiao_download_playlist

def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    return toutiao_download(url.replace('ixigua', '365yg'))

site_info = "ixigua.com"
download = ixigua_download
download_playlist = toutiao_download_playlist
<|endoftext|>


<commit_before># encoding: utf-8
from __future__ import absolute_import, print_function, unicode_literals
import os
from gozokia.i_o import Io
from gozokia.conf import settings
from gozokia.core import Rule
import time
import multiprocessing

def start_led():
    '''
    if the console is running on a RaspberryPi, run a led
    '''
    try:
        import RPi.GPIO as GPIO
    except ImportError:
        return False
    # Configure the Pi to use the BCM (Broadcom) pin names, rather than the pin positions
    GPIO.setmode(GPIO.BCM)

    red_pin = 18

    GPIO.setup(red_pin, GPIO.OUT)

    try:
        while True:
            GPIO.output(red_pin, True)  # LED on
            time.sleep(0.5)  # delay 0.5 seconds
            GPIO.output(red_pin, False)  # LED off
            time.sleep(0.5)  # delay 0.5 seconds
    finally:
        GPIO.cleanup()

class Gozokia:
    rules_map = Rule()
    '''
    GOZOKIA_DIR: The directory where gozokia have been calling.
    '''
    GOZOKIA_DIR = os.path.dirname(os.path.abspath(__file__))

    '''
    PROJECT_DIR: The directory where the project is running.
    '''
    PROJECT_DIR = os.getcwd()

    def __init__(self):
        print(settings.GOZOKIA_INPUT_TYPE)
        self.set_io(input_type=settings.GOZOKIA_INPUT_TYPE,
                    output_type=settings.GOZOKIA_OUTPUT_TYPE,
                    )

    def set_io(self, *args, **kwargs):
        self.io = Io(*args, **kwargs)

    def rule(self, rule, **options):
        """A decorator that is used to register a view function for a
        given rule.
        """
        def decorator(f):
            self.add_rule(rule, f, **options)
            return f
        return decorator

    def add_rule(self, rule, view_func=None, **options):
        self.rules_map.add({rule: view_func})

    def console(self):
        input_result = True
        p = multiprocessing.Process(target=start_led)
        p.start()
        if settings.DEBUG is True:
            print("***** Activate rules *****")
            for rule in self.rules_map:
                print(rule)
        while input_result is not False:
            input_result = self.io.listen()

            # TODO: Get logic here
            output_result = "you said: {}".format(input_result)

            self.io.response(output_result)
        p.terminate()

<commit_msg>Added debug to init gozokia<commit_after># encoding: utf-8
import os
from gozokia.i_o import Io
from gozokia.conf import settings
from gozokia.core import Rule
import time
import multiprocessing

def start_led():
    '''
    if the console is running on a RaspberryPi, run a led
    '''
    try:
        import RPi.GPIO as GPIO
    except ImportError:
        return False
    # Configure the Pi to use the BCM (Broadcom) pin names, rather than the pin positions
    GPIO.setmode(GPIO.BCM)

    red_pin = 18

    GPIO.setup(red_pin, GPIO.OUT)

    try:
        while True:
            GPIO.output(red_pin, True)  # LED on
            time.sleep(0.5)  # delay 0.5 seconds
            GPIO.output(red_pin, False)  # LED off
            time.sleep(0.5)  # delay 0.5 seconds
    finally:
        GPIO.cleanup()

class Gozokia:
    rules_map = Rule()
    '''
    GOZOKIA_DIR: The directory where gozokia have been calling.
    '''
    GOZOKIA_DIR = os.path.dirname(os.path.abspath(__file__))

    '''
    PROJECT_DIR: The directory where the project is running.
    '''
    PROJECT_DIR = os.getcwd()

    def __init__(self):
        if settings.DEBUG is True:
            print("Input selected: {}".format(settings.GOZOKIA_INPUT_TYPE))
            print("Output selected: {}".format(settings.GOZOKIA_OUTPUT_TYPE))
        self.set_io(input_type=settings.GOZOKIA_INPUT_TYPE,
                    output_type=settings.GOZOKIA_OUTPUT_TYPE,
                    )

    def set_io(self, *args, **kwargs):
        self.io = Io(*args, **kwargs)

    def rule(self, rule, **options):
        """A decorator that is used to register a view function for a
        given rule.
        """
        def decorator(f):
            self.add_rule(rule, f, **options)
            return f
        return decorator

    def add_rule(self, rule, view_func=None, **options):
        self.rules_map.add({rule: view_func})

    def console(self):
        input_result = True
        p = multiprocessing.Process(target=start_led)
        p.start()
        if settings.DEBUG is True:
            print("***** Activate rules *****")
            for rule in self.rules_map:
                print(rule)
        while input_result is not False:
            input_result = self.io.listen()

            # TODO: Get logic here
            output_result = "you said: {}".format(input_result)

            self.io.response(output_result)
        p.terminate()

<|endoftext|>


<commit_before>import IPython
if IPython.release.version < '2.0':
    raise ImportError('this module requires at least v2.0 of IPython')
from rdkit import Chem
from rdkit.Chem import rdchem, rdChemReactions
from rdkit.Chem import Draw
from cStringIO import StringIO
import copy
import json
import numpy
try:
    import Image
except ImportError:
    from PIL import Image

molSize = (450, 150)
highlightSubstructs = True
kekulizeStructures = True

_3d_initialized = False

ipython_3d = False
molSize_3d = (450, 450)
drawing_type_3d = "ball and stick"
camera_type_3d = "perspective"

def _toJSON(mol):
    """For IPython notebook, renders 3D webGL objects."""

    if not ipython_3d or not mol.GetNumConformers():
        return None

    conf = mol.GetConformer()
    if not conf.Is3D():
        return None

    mol = Chem.Mol(mol)
    Chem.Kekulize(mol)
    size = molSize_3d

    # center the molecule:
    atomps = numpy.array([list(conf.GetAtomPosition(x))
                          for x in range(mol.GetNumAtoms())])
    avgP = numpy.average(atomps, 0)
    atomps -= avgP

    lib_path = "/nbextensions/imolecule.min.js"
    # If the javascript lib has not yet been loaded, do so.
    # IPython >=2.0 does this by copying the file into ~/.ipython/nbextensions
    global _3d_initialized
    if not _3d_initialized:
        import os
        from IPython.html.nbextensions import install_nbextension
        filepath = os.path.normpath(os.path.dirname(__file__))
        install_nbextension([os.path.join(filepath, "imolecule.min.js")],
                            verbose=0)
        _3d_initialized = True

    # Convert the relevant parts of the molecule into JSON for rendering
    atoms = [{"element": atom.GetSymbol(),
              "location": list(atomps[atom.GetIdx()])}
             for atom in mol.GetAtoms()]
    bonds = [{"atoms": [bond.GetBeginAtomIdx(),
                        bond.GetEndAtomIdx()],
              "order": int(bond.GetBondTypeAsDouble())}
             for bond in mol.GetBonds()]
    mol = {"atoms": atoms, "bonds": bonds}
    json_mol = json.dumps(mol, separators=(",", ":"))

    return """require(['%s'], function () {
           var $d = $('<div/>').attr('id', 'molecule_' + utils.uuid());
           $d.width(%d); $d.height(%d);
           $d.imolecule = jQuery.extend({}, imolecule);
           $d.imolecule.create($d, {drawingType: '%s', cameraType: '%s'});
           $d.imolecule.draw(%s);
           element.append($d);
           });""" % ((lib_path,) + size + (drawing_type_3d, camera_type_3d,
                     json_mol))

def _toSVG(mol):
    if hasattr(mol, '__sssAtoms'):
        highlightAtoms = mol.__sssAtoms
    else:
        highlightAtoms = []
    try:
        mol.GetAtomWithIdx(0).GetExplicitValence()
    except RuntimeError:
        mol.UpdatePropertyCache(False)

    mc = copy.deepcopy(mol)
    sio = StringIO()
    try:
        Draw.MolToFile(mc, sio, size=molSize, imageType="svg",
                       kekulize=kekulizeStructures,
                       highlightAtoms=highlightAtoms)
    except ValueError:  # <- can happen on a kekulization failure
        mc = copy.deepcopy(mol)
        Draw.MolToFile(mc, sio, size=molSize, kekulize=False,
                       highlightAtoms=highlightAtoms, imageType="svg")

    # It seems that the svg renderer used doesn't quite hit the spec.
    # Here are some fixes to make it work in the notebook, although I think
    # the underlying issue needs to be resolved at the generation step
    svg_split = sio.getvalue().replace("svg:", "").splitlines()
    header = ('<svg version="1.1" xmlns="http://www.w3.org/2000/svg"'
              'width="%dpx" height="%dpx">') % molSize
    svg = "\n".join(svg_split[:1] + [header] + svg_split[5:])
    return svg

def _toReactionPNG(rxn):
    rc = copy.deepcopy(rxn)
    img = Draw.ReactionToImage(rc, subImgSize=(int(molSize[0]/3), molSize[1]))
    sio = StringIO()
    img.save(sio, format='PNG')
    return sio.getvalue()

def _GetSubstructMatch(mol, query, **kwargs):
    res = mol.__GetSubstructMatch(query, **kwargs)
    if highlightSubstructs:
        mol.__sssAtoms = list(res)
    else:
        mol.__sssAtoms = []
    return res

def _GetSubstructMatches(mol, query, **kwargs):
    res = mol.__GetSubstructMatches(query, **kwargs)
    mol.__sssAtoms = []
    if highlightSubstructs:
        for entry in res:
            mol.__sssAtoms.extend(list(entry))
    return res

# code for displaying PIL images directly,
def display_pil_image(img):
    """displayhook function for PIL Images, rendered as PNG"""
    sio = StringIO()
    img.save(sio, format='PNG')
    return sio.getvalue()

def InstallIPythonRenderer():
    rdchem.Mol._repr_svg_ = _toSVG
    rdchem.Mol._repr_javascript_ = _toJSON
    rdChemReactions.ChemicalReaction._repr_png_ = _toReactionPNG
    if not hasattr(rdchem.Mol, '__GetSubstructMatch'):
        rdchem.Mol.__GetSubstructMatch = rdchem.Mol.GetSubstructMatch
    rdchem.Mol.GetSubstructMatch = _GetSubstructMatch
    if not hasattr(rdchem.Mol, '__GetSubstructMatches'):
        rdchem.Mol.__GetSubstructMatches = rdchem.Mol.GetSubstructMatches
    rdchem.Mol.GetSubstructMatches = _GetSubstructMatches
    Image.Image._repr_png_ = display_pil_image
InstallIPythonRenderer()

def UninstallIPythonRenderer():
    del rdchem.Mol._repr_svg_
    del rdChemReactions.ChemicalReaction._repr_png_
    if hasattr(rdchem.Mol, '__GetSubstructMatch'):
        rdchem.Mol.GetSubstructMatch = rdchem.Mol.__GetSubstructMatch
        del rdchem.Mol.__GetSubstructMatch
    if hasattr(rdchem.Mol, '__GetSubstructMatches'):
        rdchem.Mol.GetSubstructMatches = rdchem.Mol.__GetSubstructMatches
        del rdchem.Mol.__GetSubstructMatches
    del Image.Image._repr_png_
<commit_msg>default to png rendering<commit_after>import IPython
if IPython.release.version < '2.0':
    raise ImportError('this module requires at least v2.0 of IPython')
from rdkit import Chem
from rdkit.Chem import rdchem, rdChemReactions
from rdkit.Chem import Draw
from cStringIO import StringIO
import copy
import json
import numpy
try:
    import Image
except ImportError:
    from PIL import Image

molSize = (450, 150)
highlightSubstructs = True
kekulizeStructures = True

_3d_initialized = False

ipython_useSVG = False
ipython_3d = False
molSize_3d = (450, 450)
drawing_type_3d = "ball and stick"
camera_type_3d = "perspective"

def _toJSON(mol):
    """For IPython notebook, renders 3D webGL objects."""

    if not ipython_3d or not mol.GetNumConformers():
        return None

    conf = mol.GetConformer()
    if not conf.Is3D():
        return None

    mol = Chem.Mol(mol)
    Chem.Kekulize(mol)
    size = molSize_3d

    # center the molecule:
    atomps = numpy.array([list(conf.GetAtomPosition(x))
                          for x in range(mol.GetNumAtoms())])
    avgP = numpy.average(atomps, 0)
    atomps -= avgP

    lib_path = "/nbextensions/imolecule.min.js"
    # If the javascript lib has not yet been loaded, do so.
    # IPython >=2.0 does this by copying the file into ~/.ipython/nbextensions
    global _3d_initialized
    if not _3d_initialized:
        import os
        from IPython.html.nbextensions import install_nbextension
        filepath = os.path.normpath(os.path.dirname(__file__))
        install_nbextension([os.path.join(filepath, "imolecule.min.js")],
                            verbose=0)
        _3d_initialized = True

    # Convert the relevant parts of the molecule into JSON for rendering
    atoms = [{"element": atom.GetSymbol(),
              "location": list(atomps[atom.GetIdx()])}
             for atom in mol.GetAtoms()]
    bonds = [{"atoms": [bond.GetBeginAtomIdx(),
                        bond.GetEndAtomIdx()],
              "order": int(bond.GetBondTypeAsDouble())}
             for bond in mol.GetBonds()]
    mol = {"atoms": atoms, "bonds": bonds}
    json_mol = json.dumps(mol, separators=(",", ":"))

    return """require(['%s'], function () {
           var $d = $('<div/>').attr('id', 'molecule_' + utils.uuid());
           $d.width(%d); $d.height(%d);
           $d.imolecule = jQuery.extend({}, imolecule);
           $d.imolecule.create($d, {drawingType: '%s', cameraType: '%s'});
           $d.imolecule.draw(%s);
           element.append($d);
           });""" % ((lib_path,) + size + (drawing_type_3d, camera_type_3d,
                     json_mol))

def _toPNG(mol):
    if hasattr(mol,'__sssAtoms'):
        highlightAtoms=mol.__sssAtoms
    else:
        highlightAtoms=[]
    try:
        mol.GetAtomWithIdx(0).GetExplicitValence()
    except RuntimeError:
        mol.UpdatePropertyCache(False)

    mc = copy.deepcopy(mol)
    try:
        img = Draw.MolToImage(mc,size=molSize,kekulize=kekulizeStructures,
                            highlightAtoms=highlightAtoms)
    except ValueError:  # <- can happen on a kekulization failure
        mc = copy.deepcopy(mol)
        img = Draw.MolToImage(mc,size=molSize,kekulize=False,
                            highlightAtoms=highlightAtoms)

    sio = StringIO()
    img.save(sio,format='PNG')
    return sio.getvalue()

def _toSVG(mol):
    if not ipython_useSVG:
        return None
    if hasattr(mol, '__sssAtoms'):
        highlightAtoms = mol.__sssAtoms
    else:
        highlightAtoms = []
    try:
        mol.GetAtomWithIdx(0).GetExplicitValence()
    except RuntimeError:
        mol.UpdatePropertyCache(False)

    mc = copy.deepcopy(mol)
    sio = StringIO()
    try:
        Draw.MolToFile(mc, sio, size=molSize, imageType="svg",
                       kekulize=kekulizeStructures,
                       highlightAtoms=highlightAtoms)
    except ValueError:  # <- can happen on a kekulization failure
        mc = copy.deepcopy(mol)
        Draw.MolToFile(mc, sio, size=molSize, kekulize=False,
                       highlightAtoms=highlightAtoms, imageType="svg")

    # It seems that the svg renderer used doesn't quite hit the spec.
    # Here are some fixes to make it work in the notebook, although I think
    # the underlying issue needs to be resolved at the generation step
    svg_split = sio.getvalue().replace("svg:", "").splitlines()
    header = ('<svg version="1.1" xmlns="http://www.w3.org/2000/svg"'
              'width="%dpx" height="%dpx">') % molSize
    svg = "\n".join(svg_split[:1] + [header] + svg_split[5:])
    return svg

def _toReactionPNG(rxn):
    rc = copy.deepcopy(rxn)
    img = Draw.ReactionToImage(rc, subImgSize=(int(molSize[0]/3), molSize[1]))
    sio = StringIO()
    img.save(sio, format='PNG')
    return sio.getvalue()

def _GetSubstructMatch(mol, query, **kwargs):
    res = mol.__GetSubstructMatch(query, **kwargs)
    if highlightSubstructs:
        mol.__sssAtoms = list(res)
    else:
        mol.__sssAtoms = []
    return res

def _GetSubstructMatches(mol, query, **kwargs):
    res = mol.__GetSubstructMatches(query, **kwargs)
    mol.__sssAtoms = []
    if highlightSubstructs:
        for entry in res:
            mol.__sssAtoms.extend(list(entry))
    return res

# code for displaying PIL images directly,
def display_pil_image(img):
    """displayhook function for PIL Images, rendered as PNG"""
    sio = StringIO()
    img.save(sio, format='PNG')
    return sio.getvalue()

def InstallIPythonRenderer():
    rdchem.Mol._repr_svg_ = _toSVG
    rdchem.Mol._repr_png_ = _toPNG
    rdchem.Mol._repr_javascript_ = _toJSON
    rdChemReactions.ChemicalReaction._repr_png_ = _toReactionPNG
    if not hasattr(rdchem.Mol, '__GetSubstructMatch'):
        rdchem.Mol.__GetSubstructMatch = rdchem.Mol.GetSubstructMatch
    rdchem.Mol.GetSubstructMatch = _GetSubstructMatch
    if not hasattr(rdchem.Mol, '__GetSubstructMatches'):
        rdchem.Mol.__GetSubstructMatches = rdchem.Mol.GetSubstructMatches
    rdchem.Mol.GetSubstructMatches = _GetSubstructMatches
    Image.Image._repr_png_ = display_pil_image
InstallIPythonRenderer()

def UninstallIPythonRenderer():
    del rdchem.Mol._repr_svg_
    del rdChemReactions.ChemicalReaction._repr_png_
    if hasattr(rdchem.Mol, '__GetSubstructMatch'):
        rdchem.Mol.GetSubstructMatch = rdchem.Mol.__GetSubstructMatch
        del rdchem.Mol.__GetSubstructMatch
    if hasattr(rdchem.Mol, '__GetSubstructMatches'):
        rdchem.Mol.GetSubstructMatches = rdchem.Mol.__GetSubstructMatches
        del rdchem.Mol.__GetSubstructMatches
    del Image.Image._repr_png_
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['tumblr_download']

from ..common import *
from .universal import *
from .dailymotion import dailymotion_download
from .vimeo import vimeo_download
from .vine import vine_download

def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
        universal_download(url, output_dir, merge=merge, info_only=info_only)
        return

    html = parse.unquote(get_html(url)).replace('\/', '/')
    feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)

    if feed in ['photo', 'photoset', 'entry'] or feed is None:
        # try to extract photos
        page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \
                     r1(r'<meta property="og:description" content="([^"\n]+)', html) or \
                     r1(r'<title>([^<\n]*)', html)
        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.jpg)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.png)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^";]+_\d+\.gif)', html)

        tuggles = {}
        for url in urls:
            filename = parse.unquote(url.split('/')[-1])
            title = '.'.join(filename.split('.')[:-1])
            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
            quality = int(r1(r'^tumblr_.+_(\d+)$', title))
            ext = filename.split('.')[-1]
            size = int(get_head(url)['Content-Length'])
            if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
                tuggles[tumblr_id] = {
                    'title': title,
                    'url': url,
                    'quality': quality,
                    'ext': ext,
                    'size': size,
                }

        if tuggles:
            size = sum([tuggles[t]['size'] for t in tuggles])
            print_info(site_info, page_title, None, size)

            if not info_only:
                for t in tuggles:
                    title = tuggles[t]['title']
                    ext = tuggles[t]['ext']
                    size = tuggles[t]['size']
                    url = tuggles[t]['url']
                    print_info(site_info, title, ext, size)
                    download_urls([url], title, ext, size,
                                  output_dir=output_dir)
            return

    # feed == 'audio' or feed == 'video' or feed is None
    # try to extract video / audio
    real_url = r1(r'source src=\\x22([^\\]+)\\', html)
    if not real_url:
        real_url = r1(r'audio_file=([^&]+)&', html)
        if real_url:
            real_url = real_url + '?plead=please-dont-download-this-or-our-lawyers-wont-let-us-host-audio'
    if not real_url:
        real_url = r1(r'<source src="([^"]*)"', html)
    if not real_url:
        iframe_url = r1(r'<iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
        if iframe_url[:2] == '//': iframe_url = 'http:' + iframe_url
        if re.search(r'player\.vimeo\.com', iframe_url):
            vimeo_download(iframe_url, output_dir, merge=merge, info_only=info_only,
                           referer='http://tumblr.com/', **kwargs)
            return
        elif re.search(r'dailymotion\.com', iframe_url):
            dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
            return
        elif re.search(r'vine\.co', iframe_url):
            vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
            return
        else:
            iframe_html = get_content(iframe_url)
            real_url = r1(r'<source src="([^"]*)"', iframe_html)

    title = unescape_html(r1(r'<meta property="og:title" content="([^"]*)" />', html) or
        r1(r'<meta property="og:description" content="([^"]*)" />', html) or
        r1(r'<title>([^<\n]*)', html) or url.split("/")[4]).replace('\n', '')

    type, ext, size = url_info(real_url)

    print_info(site_info, title, type, size)
    if not info_only:
        download_urls([real_url], title, ext, size, output_dir, merge = merge)

site_info = "Tumblr.com"
download = tumblr_download
download_playlist = playlist_not_supported('tumblr')
<commit_msg>Support for tumblr-hosted videos.<commit_after>#!/usr/bin/env python

__all__ = ['tumblr_download']

from ..common import *
from .universal import *
from .dailymotion import dailymotion_download
from .vimeo import vimeo_download
from .vine import vine_download

def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
    if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
        universal_download(url, output_dir, merge=merge, info_only=info_only)
        return

    html = parse.unquote(get_html(url)).replace('\/', '/')
    feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)

    if feed in ['photo', 'photoset', 'entry'] or feed is None:
        # try to extract photos
        page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \
                     r1(r'<meta property="og:description" content="([^"\n]+)', html) or \
                     r1(r'<title>([^<\n]*)', html)
        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.jpg)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.png)', html) +\
               re.findall(r'(https?://[^;"&]+/tumblr_[^";]+_\d+\.gif)', html)

        tuggles = {}
        for url in urls:
            filename = parse.unquote(url.split('/')[-1])
            title = '.'.join(filename.split('.')[:-1])
            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
            quality = int(r1(r'^tumblr_.+_(\d+)$', title))
            ext = filename.split('.')[-1]
            size = int(get_head(url)['Content-Length'])
            if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
                tuggles[tumblr_id] = {
                    'title': title,
                    'url': url,
                    'quality': quality,
                    'ext': ext,
                    'size': size,
                }

        if tuggles:
            size = sum([tuggles[t]['size'] for t in tuggles])
            print_info(site_info, page_title, None, size)

            if not info_only:
                for t in tuggles:
                    title = tuggles[t]['title']
                    ext = tuggles[t]['ext']
                    size = tuggles[t]['size']
                    url = tuggles[t]['url']
                    print_info(site_info, title, ext, size)
                    download_urls([url], title, ext, size,
                                  output_dir=output_dir)
            return

    # feed == 'audio' or feed == 'video' or feed is None
    # try to extract video / audio
    real_url = r1(r'source src=\\x22([^\\]+)\\', html)
    if not real_url:
        real_url = r1(r'audio_file=([^&]+)&', html)
        if real_url:
            real_url = real_url + '?plead=please-dont-download-this-or-our-lawyers-wont-let-us-host-audio'
    if not real_url:
        real_url = r1(r'<source src="([^"]*)"', html)
    if not real_url:
        iframe_url = r1(r'<[^>]+tumblr_video_container[^>]+><iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
        if len(iframe_url) > 0:
            iframe_html = get_content(iframe_url, headers=fake_headers)
            real_url = r1(r'<video[^>]*>[\n ]*<source[^>]+src=[\'"]([^\'"]*)[\'"]', iframe_html)
        else:
            iframe_url = r1(r'<iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
            if iframe_url[:2] == '//': iframe_url = 'http:' + iframe_url
            if re.search(r'player\.vimeo\.com', iframe_url):
                vimeo_download(iframe_url, output_dir, merge=merge, info_only=info_only,
                               referer='http://tumblr.com/', **kwargs)
                return
            elif re.search(r'dailymotion\.com', iframe_url):
                dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
                return
            elif re.search(r'vine\.co', iframe_url):
                vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
                return
            else:
                iframe_html = get_content(iframe_url)
                real_url = r1(r'<source src="([^"]*)"', iframe_html)

    title = unescape_html(r1(r'<meta property="og:title" content="([^"]*)" />', html) or
        r1(r'<meta property="og:description" content="([^"]*)" />', html) or
        r1(r'<title>([^<\n]*)', html) or url.split("/")[4]).replace('\n', '')

    type, ext, size = url_info(real_url)

    print_info(site_info, title, type, size)
    if not info_only:
        download_urls([real_url], title, ext, size, output_dir, merge = merge)

site_info = "Tumblr.com"
download = tumblr_download
download_playlist = playlist_not_supported('tumblr')
<|endoftext|>


<commit_before>#/bin/env python

import urllib, json, datetime
from xml.parsers import expat

class c4s :

  def __init__(self):
    self.cvmfsBaseVersionFile = 'cvmfsVersion.txt'
    self.requestedVersion = ''
    self.myVO = 'LHCb'
    self.cvmfsColumnNo = 202
    self.wlcgTopoColumnNo = 144
    self.topoDict = {'WLCG':{}, self.myVO:{}}
    self.ssbTimePat = '%Y-%m-%dT%H:%M:%S'
    self.topologyURL = 'http://lhcb-web-dirac.cern.ch/topology/lhcb_topology.xml'
    self.wlcgBaseUrl = 'http://wlcg-mon.cern.ch/dashboard/request.py/'
    self.wlcgGetUrl = self.wlcgBaseUrl+'getplotdata?columnid=%d&time=24&sites=all&batch=1'
    self.ssbMetrics = ['CvmfsVersion','CvmfsRepoRevision','CvmfsMountPoint','CvmfsCondDBMountPoint', 'CvmfsProbeTime', 'CvmfsStratumOnes', 'CvmfsNumSquids', 'CvmfsProbeNoInfo']
    self.ssbData = {}
    for k in self.ssbMetrics : self.ssbData[k] = {}

  ### start probe functions ###
  ### eval functions ###

  def evalCvmfsProbeNoInfo(self, val, site) :
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if self.ssbData['CvmfsVersion'][site] == 'not installed' : return ('n/a (not installed)', 'grey')
    we = val.split(':')[0]
    if we == 'WARNING' : return (val, 'orange')
    if we == 'ERROR' : return (val, 'red')
    return (val, 'green')

  def evalCvmfsVersion(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if val == 'nfs' : return (val, 'green')
    if val in ('n/a', 'not installed') : return (val, 'red')
    x = 2
    maxDiff = range(x+1)
    deplV = map(lambda x: int(x), val.split('.'))
    reqV = map(lambda x: int(x), self.requestedVersion.split('.'))
    if deplV[1] == reqV[1] and deplV[0] == reqV[0] :
      if (reqV[2] - deplV[2]) in maxDiff : return (val, 'green')
      else : return (val, 'orange')
    else : return (val, 'red')

  def evalCvmfsRepoRevision(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    return (val, 'green')

  def evalCvmfsMountPoint(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('not installed') : return ('n/a (%s)'%vers, 'grey')
    if val and val == '/cvmfs/lhcb.cern.ch' : return (val, 'green')
    else : return (val, 'orange')

  def evalCvmfsCondDBMountPoint(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if  self.ssbData['CvmfsVersion'][site] == 'not installed' : return ('n/a (not installed)', 'grey')
    if val == 'yes' : return (val, 'orange')
    else : return (val, 'green')

  def evalCvmfsProbeTime(self, val, site):
    if val == 'no probe' : return (val, 'red')
    pTime = datetime.datetime.strptime(val,self.ssbTimePat)
    curTime = datetime.datetime.now()
    delta = (curTime - pTime).seconds
    if delta < 3600 : return (val, 'green')
    elif delta < 10800 : return (val, 'orange')
    else : return (val, 'red')

  def evalCvmfsStratumOnes(self, val, site) :
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    if val : return (val, 'green')
    else: return ('none', 'red')

  def evalCvmfsNumSquids(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    if val :
      if int(val) > 1 : return (val, 'green')
      else : return (val, 'orange')
    else: return (val , 'red')

  ### retrieval functions ###

  def getValCvmfsProbeNoInfo(self, site, probe, metric):
    val = 'none'
    pat = 'INFO: Mandatory tests exectuted successfully, now continuing with testing optional repositories'
    for line in probe :
      we = line.split(':')[0]
      if line[:len(pat)] == pat : break
      elif we == 'WARNING' and val.split(':')[0] != 'ERROR' : val = line
      elif we == 'ERROR' : val = line
    self.ssbData['CvmfsProbeNoInfo'][site] = val

  def getValCvmfsVersion(self, site, probe, metric):
    pat1 = 'INFO: CVMFS version installed '
    pat2 = 'INFO: Mandatory mount point /cvmfs/lhcb.cern.ch is nfs mount point'
    pat3 = 'INFO: No cvmfs rpms found on WN, checking if this WN uses nfs mounting of CVMFS repositories'
    ver = 'n/a'
    noCVMFS = False
    cvmfsViaNFS = False
    for line in probe :
      if line[:len(pat1)] == pat1 :
        ver = line[len(pat1):]
      elif line[:len(pat2)] == pat2 :
        ver = 'nfs'
        cvmfsViaNFS = True
      elif line[:len(pat3)] == pat3 :
        noCVMFS = True
    if noCVMFS and not cvmfsViaNFS : ver = 'not installed'
    self.ssbData['CvmfsVersion'][site] = ver

  def getValCvmfsRepoRevision(self, site, probe, metric):
    pat = 'INFO: repository revision '
    rev = 'n/a'
    for line in probe :
      if line[:len(pat)] == pat :
        rev = line[len(pat):]
        break
    self.ssbData['CvmfsRepoRevision'][site] = rev

  def getValCvmfsMountPoint(self, site, probe, metric):
    pat1 = 'INFO: Variable VO_LHCB_SW_DIR points to CVMFS mount point '
    pat2 = 'INFO: Mandatory mount point /cvmfs/lhcb.cern.ch is nfs mount point'
    mp = 'n/a'
    for line in probe :
      if line[:len(pat1)] == pat1 :
        mp = line[len(pat1):]
      elif line[:len(pat2)] == pat2 :
        mp = '/cvmfs/lhcb.cern.ch'
    self.ssbData['CvmfsMountPoint'][site] = mp

  def getValCvmfsCondDBMountPoint(self, site, probe, metric):
    pat = 'INFO: repository /cvmfs/lhcb-conddb.cern.ch available'
    cm = 'no'
    for line in probe :
      if line[:len(pat)] == pat :
        cm = 'yes'
    self.ssbData['CvmfsCondDBMountPoint'][site] = cm

  def getValCvmfsProbeTime(self, site, probe, metric):
    self.ssbData['CvmfsProbeTime'][site] = metric['EndTime']

  def getValCvmfsStratumOnes(self, site, probe, metric) :
    strats = []
    pat = 'INFO: Servers: '
    for line in probe :
      if line[:len(pat)] == pat :
        stratumL = line[len(pat):]
        for serv in stratumL.split() :
          strats.append('.'.join(serv.split('/')[2].split(':')[0].split('.')[-2:]))
        break
    self.ssbData['CvmfsStratumOnes'][site] = ' '.join(strats)

  def getValCvmfsNumSquids(self, site, probe, metric) :
    numSq = 0
    pat = 'INFO: Proxies: '
    for line in probe :
      if line[:len(pat)] == pat :
        numSq = len(line[len(pat):].split())
        break
    self.ssbData['CvmfsNumSquids'][site] = numSq

  ### end probe functions ####

  def xmlStartElement(self, name, attrs):
    if name == 'atp_site' : self.currWLCGSite = attrs['name']
    if name == 'group' and attrs['type'] == 'LHCb_Site' :
      self.topoDict['WLCG'][attrs['name']] = self.currWLCGSite

  def bootstrap(self):
    # get WLCG Mon mapping VO site name <-> site ID
    topo = json.loads(urllib.urlopen(self.wlcgGetUrl%self.wlcgTopoColumnNo).read())
    for ent in topo['csvdata'] : self.topoDict[self.myVO][ent['SiteId']] = ent['Status']
    # read CVMFS base line version number
    f = open(self.cvmfsBaseVersionFile, 'r')
    self.requestedVersion = f.read()
    f.close()
    # read topology file and create mapping VO site name <-> WLCG site name
    topo = urllib.urlopen(self.topologyURL).read()
    p = expat.ParserCreate()
    p.StartElementHandler = self.xmlStartElement
    p.Parse(topo)

  def clearSsbData(self, site):
    for metric in self.ssbMetrics :
      self.ssbData[metric][site] = ''

  def collectInfo(self):
    info = json.loads(urllib.urlopen(self.wlcgGetUrl%self.cvmfsColumnNo).read())
    for metricInf in info['csvdata'] :
      site = self.topoDict[self.myVO][metricInf['SiteId']]
      tTime = datetime.datetime.strptime(metricInf['Time'], self.ssbTimePat)
      dTime = self.ssbData['CvmfsProbeTime'].get(site)
      if ( not dTime ) or ( datetime.datetime.strptime(dTime, self.ssbTimePat) < tTime ) :
        if dTime : self.clearSsbData(site)
        tl = urllib.urlopen(self.wlcgBaseUrl+metricInf['URL']).read().split('\n')
        for metr in self.ssbMetrics : eval('self.getVal'+metr)(site, tl, metricInf)
    for site in self.topoDict['WLCG'].keys() :
      if not self.ssbData['CvmfsProbeTime'].get(site) :
        for metric in self.ssbMetrics : self.ssbData[metric][site] = ''
        self.ssbData['CvmfsProbeTime'][site] = 'no probe'

  def writeSSBColumns(self):
    for k in self.ssbMetrics :
      fun = 'self.eval'+k
      colData = self.ssbData[k]
      f = open(k+'.ssb.txt', 'w')
      for site in colData.keys() :
        now = str(datetime.datetime.now())
        (val, color) = eval(fun)(colData[site], site)
        url = 'http://localhost'
        f.write('%s\t%s\t%s\t%s\t%s\n' % (now, site, val, color, url))
      f.close()

  def createWLCGLHCbMapping(self):
    f = open('WLCGSiteMapping.ssb.txt','w')
    for site in self.topoDict['WLCG'].keys() :
      now = str(datetime.datetime.now())
      val = self.topoDict['WLCG'][site]
      color = 'white'
      url = 'http://localhost'
      f.write('%s\t%s\t%s\t%s\t%s\n' % (now, site, val, color, url))

  def run(self):
    self.bootstrap()
    self.collectInfo()
    self.writeSSBColumns()
    self.createWLCGLHCbMapping()

if __name__ == '__main__' :
  c4s().run()
<commit_msg>- fix links of table cells<commit_after>#/bin/env python

import urllib, json, datetime
from xml.parsers import expat

class c4s :

  def __init__(self):
    self.cvmfsBaseVersionFile = 'cvmfsVersion.txt'
    self.requestedVersion = ''
    self.myVO = 'LHCb'
    self.cvmfsColumnNo = 202
    self.wlcgTopoColumnNo = 144
    self.topoDict = {'WLCG':{}, self.myVO:{}}
    self.ssbTimePat = '%Y-%m-%dT%H:%M:%S'
    self.dontpanic = 'http://en.wikipedia.org/wiki/Don%27t_Panic_%28The_Hitchhiker%27s_Guide_to_the_Galaxy%29#Don.27t_Panic'
    self.topologyURL = 'http://lhcb-web-dirac.cern.ch/topology/lhcb_topology.xml'
    self.wlcgBaseUrl = 'http://wlcg-mon.cern.ch/dashboard/request.py/'
    self.wlcgGetUrl = self.wlcgBaseUrl+'getplotdata?columnid=%d&time=24&sites=all&batch=1'
    self.wlcgSiteBaseLink = 'http://lhcb-web-dirac.cern.ch/DIRAC/LHCb-Production/undefined/grid/SiteStatus/display?name='
    self.ssbMetrics = ['CvmfsVersion','CvmfsRepoRevision','CvmfsMountPoint','CvmfsCondDBMountPoint', 'CvmfsProbeTime', 'CvmfsStratumOnes', 'CvmfsNumSquids', 'CvmfsProbeNoInfo', 'CvmfsProbeLink']
    self.ssbData = {}
    for k in self.ssbMetrics : self.ssbData[k] = {}

  ### start probe functions ###
  ### eval functions ###

  def evalCvmfsProbeLink(self, val, site):
    return (val, 'green')

  def evalCvmfsProbeNoInfo(self, val, site) :
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if self.ssbData['CvmfsVersion'][site] == 'not installed' : return ('n/a (not installed)', 'grey')
    we = val.split(':')[0]
    if we == 'WARNING' : return (val, 'orange')
    if we == 'ERROR' : return (val, 'red')
    return (val, 'green')

  def evalCvmfsVersion(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if val == 'nfs' : return (val, 'green')
    if val in ('n/a', 'not installed') : return (val, 'red')
    x = 2
    maxDiff = range(x+1)
    deplV = map(lambda x: int(x), val.split('.'))
    reqV = map(lambda x: int(x), self.requestedVersion.split('.'))
    if deplV[1] == reqV[1] and deplV[0] == reqV[0] :
      if (reqV[2] - deplV[2]) in maxDiff : return (val, 'green')
      else : return (val, 'orange')
    else : return (val, 'red')

  def evalCvmfsRepoRevision(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    return (val, 'green')

  def evalCvmfsMountPoint(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('not installed') : return ('n/a (%s)'%vers, 'grey')
    if val and val == '/cvmfs/lhcb.cern.ch' : return (val, 'green')
    else : return (val, 'orange')

  def evalCvmfsCondDBMountPoint(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    if  self.ssbData['CvmfsVersion'][site] == 'not installed' : return ('n/a (not installed)', 'grey')
    if val == 'yes' : return (val, 'orange')
    else : return (val, 'green')

  def evalCvmfsProbeTime(self, val, site):
    if val == 'no probe' : return (val, 'red')
    pTime = datetime.datetime.strptime(val,self.ssbTimePat)
    curTime = datetime.datetime.now()
    delta = (curTime - pTime).seconds
    if delta < 3600 : return (val, 'green')
    elif delta < 10800 : return (val, 'orange')
    else : return (val, 'red')

  def evalCvmfsStratumOnes(self, val, site) :
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    if val : return (val, 'green')
    else: return ('none', 'red')

  def evalCvmfsNumSquids(self, val, site):
    if self.ssbData['CvmfsProbeTime'][site] == 'no probe' : return ('n/a (no probe)', 'grey')
    vers = self.ssbData['CvmfsVersion'][site]
    if  vers in ('nfs', 'not installed') : return ('n/a (%s)'%vers, 'grey')
    if val :
      if int(val) > 1 : return (val, 'green')
      else : return (val, 'orange')
    else: return (val , 'red')

  ### retrieval functions ###

  def getValCvmfsProbeLink(self, site, probe, metric):
    self.ssbData['CvmfsProbeLink'][site]=metric['URL']

  def getValCvmfsProbeNoInfo(self, site, probe, metric):
    val = 'none'
    pat = 'INFO: Mandatory tests exectuted successfully, now continuing with testing optional repositories'
    for line in probe :
      we = line.split(':')[0]
      if line[:len(pat)] == pat : break
      elif we == 'WARNING' and val.split(':')[0] != 'ERROR' : val = line
      elif we == 'ERROR' : val = line
    self.ssbData['CvmfsProbeNoInfo'][site] = val

  def getValCvmfsVersion(self, site, probe, metric):
    pat1 = 'INFO: CVMFS version installed '
    pat2 = 'INFO: Mandatory mount point /cvmfs/lhcb.cern.ch is nfs mount point'
    pat3 = 'INFO: No cvmfs rpms found on WN, checking if this WN uses nfs mounting of CVMFS repositories'
    ver = 'n/a'
    noCVMFS = False
    cvmfsViaNFS = False
    for line in probe :
      if line[:len(pat1)] == pat1 :
        ver = line[len(pat1):]
      elif line[:len(pat2)] == pat2 :
        ver = 'nfs'
        cvmfsViaNFS = True
      elif line[:len(pat3)] == pat3 :
        noCVMFS = True
    if noCVMFS and not cvmfsViaNFS : ver = 'not installed'
    self.ssbData['CvmfsVersion'][site] = ver

  def getValCvmfsRepoRevision(self, site, probe, metric):
    pat = 'INFO: repository revision '
    rev = 'n/a'
    for line in probe :
      if line[:len(pat)] == pat :
        rev = line[len(pat):]
        break
    self.ssbData['CvmfsRepoRevision'][site] = rev

  def getValCvmfsMountPoint(self, site, probe, metric):
    pat1 = 'INFO: Variable VO_LHCB_SW_DIR points to CVMFS mount point '
    pat2 = 'INFO: Mandatory mount point /cvmfs/lhcb.cern.ch is nfs mount point'
    mp = 'n/a'
    for line in probe :
      if line[:len(pat1)] == pat1 :
        mp = line[len(pat1):]
      elif line[:len(pat2)] == pat2 :
        mp = '/cvmfs/lhcb.cern.ch'
    self.ssbData['CvmfsMountPoint'][site] = mp

  def getValCvmfsCondDBMountPoint(self, site, probe, metric):
    pat = 'INFO: repository /cvmfs/lhcb-conddb.cern.ch available'
    cm = 'no'
    for line in probe :
      if line[:len(pat)] == pat :
        cm = 'yes'
    self.ssbData['CvmfsCondDBMountPoint'][site] = cm

  def getValCvmfsProbeTime(self, site, probe, metric):
    self.ssbData['CvmfsProbeTime'][site] = metric['URL'].split('&')[1].split('=')[1][:-1]
#    self.ssbData['CvmfsProbeTime'][site] = metric['EndTime']

  def getValCvmfsStratumOnes(self, site, probe, metric) :
    strats = []
    pat = 'INFO: Servers: '
    for line in probe :
      if line[:len(pat)] == pat :
        stratumL = line[len(pat):]
        for serv in stratumL.split() :
          strats.append('.'.join(serv.split('/')[2].split(':')[0].split('.')[-2:]))
        break
    self.ssbData['CvmfsStratumOnes'][site] = ' '.join(strats)

  def getValCvmfsNumSquids(self, site, probe, metric) :
    numSq = 0
    pat = 'INFO: Proxies: '
    for line in probe :
      if line[:len(pat)] == pat :
        numSq = len(line[len(pat):].split())
        break
    self.ssbData['CvmfsNumSquids'][site] = numSq

  ### end probe functions ####

  def xmlStartElement(self, name, attrs):
    if name == 'atp_site' : self.currWLCGSite = attrs['name']
    if name == 'group' and attrs['type'] == 'LHCb_Site' :
      self.topoDict['WLCG'][attrs['name']] = self.currWLCGSite

  def bootstrap(self):
    # get WLCG Mon mapping VO site name <-> site ID
    topo = json.loads(urllib.urlopen(self.wlcgGetUrl%self.wlcgTopoColumnNo).read())
    for ent in topo['csvdata'] : self.topoDict[self.myVO][ent['SiteId']] = ent['Status']
    # read CVMFS base line version number
    f = open(self.cvmfsBaseVersionFile, 'r')
    self.requestedVersion = f.read()
    f.close()
    # read topology file and create mapping VO site name <-> WLCG site name
    topo = urllib.urlopen(self.topologyURL).read()
    p = expat.ParserCreate()
    p.StartElementHandler = self.xmlStartElement
    p.Parse(topo)

  def clearSsbData(self, site):
    for metric in self.ssbMetrics :
      self.ssbData[metric][site] = ''

  def collectInfo(self):
    info = json.loads(urllib.urlopen(self.wlcgGetUrl%self.cvmfsColumnNo).read())
    for metricInf in info['csvdata'] :
      site = self.topoDict[self.myVO][metricInf['SiteId']]
      tTime = datetime.datetime.strptime(metricInf['Time'], self.ssbTimePat)
      dTime = self.ssbData['CvmfsProbeTime'].get(site)
      if ( not dTime ) or ( datetime.datetime.strptime(dTime, self.ssbTimePat) < tTime ) :
        if dTime : self.clearSsbData(site)
        tl = urllib.urlopen(self.wlcgBaseUrl+metricInf['URL']).read().split('\n')
        for metr in self.ssbMetrics : eval('self.getVal'+metr)(site, tl, metricInf)
    for site in self.topoDict['WLCG'].keys() :
      if not self.ssbData['CvmfsProbeTime'].get(site) :
        for metric in self.ssbMetrics : self.ssbData[metric][site] = ''
        self.ssbData['CvmfsProbeTime'][site] = 'no probe'

  def writeSSBColumns(self):
    for k in self.ssbMetrics :
      fun = 'self.eval'+k
      colData = self.ssbData[k]
      f = open(k+'.ssb.txt', 'w')
      for site in colData.keys() :
        now = str(datetime.datetime.now())
        (val, color) = eval(fun)(colData[site], site)
        url = self.dontpanic
        if self.ssbData['CvmfsProbeLink'].get(site): url = self.wlcgBaseUrl+self.ssbData['CvmfsProbeLink'][site]
        f.write('%s\t%s\t%s\t%s\t%s\n' % (now, site, val, color, url))
      f.close()

  def createWLCGLHCbMapping(self):
    f = open('WLCGSiteMapping.ssb.txt','w')
    for site in self.topoDict['WLCG'].keys() :
      now = str(datetime.datetime.now())
      val = self.topoDict['WLCG'][site]
      color = 'white'
      url = self.wlcgSiteBaseLink+site
      f.write('%s\t%s\t%s\t%s\t%s\n' % (now, site, val, color, url))

  def run(self):
    self.bootstrap()
    self.collectInfo()
    self.writeSSBColumns()
    self.createWLCGLHCbMapping()

if __name__ == '__main__' :
  c4s().run()
<|endoftext|>