<commit_before>#!/usr/bin/env python
import logging
import os
import time
import string
import subprocess
import sys
import re
from collections import OrderedDict

import requests

logger = logging.getLogger(__name__)

# Config ENV
PORT = os.getenv("PORT", "80")
MODE = os.getenv("MODE", "http")
BALANCE = os.getenv("BALANCE", "roundrobin")
MAXCONN = os.getenv("MAXCONN", "4096")
SSL = os.getenv("SSL", "")
SESSION_COOKIE = os.getenv("SESSION_COOKIE")
OPTION = os.getenv("OPTION", "redispatch, httplog, dontlognull, forwardfor").split(",")
TIMEOUT = os.getenv("TIMEOUT", "connect 5000, client 50000, server 50000").split(",")
VIRTUAL_HOST = os.getenv("VIRTUAL_HOST", None)
TUTUM_CONTAINER_API_URL = os.getenv("TUTUM_CONTAINER_API_URL", None)

TUTUM_AUTH = os.getenv("TUTUM_AUTH")
DEBUG = os.getenv("DEBUG", False)

# Const var
CONFIG_FILE = '/etc/haproxy/haproxy.cfg'
HAPROXY_CMD = ['/usr/sbin/haproxy', '-f', CONFIG_FILE, '-db']
POLLING_PERIOD = 30
LINK_ENV_PATTERN = "_PORT_%s_TCP" % PORT
LINK_ADDR_SUFFIX = LINK_ENV_PATTERN + "_ADDR"
LINK_PORT_SUFFIX = LINK_ENV_PATTERN + "_PORT"
TUTUM_URL_SUFFIX = "_TUTUM_API_URL"

# Global Var
HAPROXY_CURRENT_SUBPROCESS = None

endpoint_match = re.compile(r"(?P<proto>tcp|udp):\/\/(?P<addr>[^:]*):(?P<port>.*)")

def get_cfg_text(cfg):
    text = ""
    for section, contents in cfg.items():
        text += "%s\n" % section
        for content in contents:
            text += "  %s\n" % content
    return text.strip()

def create_default_cfg(maxconn, mode):
    cfg = OrderedDict({
        "global": ["log 127.0.0.1 local0",
                   "log 127.0.0.1 local1 notice",
                   "maxconn %s" % maxconn,
                   "pidfile /var/run/haproxy.pid",
                   "user haproxy",
                   "group haproxy",
                   "daemon"],
        "defaults": ["log     global",
                     "mode %s" % mode]})
    for option in OPTION:
        if option:
            cfg["defaults"].append("option %s" % option.strip())
    for timeout in TIMEOUT:
        if timeout:
            cfg["defaults"].append("timeout %s" % timeout.strip())

    return cfg

def get_backend_routes(dict_var):
    # Return sth like: {'HELLO_WORLD_1': {'addr': '172.17.0.103', 'port': '80'},
    # 'HELLO_WORLD_2': {'addr': '172.17.0.95', 'port': '80'}}
    addr_port_dict = {}
    for name, value in dict_var.iteritems():
        position = string.find(name, LINK_ENV_PATTERN)
        if position != -1:
            container_name = name[:position]
            add_port = addr_port_dict.get(container_name, {'addr': "", 'port': ""})
            if name.endswith(LINK_ADDR_SUFFIX):
                add_port['addr'] = value
            elif name.endswith(LINK_PORT_SUFFIX):
                add_port['port'] = value
            addr_port_dict[container_name] = add_port

    return addr_port_dict

def update_cfg(cfg, backend_routes, vhost):
    logger.debug("Updating cfg: \n old cfg: %s\n backend_routes: %s\n vhost: %s", cfg, backend_routes, vhost)
    # Set frontend
    frontend = []
    frontend.append("bind 0.0.0.0:80")
    if SSL:
        frontend.append("redirect scheme https code 301 if !{ ssl_fc }"),
        frontend.append("bind 0.0.0.0:443 %s" % SSL)
    if vhost:
        for service_name, domain_name in vhost.iteritems():
            service_name = service_name.upper()
            frontend.append("acl host_%s hdr(host) -i %s" % (service_name, domain_name))
            frontend.append("use_backend %s_cluster if host_%s" % (service_name, service_name))
    else:
        frontend.append("default_backend default_service")
    cfg["frontend default_frontend"] = frontend

    # Set backend
    if vhost:
        for service_name, domain_name in vhost.iteritems():
            service_name = service_name.upper()
            backend = []
            if SESSION_COOKIE:
                backend.append("appsession %s len 64 timeout 3h request-learn prefix" % (SESSION_COOKIE, ))

            backend.append("balance %s" % BALANCE)
            for container_name, addr_port in backend_routes.iteritems():
                if container_name.startswith(service_name):
                    server_string = "server %s %s:%s" % (container_name, addr_port["addr"], addr_port["port"])
                    if SESSION_COOKIE:
                        server_string += " cookie check"

                    # Do not add duplicate backend routes
                    duplicated = False
                    for server_str in backend:
                        if "%s:%s" % (addr_port["addr"], addr_port["port"]) in server_str:
                            duplicated = True
                            break
                    if not duplicated:
                        backend.append(server_string)
            if backend:
                cfg["backend %s_cluster" % service_name] = sorted(backend)

    else:
        backend = []
        if SESSION_COOKIE:
            backend.append("appsession %s len 64 timeout 3h request-learn prefix" % (SESSION_COOKIE, ))

        backend.append("balance %s" % BALANCE)
        for container_name, addr_port in backend_routes.iteritems():
            server_string = "server %s %s:%s" % (container_name, addr_port["addr"], addr_port["port"])
            if SESSION_COOKIE:
                server_string += " cookie check"

            # Do not add duplicate backend routes
            duplicated = False
            for server_str in backend:
                if "%s:%s" % (addr_port["addr"], addr_port["port"]) in server_str:
                    duplicated = True
                    break
            if not duplicated:
                backend.append(server_string)

        cfg["backend default_service"] = sorted(backend)

    logger.debug("New cfg: %s", cfg)

def save_config_file(cfg_text, config_file):
    try:
        directory = os.path.dirname(config_file)
        if not os.path.exists(directory):
            os.makedirs(directory)
        f = open(config_file, 'w')
    except Exception as e:
        logger.error(e)
    else:
        f.write(cfg_text)
        logger.info("Config file is updated")
        f.close()

def reload_haproxy():
    global HAPROXY_CURRENT_SUBPROCESS
    if HAPROXY_CURRENT_SUBPROCESS:
        # Reload haproxy
        logger.info("Reloading haproxy")
        process = subprocess.Popen(HAPROXY_CMD + ["-sf", str(HAPROXY_CURRENT_SUBPROCESS.pid)])
        HAPROXY_CURRENT_SUBPROCESS.wait()
        HAPROXY_CURRENT_SUBPROCESS = process
    else:
        # Launch haproxy
        logger.info("Lauching haproxy")
        HAPROXY_CURRENT_SUBPROCESS = subprocess.Popen(HAPROXY_CMD)

if __name__ == "__main__":
    logging.basicConfig(stream=sys.stdout)
    logging.getLogger(__name__).setLevel(logging.DEBUG if DEBUG else logging.INFO)

    cfg = create_default_cfg(MAXCONN, MODE)

    # Parse Virtual Host
    vhost = {}
    if VIRTUAL_HOST:
        for host in VIRTUAL_HOST.split(","):
            tmp = host.split("=", 2)
            if len(tmp) == 2:
                vhost[tmp[0].strip()] = tmp[1].strip()

    # Tell the user the mode of autoupdate we are using, if any
    if TUTUM_CONTAINER_API_URL:
        if TUTUM_AUTH:
            logger.info("HAproxy has access to Tutum API - will reload list of backends every %d seconds", POLLING_PERIOD)
        else:
            logger.warning("HAproxy doesn't have access to Tutum API and it's running in Tutum - you might want to give "
                "an API role to this service for automatic backend reconfiguration")
    else:
        logger.info("HAproxy is not running in Tutum")
    session = requests.Session()
    headers = {"Authorization": TUTUM_AUTH}

    # Main loop
    old_text = ""
    while True:
        try:
            if TUTUM_CONTAINER_API_URL and TUTUM_AUTH:
                # Running on Tutum with API access - fetch updated list of environment variables
                r = session.get(TUTUM_CONTAINER_API_URL, headers=headers)
                r.raise_for_status()
                container_details = r.json()

                backend_routes = {}
                for link in container_details.get("linked_to_container", []):
                    for port, endpoint in link.get("endpoints", {}).iteritems():
                        if port == "%s/tcp" % PORT:
                            backend_routes[link["name"]] = endpoint_match.match(endpoint).groupdict()
            else:
                # No Tutum API access - configuring backends based on static environment variables
                backend_routes = get_backend_routes(os.environ)

            # Update backend routes
            update_cfg(cfg, backend_routes, vhost)
            cfg_text = get_cfg_text(cfg)

            # If cfg changes, write to file
            if old_text != cfg_text:
                logger.info("HAProxy configuration has been changed:\n%s" % cfg_text)
                save_config_file(cfg_text, CONFIG_FILE)
                reload_haproxy()
                old_text = cfg_text
        except Exception as e:
            logger.exception("Error: %s" % e)

        time.sleep(POLLING_PERIOD)
<commit_msg>set header X-Forwarded-Proto to https if SSL is used<commit_after>#!/usr/bin/env python
import logging
import os
import time
import string
import subprocess
import sys
import re
from collections import OrderedDict

import requests

logger = logging.getLogger(__name__)

# Config ENV
PORT = os.getenv("PORT", "80")
MODE = os.getenv("MODE", "http")
BALANCE = os.getenv("BALANCE", "roundrobin")
MAXCONN = os.getenv("MAXCONN", "4096")
SSL = os.getenv("SSL", "")
SESSION_COOKIE = os.getenv("SESSION_COOKIE")
OPTION = os.getenv("OPTION", "redispatch, httplog, dontlognull, forwardfor").split(",")
TIMEOUT = os.getenv("TIMEOUT", "connect 5000, client 50000, server 50000").split(",")
VIRTUAL_HOST = os.getenv("VIRTUAL_HOST", None)
TUTUM_CONTAINER_API_URL = os.getenv("TUTUM_CONTAINER_API_URL", None)

TUTUM_AUTH = os.getenv("TUTUM_AUTH")
DEBUG = os.getenv("DEBUG", False)

# Const var
CONFIG_FILE = '/etc/haproxy/haproxy.cfg'
HAPROXY_CMD = ['/usr/sbin/haproxy', '-f', CONFIG_FILE, '-db']
POLLING_PERIOD = 30
LINK_ENV_PATTERN = "_PORT_%s_TCP" % PORT
LINK_ADDR_SUFFIX = LINK_ENV_PATTERN + "_ADDR"
LINK_PORT_SUFFIX = LINK_ENV_PATTERN + "_PORT"
TUTUM_URL_SUFFIX = "_TUTUM_API_URL"

# Global Var
HAPROXY_CURRENT_SUBPROCESS = None

endpoint_match = re.compile(r"(?P<proto>tcp|udp):\/\/(?P<addr>[^:]*):(?P<port>.*)")

def get_cfg_text(cfg):
    text = ""
    for section, contents in cfg.items():
        text += "%s\n" % section
        for content in contents:
            text += "  %s\n" % content
    return text.strip()

def create_default_cfg(maxconn, mode):
    cfg = OrderedDict({
        "global": ["log 127.0.0.1 local0",
                   "log 127.0.0.1 local1 notice",
                   "maxconn %s" % maxconn,
                   "pidfile /var/run/haproxy.pid",
                   "user haproxy",
                   "group haproxy",
                   "daemon"],
        "defaults": ["log     global",
                     "mode %s" % mode]})
    for option in OPTION:
        if option:
            cfg["defaults"].append("option %s" % option.strip())
    for timeout in TIMEOUT:
        if timeout:
            cfg["defaults"].append("timeout %s" % timeout.strip())

    return cfg

def get_backend_routes(dict_var):
    # Return sth like: {'HELLO_WORLD_1': {'addr': '172.17.0.103', 'port': '80'},
    # 'HELLO_WORLD_2': {'addr': '172.17.0.95', 'port': '80'}}
    addr_port_dict = {}
    for name, value in dict_var.iteritems():
        position = string.find(name, LINK_ENV_PATTERN)
        if position != -1:
            container_name = name[:position]
            add_port = addr_port_dict.get(container_name, {'addr': "", 'port': ""})
            if name.endswith(LINK_ADDR_SUFFIX):
                add_port['addr'] = value
            elif name.endswith(LINK_PORT_SUFFIX):
                add_port['port'] = value
            addr_port_dict[container_name] = add_port

    return addr_port_dict

def update_cfg(cfg, backend_routes, vhost):
    logger.debug("Updating cfg: \n old cfg: %s\n backend_routes: %s\n vhost: %s", cfg, backend_routes, vhost)
    # Set frontend
    frontend = []
    frontend.append("bind 0.0.0.0:80")
    if SSL:
        frontend.append("redirect scheme https code 301 if !{ ssl_fc }"),
        frontend.append("bind 0.0.0.0:443 %s" % SSL)
	frontend.append("reqadd X-Forwarded-Proto:\ https")
    if vhost:
        for service_name, domain_name in vhost.iteritems():
            service_name = service_name.upper()
            frontend.append("acl host_%s hdr(host) -i %s" % (service_name, domain_name))
            frontend.append("use_backend %s_cluster if host_%s" % (service_name, service_name))
    else:
        frontend.append("default_backend default_service")
    cfg["frontend default_frontend"] = frontend

    # Set backend
    if vhost:
        for service_name, domain_name in vhost.iteritems():
            service_name = service_name.upper()
            backend = []
            if SESSION_COOKIE:
                backend.append("appsession %s len 64 timeout 3h request-learn prefix" % (SESSION_COOKIE, ))

            backend.append("balance %s" % BALANCE)
            for container_name, addr_port in backend_routes.iteritems():
                if container_name.startswith(service_name):
                    server_string = "server %s %s:%s" % (container_name, addr_port["addr"], addr_port["port"])
                    if SESSION_COOKIE:
                        server_string += " cookie check"

                    # Do not add duplicate backend routes
                    duplicated = False
                    for server_str in backend:
                        if "%s:%s" % (addr_port["addr"], addr_port["port"]) in server_str:
                            duplicated = True
                            break
                    if not duplicated:
                        backend.append(server_string)
            if backend:
                cfg["backend %s_cluster" % service_name] = sorted(backend)

    else:
        backend = []
        if SESSION_COOKIE:
            backend.append("appsession %s len 64 timeout 3h request-learn prefix" % (SESSION_COOKIE, ))

        backend.append("balance %s" % BALANCE)
        for container_name, addr_port in backend_routes.iteritems():
            server_string = "server %s %s:%s" % (container_name, addr_port["addr"], addr_port["port"])
            if SESSION_COOKIE:
                server_string += " cookie check"

            # Do not add duplicate backend routes
            duplicated = False
            for server_str in backend:
                if "%s:%s" % (addr_port["addr"], addr_port["port"]) in server_str:
                    duplicated = True
                    break
            if not duplicated:
                backend.append(server_string)

        cfg["backend default_service"] = sorted(backend)

    logger.debug("New cfg: %s", cfg)

def save_config_file(cfg_text, config_file):
    try:
        directory = os.path.dirname(config_file)
        if not os.path.exists(directory):
            os.makedirs(directory)
        f = open(config_file, 'w')
    except Exception as e:
        logger.error(e)
    else:
        f.write(cfg_text)
        logger.info("Config file is updated")
        f.close()

def reload_haproxy():
    global HAPROXY_CURRENT_SUBPROCESS
    if HAPROXY_CURRENT_SUBPROCESS:
        # Reload haproxy
        logger.info("Reloading haproxy")
        process = subprocess.Popen(HAPROXY_CMD + ["-sf", str(HAPROXY_CURRENT_SUBPROCESS.pid)])
        HAPROXY_CURRENT_SUBPROCESS.wait()
        HAPROXY_CURRENT_SUBPROCESS = process
    else:
        # Launch haproxy
        logger.info("Lauching haproxy")
        HAPROXY_CURRENT_SUBPROCESS = subprocess.Popen(HAPROXY_CMD)

if __name__ == "__main__":
    logging.basicConfig(stream=sys.stdout)
    logging.getLogger(__name__).setLevel(logging.DEBUG if DEBUG else logging.INFO)

    cfg = create_default_cfg(MAXCONN, MODE)

    # Parse Virtual Host
    vhost = {}
    if VIRTUAL_HOST:
        for host in VIRTUAL_HOST.split(","):
            tmp = host.split("=", 2)
            if len(tmp) == 2:
                vhost[tmp[0].strip()] = tmp[1].strip()

    # Tell the user the mode of autoupdate we are using, if any
    if TUTUM_CONTAINER_API_URL:
        if TUTUM_AUTH:
            logger.info("HAproxy has access to Tutum API - will reload list of backends every %d seconds", POLLING_PERIOD)
        else:
            logger.warning("HAproxy doesn't have access to Tutum API and it's running in Tutum - you might want to give "
                "an API role to this service for automatic backend reconfiguration")
    else:
        logger.info("HAproxy is not running in Tutum")
    session = requests.Session()
    headers = {"Authorization": TUTUM_AUTH}

    # Main loop
    old_text = ""
    while True:
        try:
            if TUTUM_CONTAINER_API_URL and TUTUM_AUTH:
                # Running on Tutum with API access - fetch updated list of environment variables
                r = session.get(TUTUM_CONTAINER_API_URL, headers=headers)
                r.raise_for_status()
                container_details = r.json()

                backend_routes = {}
                for link in container_details.get("linked_to_container", []):
                    for port, endpoint in link.get("endpoints", {}).iteritems():
                        if port == "%s/tcp" % PORT:
                            backend_routes[link["name"]] = endpoint_match.match(endpoint).groupdict()
            else:
                # No Tutum API access - configuring backends based on static environment variables
                backend_routes = get_backend_routes(os.environ)

            # Update backend routes
            update_cfg(cfg, backend_routes, vhost)
            cfg_text = get_cfg_text(cfg)

            # If cfg changes, write to file
            if old_text != cfg_text:
                logger.info("HAProxy configuration has been changed:\n%s" % cfg_text)
                save_config_file(cfg_text, CONFIG_FILE)
                reload_haproxy()
                old_text = cfg_text
        except Exception as e:
            logger.exception("Error: %s" % e)

        time.sleep(POLLING_PERIOD)
<|endoftext|>


<commit_before>
#F="gap-log-halloween-11.50am"
F="gap-log-at-1.09pm"

class LogInfo(object):
    pre_line = None
    lg_line = None
    lg_line_no = 0
    delay = 0

    def __init__(self):
        pass

    def set(self, pre_line, lg_line, lg_line_no, delay):
        self.pre_line = pre_line
        self.lg_line = lg_line
        self.lg_line_no = lg_line_no
        self.delay = delay

    def do_print(self, total_lines):
        print "\n longest delay: {} seconds ({} minutes) at line {} of {} \n\n line {}: {} \n line {}: {} \n".\
            format(self.delay, self.delay / 60, self.lg_line_no, total_lines, self.lg_line_no-1, self.pre_line, self.lg_line_no, self.lg_line)

class RequestDwell(object):
    """ parse an app.log file
        line should start with a time stamp "hh:mm:ss"
    """
    info = LogInfo()
    line_count = 0
    number_of_delays = 0

    def __init__(self, args):
        self.log_file = args.file_name
        self.min_distance = args.distance

    @classmethod
    def get_args(cls):
        import argparse
        parser = argparse.ArgumentParser(prog='log-dwell', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
        parser.add_argument('--file_name',  '-f',  help='Log file', default="app.log")
        parser.add_argument('--distance',   '-d',  help='Number of seconds to capture', default=60)
        args = parser.parse_args()
        return args

    def process(self):
        #import pdb; pdb.set_trace()

        largest = 0
        old_time = 0

        with open(self.log_file) as f:
            last_line = None
            for i, line in enumerate(f):
                if len(line) >= 8:
                    time_str = line[:8]
                    new_time = self.timestamp_to_seconds(time_str)
                    if new_time is None:
                        continue
                    dist = new_time - old_time
                    if dist > self.min_distance:
                        self.number_of_delays += 1
                    if dist > largest:
                        #import pdb; pdb.set_trace()
                        largest = dist
                        self.info.set(last_line, line, i, dist)

                    old_time = new_time
                    last_line = line

            self.line_count = i

    def do_print(self):
        print "\n total number of delays that exceeded {} seconds:  {}".format(self.min_distance, self.number_of_delays)
        self.info.do_print(self.line_count)

    @classmethod
    def timestamp_to_seconds(cls, time_str):
        """ take in a h:m:s string, ala 11:22:30 and return total seconds """
        ret_val = None
        try:
            h, m, s = time_str.split(':')
            ret_val = int(h) * 3600 + int(m) * 60 + int(s)
        except:
            pass
        return ret_val

    @classmethod
    def factory(cls):
        args = cls.get_args()
        inst = cls(args)
        return inst

def main():
    rd = RequestDwell.factory()
    rd.process()
    rd.do_print()

if __name__ == "__main__":
    main()
<commit_msg>fix parser to pull in int for seconds<commit_after>
#F="gap-log-halloween-11.50am"
F="gap-log-at-1.09pm"

class LogInfo(object):
    pre_line = None
    lg_line = None
    lg_line_no = 0
    delay = 0

    def __init__(self):
        pass

    def set(self, pre_line, lg_line, lg_line_no, delay):
        self.pre_line = pre_line
        self.lg_line = lg_line
        self.lg_line_no = lg_line_no
        self.delay = delay

    def do_print(self, total_lines):
        print "\n longest delay: {} seconds ({} minutes) at line {} of {} \n\n line {}: {} \n line {}: {} \n".\
            format(self.delay, self.delay / 60, self.lg_line_no, total_lines, self.lg_line_no-1, self.pre_line, self.lg_line_no, self.lg_line)

class RequestDwell(object):
    """ parse an app.log file
        line should start with a time stamp "hh:mm:ss"
    """
    info = LogInfo()
    line_count = 0
    number_of_delays = 0

    def __init__(self, args):
        self.log_file = args.file_name
        self.min_distance = args.distance

    @classmethod
    def get_args(cls):
        import argparse
        parser = argparse.ArgumentParser(prog='log-dwell', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
        parser.add_argument('--file_name',  '-f',  help='Log file', default="app.log")
        parser.add_argument('--distance',   '-d',  help='Number of seconds to capture', type=int, default=60)
        args = parser.parse_args()
        return args

    def process(self):
        #import pdb; pdb.set_trace()

        largest = 0
        old_time = 0

        with open(self.log_file) as f:
            last_line = None
            for i, line in enumerate(f):
                if len(line) >= 8:
                    time_str = line[:8]
                    new_time = self.timestamp_to_seconds(time_str)
                    if new_time is None:
                        continue
                    dist = new_time - old_time
                    if dist > self.min_distance:
                        self.number_of_delays += 1
                    if dist > largest:
                        #import pdb; pdb.set_trace()
                        largest = dist
                        self.info.set(last_line, line, i, dist)

                    old_time = new_time
                    last_line = line

            self.line_count = i

    def do_print(self):
        print "\n total number of delays that exceeded {} seconds:  {}".format(self.min_distance, self.number_of_delays)
        self.info.do_print(self.line_count)

    @classmethod
    def timestamp_to_seconds(cls, time_str):
        """ take in a h:m:s string, ala 11:22:30 and return total seconds """
        ret_val = None
        try:
            h, m, s = time_str.split(':')
            ret_val = int(h) * 3600 + int(m) * 60 + int(s)
        except:
            pass
        return ret_val

    @classmethod
    def factory(cls):
        args = cls.get_args()
        inst = cls(args)
        return inst

def main():
    rd = RequestDwell.factory()
    rd.process()
    rd.do_print()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Attempt to manage the disaster that is IEM symlinking"""
import sys
import os

# LINK , TARGET
PAIRS = [
         ['/mesonet/merra', '/mnt/mtarchive/longterm/merra'],
         ['/mesonet/nawips', '/mnt/mesonet2/gempak'],
         ['/mesonet/scripts', '/mnt/mesonet2/scripts'],
         ['/mesonet/TABLES', '/mnt/mesonet2/TABLES'],
         ['/mesonet/tmp', '/mnt/mesonet2/tmp'],
         ['/mesonet/wepp', '/mnt/idep'],
         ['/mesonet/ARCHIVE/dailydata', '/mnt/mtarchive/longterm/dailydata'],
         ['/mesonet/ARCHIVE/data', '/mnt/mesonet2/ARCHIVE/data'],
         ['/mesonet/ARCHIVE/gempak', '/mnt/mtarchive/longterm/gempak'],
         ['/mesonet/ARCHIVE/nexrad', '/mnt/mtarchive/longterm/nexrad3_iowa'],
         ['/mesonet/ARCHIVE/raw', '/mnt/mesonet2/ARCHIVE/raw'],
         ['/mesonet/ARCHIVE/rer', '/mnt/mesonet/ARCHIVE/rer'],
         ['/mesonet/ARCHIVE/wunder', '/mnt/mesonet2/ARCHIVE/wunder'],
         ['/mesonet/data/alerts', '/mnt/mesonet2/data/alerts'],
         ['/mesonet/data/dm', '/mnt/mesonet2/data/dm'],
         ['/mesonet/data/dotcams', '/mnt/mesonet2/data/dotcams'],
         ['/mesonet/data/gempak', '/mnt/mesonet2/data/gempak'],
         ['/mesonet/data/harry', '/mnt/mesonet/ARCHIVE/raw/harry'],
         ['/mesonet/data/iemre', '/mnt/mesonet/data/iemre'],
         ['/mesonet/data/prism', '/mnt/mesonet/data/prism'],
         ['/mesonet/data/incoming', '/mnt/mesonet2/data/incoming'],
         ['/mesonet/data/logs', '/mnt/mesonet2/data/logs'],
         ['/mesonet/data/madis', '/mnt/mesonet2/data/madis'],
         ['/mesonet/data/model', '/mnt/mesonet2/data/model'],
         ['/mesonet/data/nccf', '/mnt/mesonet2/data/nccf'],
         ['/mesonet/data/nexrad', '/mnt/nexrad3/nexrad'],
         ['/mesonet/data/smos', '/mnt/mesonet2/data/smos'],
         ['/mesonet/data/text', '/mnt/mesonet2/data/text'],
         ['/mesonet/share/cases', '/mnt/mesonet/share/cases'],
         ['/mesonet/share/climodat', '/mnt/mesonet2/share/climodat'],
         ['/mesonet/share/features', '/mnt/mesonet/share/features'],
         ['/mesonet/share/frost', '/mnt/mesonet/share/frost'],
         ['/mesonet/share/iemmaps', '/mnt/mesonet/share/iemmaps'],
         ['/mesonet/share/iemre', '/mnt/mesonet/data/iemre'],
         ['/mesonet/share/lapses', '/mnt/mesonet/share/lapses'],
         ['/mesonet/share/mec', '/mnt/mesonet/share/mec'],
         ['/mesonet/share/pickup', '/mnt/mesonet2/share/pickup'],
         ['/mesonet/share/pics', '/mnt/mesonet/share/pics'],
         ['/mesonet/share/present', '/mnt/mesonet/share/present'],
         ['/mesonet/share/usage', '/mnt/mesonet/share/usage'],
         ['/mesonet/share/windrose', '/mnt/mesonet2/share/windrose'],
         ]

def do(link, target):
    """Do things"""
    if not os.path.isdir(target):
        print('ERROR: link target: %s is not found' % (target,))
        return
    if not os.path.islink(link) and os.path.isdir(link):
        print('ERROR: symlink: %s is already a directory!' % (link,))
        return
    if os.path.islink(link):
        oldtarget = os.path.realpath(link)
        if oldtarget == target:
            return
        os.unlink(link)
    print("%s -> %s" % (link, target))
    os.symlink(target, link)

def main(argv):
    """Go Main"""
    for (link, target) in PAIRS:
        do(link, target)

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>go to mesonet2, more space<commit_after>"""Attempt to manage the disaster that is IEM symlinking"""
import sys
import os

# LINK , TARGET
PAIRS = [
         ['/mesonet/merra', '/mnt/mtarchive/longterm/merra'],
         ['/mesonet/nawips', '/mnt/mesonet2/gempak'],
         ['/mesonet/scripts', '/mnt/mesonet2/scripts'],
         ['/mesonet/TABLES', '/mnt/mesonet2/TABLES'],
         ['/mesonet/tmp', '/mnt/mesonet2/tmp'],
         ['/mesonet/wepp', '/mnt/idep'],
         ['/mesonet/ARCHIVE/dailydata', '/mnt/mtarchive/longterm/dailydata'],
         ['/mesonet/ARCHIVE/data', '/mnt/mesonet2/ARCHIVE/data'],
         ['/mesonet/ARCHIVE/gempak', '/mnt/mtarchive/longterm/gempak'],
         ['/mesonet/ARCHIVE/nexrad', '/mnt/mtarchive/longterm/nexrad3_iowa'],
         ['/mesonet/ARCHIVE/raw', '/mnt/mesonet2/ARCHIVE/raw'],
         ['/mesonet/ARCHIVE/rer', '/mnt/mesonet/ARCHIVE/rer'],
         ['/mesonet/ARCHIVE/wunder', '/mnt/mesonet2/ARCHIVE/wunder'],
         ['/mesonet/data/alerts', '/mnt/mesonet2/data/alerts'],
         ['/mesonet/data/dm', '/mnt/mesonet2/data/dm'],
         ['/mesonet/data/dotcams', '/mnt/mesonet2/data/dotcams'],
         ['/mesonet/data/gempak', '/mnt/mesonet2/data/gempak'],
         ['/mesonet/data/harry', '/mnt/mesonet/ARCHIVE/raw/harry'],
         ['/mesonet/data/iemre', '/mnt/mesonet/data/iemre'],
         ['/mesonet/data/prism', '/mnt/mesonet2/data/prism'],
         ['/mesonet/data/incoming', '/mnt/mesonet2/data/incoming'],
         ['/mesonet/data/logs', '/mnt/mesonet2/data/logs'],
         ['/mesonet/data/madis', '/mnt/mesonet2/data/madis'],
         ['/mesonet/data/model', '/mnt/mesonet2/data/model'],
         ['/mesonet/data/nccf', '/mnt/mesonet2/data/nccf'],
         ['/mesonet/data/nexrad', '/mnt/nexrad3/nexrad'],
         ['/mesonet/data/smos', '/mnt/mesonet2/data/smos'],
         ['/mesonet/data/text', '/mnt/mesonet2/data/text'],
         ['/mesonet/share/cases', '/mnt/mesonet/share/cases'],
         ['/mesonet/share/climodat', '/mnt/mesonet2/share/climodat'],
         ['/mesonet/share/features', '/mnt/mesonet/share/features'],
         ['/mesonet/share/frost', '/mnt/mesonet/share/frost'],
         ['/mesonet/share/iemmaps', '/mnt/mesonet/share/iemmaps'],
         ['/mesonet/share/iemre', '/mnt/mesonet/data/iemre'],
         ['/mesonet/share/lapses', '/mnt/mesonet/share/lapses'],
         ['/mesonet/share/mec', '/mnt/mesonet/share/mec'],
         ['/mesonet/share/pickup', '/mnt/mesonet2/share/pickup'],
         ['/mesonet/share/pics', '/mnt/mesonet/share/pics'],
         ['/mesonet/share/present', '/mnt/mesonet/share/present'],
         ['/mesonet/share/usage', '/mnt/mesonet/share/usage'],
         ['/mesonet/share/windrose', '/mnt/mesonet2/share/windrose'],
         ]

def do(link, target):
    """Do things"""
    if not os.path.isdir(target):
        print('ERROR: link target: %s is not found' % (target,))
        return
    if not os.path.islink(link) and os.path.isdir(link):
        print('ERROR: symlink: %s is already a directory!' % (link,))
        return
    if os.path.islink(link):
        oldtarget = os.path.realpath(link)
        if oldtarget == target:
            return
        os.unlink(link)
    print("%s -> %s" % (link, target))
    os.symlink(target, link)

def main(argv):
    """Go Main"""
    for (link, target) in PAIRS:
        do(link, target)

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>#
# Copyright 2010-2012 Red Hat, Inc. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Refer to the README and COPYING files for full details of the license.
#

import logging
import os
import socket
import subprocess
import threading
import time
from OVirtAgentLogic import AgentLogicBase, DataRetriverBase

try:
    from CredServer import CredServer
except ImportError:
    # The CredServer doesn't exist in RHEL-5. So we provide a
    # fake server that do nothing.
    class CredServer(threading.Thread):
        def user_authenticated(self, credentials):
            pass

class PkgMgr(object):

    def rpm_list_packages(self, app_list):
        """ Implementes the package retrieval for rpm based environments """
        apps = set()
        for name in app_list.split():
            ts = self.rpm.TransactionSet()
            for app in ts.dbMatch('name', name):
                apps.add("%s-%s-%s" %
                         (app['name'], app['version'], app['release']))
        return apps

    def apt_list_packages(self, app_list):
        """ Implementes the package retrieval for apt based environments """
        INSTALLED_STATE = self.apt_pkg.CURSTATE_INSTALLED
        apps = set()
        cache = self.apt_pkg.Cache()
        for app in app_list.split():
            if app in cache:
                pkg = cache[app]
                # Normal package
                if pkg.current_state == INSTALLED_STATE:
                    detail = (app, pkg.current_ver.ver_str)
                    apps.add("%s-%s" % (detail))
                # virtual package
                elif len(pkg.provides_list) > 0:
                    for _, _, pkg in pkg.provides_list:
                        if pkg.parent_pkg.current_state == INSTALLED_STATE:
                            detail = (app, pkg.parent_pkg.current_ver.ver_str)
                            apps.add("%s-%s" % (detail))
        return apps

    def list_pkgs(self, app_list):
        """ Implements the package retrieval for apt and rpm if present and
            returns a joined list of packages installed on the system. """
        apps = set()
        if self.rpm:
            apps.update(self.rpm_list_packages(app_list))
        if self.apt_pkg:
            apps.update(self.apt_list_packages(app_list))
        apps = list(apps)
        logging.debug("PkgMgr: list_pkgs returns [%s]" % (str(apps)))
        return apps

    def __init__(self):
        self.rpm = None
        self.apt_pkg = None
        try:
            import rpm
            self.rpm = rpm
        except ImportError:
            pass

        try:
            from apt import apt_pkg
            apt_pkg.init()
            self.apt_pkg = apt_pkg
        except ImportError:
            pass

        if not self.rpm and not self.apt_pkg:
            logging.info("Unknown package management. Application list "
                         "report is disabled.")

class NicMgr(object):

    def __init__(self):
        try:
            import ethtool
        except ImportError:
            raise NotImplementedError
        self.ethtool = ethtool
        self.list_nics = self.ethtool_list_nics

    def ethtool_list_nics(self):
        interfaces = list()
        try:
            for dev in self.ethtool.get_active_devices():
                flags = self.ethtool.get_flags(dev)
                if not(flags & self.ethtool.IFF_LOOPBACK):
                    devinfo = self.ethtool.get_interfaces_info(dev)[0]
                    interfaces.append(
                        {'name': dev,
                         'inet': [self.ethtool.get_ipaddr(dev)],
                         'inet6': map(lambda ip: ip.address,
                                      devinfo.get_ipv6_addresses()),
                         'hw': self.ethtool.get_hwaddr(dev)})
        except:
            logging.exception("Error retrieving network interfaces.")
        return interfaces

class CommandHandlerLinux:

    def __init__(self, agent):
        self.agent = agent

    def lock_screen(self):
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-locksession']
        logging.debug("Executing lock session command: '%s'", cmd)
        subprocess.call(cmd)

    def login(self, credentials):
        self.agent.cred_server.user_authenticated(credentials)

    def logoff(self):
        pass

    def shutdown(self, timeout, msg):
        # The shutdown command works with minutes while vdsm send value in
        # seconds, so we round up the value to minutes.
        delay = (int(timeout) + 59) / 60
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-shutdown', '-h',
               "+%d" % (delay), "\"%s\"" % (msg)]
        logging.debug("Executing shutdown command: %s", cmd)
        subprocess.call(cmd)

    def hibernate(self, state):
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-hibernate', state]
        logging.debug("Executing hibernate command: %s", cmd)
        subprocess.call(cmd)

class LinuxDataRetriver(DataRetriverBase):

    def __init__(self):
        try:
            pkgmgr = PkgMgr()
        except NotImplementedError:
            self.list_pkgs = lambda app_list: []
        else:
            self.list_pkgs = pkgmgr.list_pkgs
        try:
            nicmgr = NicMgr()
        except NotImplementedError:
            self.list_nics = lambda: []
        else:
            self.list_nics = nicmgr.list_nics
        self.app_list = ""
        self.ignored_fs = ""
        self._init_vmstat()
        DataRetriverBase.__init__(self)

    def getMachineName(self):
        return socket.getfqdn()

    def getOsVersion(self):
        return os.uname()[2]

    def getAllNetworkInterfaces(self):
        return self.list_nics()

    def getApplications(self):
        return self.list_pkgs(self.app_list)

    def getAvailableRAM(self):
        free = 0
        for line in open('/proc/meminfo'):
            var, value = line.strip().split()[0:2]
            if var in ('MemFree:', 'Buffers:', 'Cached:'):
                free += long(value)
        return str(free / 1024)

    def getUsers(self):
        users = ''
        try:
            cmdline = '/usr/bin/users | /usr/bin/tr " " "\n" | /usr/bin/uniq'
            users = ' '.join(os.popen(cmdline).read().split())
        except:
            logging.exception("Error retrieving logged in users.")
        return users

    def getActiveUser(self):
        users = os.popen('/usr/bin/users').read().split()
        try:
            user = users[0]
        except:
            user = 'None'
        return user

    def getDisksUsage(self):
        usages = list()
        try:
            mounts = open('/proc/mounts')
            for mount in mounts:
                (device, path, fs) = mount.split()[:3]
                if not fs in self.ignored_fs:
                    # path might include spaces.
                    path = path.decode("string-escape")
                    statvfs = os.statvfs(path)
                    total = statvfs.f_bsize * statvfs.f_blocks
                    used = total - statvfs.f_bsize * statvfs.f_bfree
                    usages.append({'path': path, 'fs': fs, 'total': total,
                                   'used': used})
            mounts.close()
        except:
            logging.exception("Error retrieving disks usages.")
        return usages

    def getMemoryStats(self):
        try:
            self._get_meminfo()
            self._get_vmstat()
        except:
            logging.exception("Error retrieving memory stats.")
        return self.memStats

    def _init_vmstat(self):
        self.vmstat = {}
        self.vmstat['timestamp_prev'] = time.time()
        fields = ['swap_in', 'swap_out', 'pageflt', 'majflt']
        for field in fields:
            self.vmstat[field + '_prev'] = None
            self.vmstat[field + '_cur'] = None

    def _get_meminfo(self):
        fields = {'MemTotal:' : 0, 'MemFree:' : 0, 'Buffers:' : 0, \
                  'Cached:' : 0}
        free = 0
        for line in open('/proc/meminfo'):
            (key, value) = line.strip().split()[0:2]
            if key in fields.keys():
                fields[key] = int(value)
            if key in ('MemFree:', 'Buffers:', 'Cached:'):
                free += int(value)
        self.memStats['mem_total'] = fields['MemTotal:']
        self.memStats['mem_unused'] = fields['MemFree:']
        self.memStats['mem_free'] = free

    def _get_vmstat(self):
        """
        /proc/vmstat reports cumulative statistics so we must subtract the
        previous values to get the difference since the last collection.
        """
        fields = {'pswpin': 'swap_in', 'pswpout': 'swap_out',
                  'pgfault': 'pageflt', 'pgmajfault': 'majflt'}

        self.vmstat['timestamp_cur'] = time.time()
        interval = self.vmstat['timestamp_cur'] - self.vmstat['timestamp_prev']
        self.vmstat['timestamp_prev'] = self.vmstat['timestamp_cur']

        for line in open('/proc/vmstat'):
            (key, value) = line.strip().split()[0:2]
            if key in fields.keys():
                name = fields[key]
                self.vmstat[name + '_prev'] = self.vmstat[name + '_cur']
                self.vmstat[name + '_cur'] = int(value)
                if self.vmstat[name + '_prev'] is None:
                    self.vmstat[name + '_prev'] = self.vmstat[name + '_cur']
                self.memStats[name] = int((self.vmstat[name + '_cur'] -
                                           self.vmstat[name + '_prev']) /
                                          interval)

class LinuxVdsAgent(AgentLogicBase):

    def __init__(self, config):
        AgentLogicBase.__init__(self, config)
        self.dr = LinuxDataRetriver()
        self.dr.app_list = config.get("general", "applications_list")
        self.dr.ignored_fs = set(config.get("general", "ignored_fs").split())
        self.commandHandler = CommandHandlerLinux(self)
        self.cred_server = CredServer()

    def run(self):
        self.cred_server.start()
        AgentLogicBase.run(self)

    def stop(self):
        self.cred_server.join()
        AgentLogicBase.stop(self)

def test():
    dr = LinuxDataRetriver()
    dr.app_list = "kernel kernel-headers aspell"
    dr.ignored_fs = set("rootfs tmpfs autofs cgroup selinuxfs udev mqueue "
                        "nfsd proc sysfs devtmpfs hugetlbfs rpc_pipefs devpts "
                        "securityfs debugfs binfmt_misc".split())
    print "Machine Name:", dr.getMachineName()
    print "OS Version:", dr.getOsVersion()
    print "Network Interfaces:", dr.getAllNetworkInterfaces()
    print "Installed Applications:", dr.getApplications()
    print "Available RAM:", dr.getAvailableRAM()
    print "Logged in Users:", dr.getUsers()
    print "Active User:", dr.getActiveUser()
    print "Disks Usage:", dr.getDisksUsage()
    print "Memory Stats:", dr.getMemoryStats()

if __name__ == '__main__':
    test()
<commit_msg>Fixed support for reporting devices with only ipv6.<commit_after>#
# Copyright 2010-2012 Red Hat, Inc. and/or its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Refer to the README and COPYING files for full details of the license.
#

import logging
import os
import socket
import subprocess
import threading
import time
from OVirtAgentLogic import AgentLogicBase, DataRetriverBase

try:
    from CredServer import CredServer
except ImportError:
    # The CredServer doesn't exist in RHEL-5. So we provide a
    # fake server that do nothing.
    class CredServer(threading.Thread):
        def user_authenticated(self, credentials):
            pass

class PkgMgr(object):

    def rpm_list_packages(self, app_list):
        """ Implementes the package retrieval for rpm based environments """
        apps = set()
        for name in app_list.split():
            ts = self.rpm.TransactionSet()
            for app in ts.dbMatch('name', name):
                apps.add("%s-%s-%s" %
                         (app['name'], app['version'], app['release']))
        return apps

    def apt_list_packages(self, app_list):
        """ Implementes the package retrieval for apt based environments """
        INSTALLED_STATE = self.apt_pkg.CURSTATE_INSTALLED
        apps = set()
        cache = self.apt_pkg.Cache()
        for app in app_list.split():
            if app in cache:
                pkg = cache[app]
                # Normal package
                if pkg.current_state == INSTALLED_STATE:
                    detail = (app, pkg.current_ver.ver_str)
                    apps.add("%s-%s" % (detail))
                # virtual package
                elif len(pkg.provides_list) > 0:
                    for _, _, pkg in pkg.provides_list:
                        if pkg.parent_pkg.current_state == INSTALLED_STATE:
                            detail = (app, pkg.parent_pkg.current_ver.ver_str)
                            apps.add("%s-%s" % (detail))
        return apps

    def list_pkgs(self, app_list):
        """ Implements the package retrieval for apt and rpm if present and
            returns a joined list of packages installed on the system. """
        apps = set()
        if self.rpm:
            apps.update(self.rpm_list_packages(app_list))
        if self.apt_pkg:
            apps.update(self.apt_list_packages(app_list))
        apps = list(apps)
        logging.debug("PkgMgr: list_pkgs returns [%s]" % (str(apps)))
        return apps

    def __init__(self):
        self.rpm = None
        self.apt_pkg = None
        try:
            import rpm
            self.rpm = rpm
        except ImportError:
            pass

        try:
            from apt import apt_pkg
            apt_pkg.init()
            self.apt_pkg = apt_pkg
        except ImportError:
            pass

        if not self.rpm and not self.apt_pkg:
            logging.info("Unknown package management. Application list "
                         "report is disabled.")

class NicMgr(object):

    def __init__(self):
        try:
            import ethtool
        except ImportError:
            raise NotImplementedError
        self.ethtool = ethtool
        self.list_nics = self.ethtool_list_nics

    def _get_ipv4_addresses(self, dev):
        return [dev.ipv4_address] if dev.ipv4_address is not None else []

    def _get_ipv6_addresses(self, dev):
        return [ip.address for ip in dev.get_ipv6_addresses()]

    def ethtool_list_nics(self):
        interfaces = list()
        try:
            for dev in self.ethtool.get_devices():
                flags = self.ethtool.get_flags(dev)
                if flags & self.ethtool.IFF_UP and \
                        not(flags & self.ethtool.IFF_LOOPBACK):
                    devinfo = self.ethtool.get_interfaces_info(dev)[0]
                    interfaces.append(
                        {'name': dev,
                         'inet': self._get_ipv4_addresses(devinfo),
                         'inet6': self._get_ipv6_addresses(devinfo),
                         'hw': self.ethtool.get_hwaddr(dev)})
        except:
            logging.exception("Error retrieving network interfaces.")
        return interfaces

class CommandHandlerLinux:

    def __init__(self, agent):
        self.agent = agent

    def lock_screen(self):
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-locksession']
        logging.debug("Executing lock session command: '%s'", cmd)
        subprocess.call(cmd)

    def login(self, credentials):
        self.agent.cred_server.user_authenticated(credentials)

    def logoff(self):
        pass

    def shutdown(self, timeout, msg):
        # The shutdown command works with minutes while vdsm send value in
        # seconds, so we round up the value to minutes.
        delay = (int(timeout) + 59) / 60
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-shutdown', '-h',
               "+%d" % (delay), "\"%s\"" % (msg)]
        logging.debug("Executing shutdown command: %s", cmd)
        subprocess.call(cmd)

    def hibernate(self, state):
        cmd = ['/usr/share/ovirt-guest-agent/ovirt-hibernate', state]
        logging.debug("Executing hibernate command: %s", cmd)
        subprocess.call(cmd)

class LinuxDataRetriver(DataRetriverBase):

    def __init__(self):
        try:
            pkgmgr = PkgMgr()
        except NotImplementedError:
            self.list_pkgs = lambda app_list: []
        else:
            self.list_pkgs = pkgmgr.list_pkgs
        try:
            nicmgr = NicMgr()
        except NotImplementedError:
            self.list_nics = lambda: []
        else:
            self.list_nics = nicmgr.list_nics
        self.app_list = ""
        self.ignored_fs = ""
        self._init_vmstat()
        DataRetriverBase.__init__(self)

    def getMachineName(self):
        return socket.getfqdn()

    def getOsVersion(self):
        return os.uname()[2]

    def getAllNetworkInterfaces(self):
        return self.list_nics()

    def getApplications(self):
        return self.list_pkgs(self.app_list)

    def getAvailableRAM(self):
        free = 0
        for line in open('/proc/meminfo'):
            var, value = line.strip().split()[0:2]
            if var in ('MemFree:', 'Buffers:', 'Cached:'):
                free += long(value)
        return str(free / 1024)

    def getUsers(self):
        users = ''
        try:
            cmdline = '/usr/bin/users | /usr/bin/tr " " "\n" | /usr/bin/uniq'
            users = ' '.join(os.popen(cmdline).read().split())
        except:
            logging.exception("Error retrieving logged in users.")
        return users

    def getActiveUser(self):
        users = os.popen('/usr/bin/users').read().split()
        try:
            user = users[0]
        except:
            user = 'None'
        return user

    def getDisksUsage(self):
        usages = list()
        try:
            mounts = open('/proc/mounts')
            for mount in mounts:
                (device, path, fs) = mount.split()[:3]
                if not fs in self.ignored_fs:
                    # path might include spaces.
                    path = path.decode("string-escape")
                    statvfs = os.statvfs(path)
                    total = statvfs.f_bsize * statvfs.f_blocks
                    used = total - statvfs.f_bsize * statvfs.f_bfree
                    usages.append({'path': path, 'fs': fs, 'total': total,
                                   'used': used})
            mounts.close()
        except:
            logging.exception("Error retrieving disks usages.")
        return usages

    def getMemoryStats(self):
        try:
            self._get_meminfo()
            self._get_vmstat()
        except:
            logging.exception("Error retrieving memory stats.")
        return self.memStats

    def _init_vmstat(self):
        self.vmstat = {}
        self.vmstat['timestamp_prev'] = time.time()
        fields = ['swap_in', 'swap_out', 'pageflt', 'majflt']
        for field in fields:
            self.vmstat[field + '_prev'] = None
            self.vmstat[field + '_cur'] = None

    def _get_meminfo(self):
        fields = {'MemTotal:' : 0, 'MemFree:' : 0, 'Buffers:' : 0, \
                  'Cached:' : 0}
        free = 0
        for line in open('/proc/meminfo'):
            (key, value) = line.strip().split()[0:2]
            if key in fields.keys():
                fields[key] = int(value)
            if key in ('MemFree:', 'Buffers:', 'Cached:'):
                free += int(value)
        self.memStats['mem_total'] = fields['MemTotal:']
        self.memStats['mem_unused'] = fields['MemFree:']
        self.memStats['mem_free'] = free

    def _get_vmstat(self):
        """
        /proc/vmstat reports cumulative statistics so we must subtract the
        previous values to get the difference since the last collection.
        """
        fields = {'pswpin': 'swap_in', 'pswpout': 'swap_out',
                  'pgfault': 'pageflt', 'pgmajfault': 'majflt'}

        self.vmstat['timestamp_cur'] = time.time()
        interval = self.vmstat['timestamp_cur'] - self.vmstat['timestamp_prev']
        self.vmstat['timestamp_prev'] = self.vmstat['timestamp_cur']

        for line in open('/proc/vmstat'):
            (key, value) = line.strip().split()[0:2]
            if key in fields.keys():
                name = fields[key]
                self.vmstat[name + '_prev'] = self.vmstat[name + '_cur']
                self.vmstat[name + '_cur'] = int(value)
                if self.vmstat[name + '_prev'] is None:
                    self.vmstat[name + '_prev'] = self.vmstat[name + '_cur']
                self.memStats[name] = int((self.vmstat[name + '_cur'] -
                                           self.vmstat[name + '_prev']) /
                                          interval)

class LinuxVdsAgent(AgentLogicBase):

    def __init__(self, config):
        AgentLogicBase.__init__(self, config)
        self.dr = LinuxDataRetriver()
        self.dr.app_list = config.get("general", "applications_list")
        self.dr.ignored_fs = set(config.get("general", "ignored_fs").split())
        self.commandHandler = CommandHandlerLinux(self)
        self.cred_server = CredServer()

    def run(self):
        self.cred_server.start()
        AgentLogicBase.run(self)

    def stop(self):
        self.cred_server.join()
        AgentLogicBase.stop(self)

def test():
    dr = LinuxDataRetriver()
    dr.app_list = "kernel kernel-headers aspell"
    dr.ignored_fs = set("rootfs tmpfs autofs cgroup selinuxfs udev mqueue "
                        "nfsd proc sysfs devtmpfs hugetlbfs rpc_pipefs devpts "
                        "securityfs debugfs binfmt_misc".split())
    print "Machine Name:", dr.getMachineName()
    print "OS Version:", dr.getOsVersion()
    print "Network Interfaces:", dr.getAllNetworkInterfaces()
    print "Installed Applications:", dr.getApplications()
    print "Available RAM:", dr.getAvailableRAM()
    print "Logged in Users:", dr.getUsers()
    print "Active User:", dr.getActiveUser()
    print "Disks Usage:", dr.getDisksUsage()
    print "Memory Stats:", dr.getMemoryStats()

if __name__ == '__main__':
    test()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from moebot import MwApi
import json
import re
import datetime
from shutil import copyfile
import requests
import schedule
import time
import argparse
import sys
import traceback
import copy
import yaml

config = yaml.load(open('../config.yaml', 'r'))

def extract(mw, title):
    rep = mw.pageid(title, convert_titles='zh-cn', redirect=True)
    if not rep['success']:
        print('Mediawiki API 发生错误')
        return ''
    pageid = list(rep['contents'].keys())[0]
    if int(pageid) == -1:
        print('页面【{}】不存在'.format(title))
        return ''
    rep = mw.content(pid=pageid)
    if not rep['success']:
        print('Mediawiki API 发生错误')
        return ''
    content = rep['contents'][pageid]['revisions'][0]['*']
    return content

def post_process(content, lang='zh'):
    content = re.sub(r'{{ruby-zh\|(.*?)\|(.*?)}}', r'\1(\2)', content)
    if lang == 'zh':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?中文译文\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    elif lang == 'jp':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?日文台词\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    results = re.findall(pattern, content)
    namemap = json.load(open('../data/namemap.json', 'r'))
    post_results = []
    for item in results:
        filename, dialogue = item
        if not filename or not dialogue or len(filename.split('-')) < 2:
            continue
        dialogue = re.sub(r'<br.*?>', '', dialogue)
        dialogue = re.sub(r'<(.*)>.*?<\/\1>', '', dialogue)
        dialogue = dialogue.replace('\n', '')
        no = filename.split('-')[0].strip()
        if filename.split('-')[1] in namemap:
            mp3 = namemap[filename.split('-')[1]]
        else:
            continue
        print('{}, {}.mp3, {}'.format(no, mp3, dialogue))
        post_results.append((no, mp3, dialogue))
    return post_results

def search_ship(ships, sortno):
    for ship in ships:
        if not ship:
            continue
        if ship['sort_no'] == sortno:
            return ship

def post_process_for_season(content, lang='zh'):
    content = re.sub(r'{{ruby-zh\|(.*?)\|(.*?)}}', r'\1(\2)', content)
    if lang == 'zh':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?中文译文\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    elif lang == 'jp':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?日文台词\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    results = re.findall(pattern, content)
    post_results = []
    for item in results:
        filename, dialogue = item
        if not filename or not dialogue or len(filename.split('-')) < 2:
            continue
        dialogue = re.sub(r'<br.*?>', '', dialogue).strip()
        dialogue = re.sub(r'<(.*)>.*?<\/\1>', '', dialogue)
        no = filename.split('-')[0]
        archive_name = filename.split('-')[1]
        with open('../data/namemap.json', 'r') as f:
            namemap = json.load(f)
            pattern = re.compile(r'({})'.format('|'.join(namemap.keys())))
            voice = re.findall(pattern, filename)
            if len(voice) < 1:
                print('季节性档名异常，档名：{}'.format(filename))
                voice_no = 2
            else:
                voice_no = namemap[voice[0]]
        post_results.append((no, dialogue, voice_no))
    return post_results

def handleSeason(mw, ships, subtitles, title, lang):
    content = extract(mw, title)
    results = post_process_for_season(content, lang)
    print(len(results))
    for sortno, dialogue, voice_no in results:
        ship = search_ship(ships, int(sortno))
        print(ship['id'], ship['name'], dialogue, voice_no)
        if ship['id'] not in subtitles:
            subtitles[ship['id']] = {}
        subtitles[ship['id']][voice_no] = dialogue
        loop_count = 0
        while int(ship['after_ship_id']) > 0 and loop_count < 10:
            loop_count += 1
            ship = ships[int(ship['after_ship_id'])]
            if ship['id'] not in subtitles:
                subtitles[ship['id']] = {}
            subtitles[ship['id']][voice_no] = dialogue
    return subtitles

def generate_subtitles(results, ships, ship, subtitles_map, subtitles_distinct):
    subtitles = {}
    for no, mp3, dialogue in results:
        if no.endswith('a'):
            continue
        if int(no) == int(ship['sort_no']):
            subtitles[mp3] = dialogue
    subtitles_map[int(ship['id'])] = subtitles
    subtitles_distinct[int(ship['id'])] = copy.copy(subtitles)
    if int(ship['after_ship_id']) <= 0:
        return
    shipKai = ships[int(ship['after_ship_id'])]
    subtitlesKai = subtitles.copy()
    subtitles_distinct[int(shipKai['id'])] = {}
    print('读取【{}】的翻译'.format(shipKai['name']))
    for no, mp3, dialogue in results:
        if no.endswith('a') and int(no[:-1]) == int(ship['sort_no'])\
                or not no.endswith('a') and int(no) == int(shipKai['sort_no']):
            subtitlesKai[mp3] = dialogue
            subtitles_distinct[int(shipKai['id'])][mp3] = dialogue
    subtitles_map[int(shipKai['id'])] = subtitlesKai
    while int(shipKai['after_ship_id']) > 0 and\
            not int(shipKai['after_ship_id']) in subtitles_map:
        shipKai = ships[int(shipKai['after_ship_id'])]
        subtitlesKai = subtitlesKai.copy()
        subtitles_distinct[int(shipKai['id'])] = {}
        print('读取【{}】的翻译'.format(shipKai['name']))
        for no, mp3, dialogue in results:
            if no.endswith('a'):
                continue
            if int(no) == int(shipKai['sort_no']):
                subtitlesKai[mp3] = dialogue
                subtitles_distinct[int(shipKai['id'])][mp3] = dialogue
        subtitles_map[int(shipKai['id'])] = subtitlesKai

def main():
    mw = MwApi('https://zh.kcwiki.moe/api.php')
    mw.login(config['account']['username'], config['account']['password'])
    subtitles_map_zh = {}
    subtitles_map_jp = {}
    subtitles_distinct = {'zh': {}, 'jp': {}}
    ships = json.load(open('../data/ship.json', 'r'))
    ships = ships[:490]
    missing = []
    ext_title_map = {
        'まるゆ': '丸输',
        '武蔵': '武藏',
        'Bismarck': '俾斯麦',
        'あきつ丸': '秋津丸'
    }
    for ship in ships:
        if not ship or ship['chinese_name'].find('改') >= 0\
                or int(ship['id']) in subtitles_map_zh\
                or ship['chinese_name'].find('zwei') >= 0\
                or ship['chinese_name'].find('drei') >= 0:
            continue
        title = ship['chinese_name'].replace('黒', '黑')\
            .replace('巻', '卷').replace('満', '满')\
            .replace('穂', '穗').replace('歳', '岁')\
            .replace('叡', '睿')
        if title in ext_title_map:
            title = ext_title_map[title]
        print('正在获取【{}】页面'.format(title))
        content = extract(mw, title)
        results = post_process(content, 'zh')
        if len(results) <= 0:
            print('缺少【{}】的语音翻译'.format(title))
            missing.append(title)
            continue
        generate_subtitles(results, ships, ship,
                           subtitles_map_zh, subtitles_distinct['zh'])
        results = post_process(content, 'jp')
        generate_subtitles(results, ships, ship,
                           subtitles_map_jp, subtitles_distinct['jp'])

    handleSeason(mw, ships, subtitles_map_zh, '季节性/2016年初夏季节', 'zh')
    handleSeason(mw, ships, subtitles_map_jp, '季节性/2016年初夏季节', 'jp')
    handleSeason(mw, ships, subtitles_map_zh, '季节性/2016年盛夏季节', 'zh')
    handleSeason(mw, ships, subtitles_map_jp, '季节性/2016年盛夏季节', 'jp')

    suffix = ''
    now = datetime.datetime.now().strftime('%Y%m%d%H') + suffix
    subtitles_map_zh['version'] = now
    subtitles_map_jp['version'] = now
    deployFilename = '{}.json'.format(now)
    json.dump(subtitles_map_zh, open('../data/subtitles.json', 'w'))
    json.dump(subtitles_map_jp, open('../data/subtitlesJP.json', 'w'))
    json.dump(subtitles_map_zh, open('../data/' + deployFilename, 'w'))
    json.dump(subtitles_distinct, open('../data/subtitles_distinct.json', 'w'))
    json.dump(missing, open('../data/missing.json', 'w'))
    return (deployFilename, 'subtitlesJP.json')

def test():
    mw = MwApi('http://zh.kcwiki.moe/api.php')
    mw.login(config['account']['username'], config['account']['password'])
    title = '伊19'
    results = extract(mw, title)
    results = post_process(results)
    if len(results) <= 0:
        print('缺少【{}】的语音翻译'.format(title))

def deploy():
    try:
        env = config['env']
        # deployDir = '/var/www/api.kcwiki.moe/storage/app/subtitles/'
        deployDir = config[env]['subtitle']
        deployFilename, deployJpFilename = main()
        if not deployFilename:
            return
        copyfile('../data/' + deployFilename, deployDir + 'zh-cn/' + deployFilename)
        copyfile('../data/' + deployJpFilename, deployDir + 'jp/' + deployFilename)
        copyfile('../data/subtitles_distinct.json', deployDir + 'subtitles_distinct.json')
        # TODO git commit
        metaFile = deployDir + 'meta.json'
        meta = json.load(open(metaFile, 'r'))
        meta['latest'] = deployFilename[:-5]
        json.dump(meta, open(metaFile, 'w'))
        # Purge cache in api.kcwiki.moe
        requests.get('http://api.kcwiki.moe/purge')
    except KeyboardInterrupt as e:
        sys.exit(0)
    except Exception as e:
        traceback.print_exc()

def plan():
    schedule.every().day.at("05:00").do(deploy)
    while True:
        schedule.run_pending()
        time.sleep(1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("test", choices=['main', 'test', 'deploy', 'plan'])
    args = parser.parse_args()
    if args.test == 'main':
        main()
    elif args.test == 'test':
        test()
    elif args.test == 'deploy':
        deploy()
    elif args.test == 'plan':
        plan()
<commit_msg>fix(upadte seasonal quotes): sortno -> wikiid<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function
from __future__ import unicode_literals
from moebot import MwApi
import json
import re
import datetime
from shutil import copyfile
import requests
import schedule
import time
import argparse
import sys
import traceback
import copy
import yaml

config = yaml.load(open('../config.yaml', 'r'))

def extract(mw, title):
    rep = mw.pageid(title, convert_titles='zh-cn', redirect=True)
    if not rep['success']:
        print('Mediawiki API 发生错误')
        return ''
    pageid = list(rep['contents'].keys())[0]
    if int(pageid) == -1:
        print('页面【{}】不存在'.format(title))
        return ''
    rep = mw.content(pid=pageid)
    if not rep['success']:
        print('Mediawiki API 发生错误')
        return ''
    content = rep['contents'][pageid]['revisions'][0]['*']
    return content

def post_process(content, lang='zh'):
    content = re.sub(r'{{ruby-zh\|(.*?)\|(.*?)}}', r'\1(\2)', content)
    if lang == 'zh':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?中文译文\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    elif lang == 'jp':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?日文台词\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    results = re.findall(pattern, content)
    namemap = json.load(open('../data/namemap.json', 'r'))
    post_results = []
    for item in results:
        filename, dialogue = item
        if not filename or not dialogue or len(filename.split('-')) < 2:
            continue
        dialogue = re.sub(r'<br.*?>', '', dialogue)
        dialogue = re.sub(r'<(.*)>.*?<\/\1>', '', dialogue)
        dialogue = dialogue.replace('\n', '')
        no = filename.split('-')[0].strip()
        if filename.split('-')[1] in namemap:
            mp3 = namemap[filename.split('-')[1]]
        else:
            continue
        print('{}, {}.mp3, {}'.format(no, mp3, dialogue))
        post_results.append((no, mp3, dialogue))
    return post_results

def search_ship(ships, wiki_id):
    for ship in ships:
        if not ship:
            continue
        if ship['wiki_id'] == wiki_id:
            return ship
    print('ship({}) is not found.'.format(wiki_id))

def post_process_for_season(content, lang='zh'):
    content = re.sub(r'{{ruby-zh\|(.*?)\|(.*?)}}', r'\1(\2)', content)
    if lang == 'zh':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?中文译文\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    elif lang == 'jp':
        pattern = re.compile(r'\{\{台词翻译表(?:.|\n)*?档名\s*?=(.*?)\s*?\n(?:.|\n)*?日文台词\s*?=([^\|\{\}]*?)(?:\|(?:.|\n)*?)?\}\}',
                             re.MULTILINE)
    results = re.findall(pattern, content)
    post_results = []
    for item in results:
        filename, dialogue = item
        if not filename or not dialogue or len(filename.split('-')) < 2:
            continue
        dialogue = re.sub(r'<br.*?>', '', dialogue).strip()
        dialogue = re.sub(r'<(.*)>.*?<\/\1>', '', dialogue)
        no = filename.split('-')[0]
        archive_name = filename.split('-')[1]
        with open('../data/namemap.json', 'r') as f:
            namemap = json.load(f)
            pattern = re.compile(r'({})'.format('|'.join(namemap.keys())))
            voice = re.findall(pattern, filename)
            if len(voice) < 1:
                print('季节性档名异常，档名：{}'.format(filename))
                voice_no = 2
            else:
                voice_no = namemap[voice[0]]
        post_results.append((no, dialogue, voice_no))
    return post_results

def handleSeason(mw, ships, subtitles, title, lang):
    content = extract(mw, title)
    results = post_process_for_season(content, lang)
    print(len(results))
    for wiki_id, dialogue, voice_no in results:
        ship = search_ship(ships, wiki_id.strip())
        print(ship['id'], ship['name'], dialogue, voice_no)
        if ship['id'] not in subtitles:
            subtitles[ship['id']] = {}
        subtitles[ship['id']][voice_no] = dialogue
        loop_count = 0
        while int(ship['after_ship_id']) > 0 and loop_count < 10:
            loop_count += 1
            ship = ships[int(ship['after_ship_id'])]
            if ship['id'] not in subtitles:
                subtitles[ship['id']] = {}
            subtitles[ship['id']][voice_no] = dialogue
    return subtitles

def generate_subtitles(results, ships, ship, subtitles_map, subtitles_distinct):
    subtitles = {}
    for no, mp3, dialogue in results:
        if no.endswith('a'):
            continue
        if int(no) == int(ship['sort_no']):
            subtitles[mp3] = dialogue
    subtitles_map[int(ship['id'])] = subtitles
    subtitles_distinct[int(ship['id'])] = copy.copy(subtitles)
    if int(ship['after_ship_id']) <= 0:
        return
    shipKai = ships[int(ship['after_ship_id'])]
    subtitlesKai = subtitles.copy()
    subtitles_distinct[int(shipKai['id'])] = {}
    print('读取【{}】的翻译'.format(shipKai['name']))
    for no, mp3, dialogue in results:
        if no.endswith('a') and int(no[:-1]) == int(ship['sort_no'])\
                or not no.endswith('a') and int(no) == int(shipKai['sort_no']):
            subtitlesKai[mp3] = dialogue
            subtitles_distinct[int(shipKai['id'])][mp3] = dialogue
    subtitles_map[int(shipKai['id'])] = subtitlesKai
    while int(shipKai['after_ship_id']) > 0 and\
            not int(shipKai['after_ship_id']) in subtitles_map:
        shipKai = ships[int(shipKai['after_ship_id'])]
        subtitlesKai = subtitlesKai.copy()
        subtitles_distinct[int(shipKai['id'])] = {}
        print('读取【{}】的翻译'.format(shipKai['name']))
        for no, mp3, dialogue in results:
            if no.endswith('a'):
                continue
            if int(no) == int(shipKai['sort_no']):
                subtitlesKai[mp3] = dialogue
                subtitles_distinct[int(shipKai['id'])][mp3] = dialogue
        subtitles_map[int(shipKai['id'])] = subtitlesKai

def main():
    mw = MwApi('https://zh.kcwiki.moe/api.php')
    mw.login(config['account']['username'], config['account']['password'])
    subtitles_map_zh = {}
    subtitles_map_jp = {}
    subtitles_distinct = {'zh': {}, 'jp': {}}
    ships = json.load(open('../data/ship.json', 'r'))
    ships = ships[:490]
    missing = []
    ext_title_map = {
        'まるゆ': '丸输',
        '武蔵': '武藏',
        'Bismarck': '俾斯麦',
        'あきつ丸': '秋津丸'
    }
    for ship in ships:
        if not ship or ship['chinese_name'].find('改') >= 0\
                or int(ship['id']) in subtitles_map_zh\
                or ship['chinese_name'].find('zwei') >= 0\
                or ship['chinese_name'].find('drei') >= 0:
            continue
        title = ship['chinese_name'].replace('黒', '黑')\
            .replace('巻', '卷').replace('満', '满')\
            .replace('穂', '穗').replace('歳', '岁')\
            .replace('叡', '睿')
        if title in ext_title_map:
            title = ext_title_map[title]
        print('正在获取【{}】页面'.format(title))
        content = extract(mw, title)
        results = post_process(content, 'zh')
        if len(results) <= 0:
            print('缺少【{}】的语音翻译'.format(title))
            missing.append(title)
            continue
        generate_subtitles(results, ships, ship,
                           subtitles_map_zh, subtitles_distinct['zh'])
        results = post_process(content, 'jp')
        generate_subtitles(results, ships, ship,
                           subtitles_map_jp, subtitles_distinct['jp'])

    # handleSeason(mw, ships, subtitles_map_zh, '季节性/2016年初夏季节', 'zh')
    # handleSeason(mw, ships, subtitles_map_jp, '季节性/2016年初夏季节', 'jp')
    handleSeason(mw, ships, subtitles_map_zh, '季节性/2016年初秋季节', 'zh')
    handleSeason(mw, ships, subtitles_map_jp, '季节性/2016年初秋季节', 'jp')

    suffix = ''
    now = datetime.datetime.now().strftime('%Y%m%d%H') + suffix
    subtitles_map_zh['version'] = now
    subtitles_map_jp['version'] = now
    deployFilename = '{}.json'.format(now)
    json.dump(subtitles_map_zh, open('../data/subtitles.json', 'w'))
    json.dump(subtitles_map_jp, open('../data/subtitlesJP.json', 'w'))
    json.dump(subtitles_map_zh, open('../data/' + deployFilename, 'w'))
    json.dump(subtitles_distinct, open('../data/subtitles_distinct.json', 'w'))
    json.dump(missing, open('../data/missing.json', 'w'))
    return (deployFilename, 'subtitlesJP.json')

def test():
    mw = MwApi('http://zh.kcwiki.moe/api.php')
    mw.login(config['account']['username'], config['account']['password'])
    title = '伊19'
    results = extract(mw, title)
    results = post_process(results)
    if len(results) <= 0:
        print('缺少【{}】的语音翻译'.format(title))

def deploy():
    try:
        env = config['env']
        # deployDir = '/var/www/api.kcwiki.moe/storage/app/subtitles/'
        deployDir = config[env]['subtitle']
        deployFilename, deployJpFilename = main()
        if not deployFilename:
            return
        copyfile('../data/' + deployFilename, deployDir + 'zh-cn/' + deployFilename)
        copyfile('../data/' + deployJpFilename, deployDir + 'jp/' + deployFilename)
        copyfile('../data/subtitles_distinct.json', deployDir + 'subtitles_distinct.json')
        # TODO git commit
        metaFile = deployDir + 'meta.json'
        meta = json.load(open(metaFile, 'r'))
        meta['latest'] = deployFilename[:-5]
        json.dump(meta, open(metaFile, 'w'))
        # Purge cache in api.kcwiki.moe
        requests.get('http://api.kcwiki.moe/purge')
    except KeyboardInterrupt as e:
        sys.exit(0)
    except Exception as e:
        traceback.print_exc()

def plan():
    schedule.every().day.at("05:00").do(deploy)
    while True:
        schedule.run_pending()
        time.sleep(1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("test", choices=['main', 'test', 'deploy', 'plan'])
    args = parser.parse_args()
    if args.test == 'main':
        main()
    elif args.test == 'test':
        test()
    elif args.test == 'deploy':
        deploy()
    elif args.test == 'plan':
        plan()
<|endoftext|>