<commit_before>#!/usr/bin/python3

import argparse
import configparser
import copy
import http.client
import json
import logging
import os
import sys
import shutil
import stat
import time
import datetime

# default values which is set by preprocessor on server
default_value = {
    "docker": "$DOCKER$",
    "master": "$MASTER$",
    "secret": "$SECRET$",
    "timeout": "60"
}

class CachedValue:

    def __init__(self, loader, time_after_write):
        self.__data = None
        self.__load_time = None
        self.__loader = loader
        self.__taw = time_after_write

    def get(self, *args, **kwargs):
        t = time.time()
        if not self.__data or self.__load_time + self.__taw >= t:
            self.__data = self.__loader(*args, **kwargs)
            self.__load_time = time.time()
        return self.__data

class Docker:

    def __init__(self, address):
        self.host, self.port = address.split(':')
        self.conn = None
        self.last_req_info = None
        self.__id = None
        self.__info = CachedValue(loader=lambda: self.__send("/info"),
                                  time_after_write=600)
        self.__containers = CachedValue(loader=lambda: self.__send("/containers/json?all=1"),
                                        time_after_write=600)

    def __open(self):
        if self.conn:
            return
        self.conn = http.client.HTTPConnection(self.host, self.port)

    def __send(self, path, method='GET', data=None):
        self.last_req_info = {'method': method, 'path': path, 'data': data}
        try:
            self.__open()
            self.conn.request(method, path)
        except:
            self.conn.close()
            self.conn = None
            logging.error("Can not connect to docker %s due to error: %s", self.gather_error_info(), sys.exc_info()[1])
            raise
        resp = self.conn.getresponse()
        if resp.status != http.client.OK:
            raise Exception("Invalid response: {} {} from {}"
                            .format(resp.status, resp.reason, self.gather_error_info()))
        return json.loads(resp.read().decode('utf8'))

    def gather_error_info(self):
        info = {}
        info.update(self.last_req_info)
        info['docker'] = self.host + ":" + self.port
        return info

    def info(self):
        return copy.deepcopy(self.__info.get())

    def containers(self):
        return copy.deepcopy(self.__containers.get())

    def id(self):
        if not self.__id:
            info = self.__info.get()
            self.__id = info["ID"].replace(":", "")
        return self.__id

class DockerMaster:

    def __init__(self, address, docker, timeout, secret):
        self.host, self.port = address.split(':')
        self.docker = docker
        self.timeout = timeout
        self.secret = secret
        self.conn = None
        self.open()

    def open(self):
        if self.conn:
            return
        self.conn = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout*20)

    def process_container(self, c):
        # name is looked like /nodename/containername or /containername and we need to correct it
        name_src = c['Names'][0].split('/')
        name = name_src[len(name_src) - 1]
        return {
            'id': c['Id'],
            'name': name,
            'image': c['Image'],
            'labels': c['Labels']
        }

    def get_status(self):
        status = {
            # cpu load 1.0 - 100%, .5 - 50% and etc (float)
            'cpuLoad': 0.0,
            # memory in bytes (float)
            'memory': {
                'total': 0.0,
                'available': 0.0,
                'used': 0.0
            },
            # disk usage for each mount point, in bytes, (float)
            'disks': {
                '/': {'total': 0.0, 'used': 0.0},
                '/home': {'total': 0.0, 'used': 0.0},
            },
            'net': {
            }
        }
        try:
            import psutil
            status['cpuLoad'] = psutil.cpu_percent()
            mem = psutil.virtual_memory()
            status['memory'] = {
                'total': mem.total,
                'available': mem.available,
                'used': mem.used
            }
            disks = {}
            for part in psutil.disk_partitions():
                usage = psutil.disk_usage(part.mountpoint)
                disks[part.mountpoint] = {'total': usage.total, 'used': usage.used}
            status['disks'] = disks
            net = {}
            for id, nic in psutil.net_io_counters(pernic=True).items():
                if id == 'lo':
                    continue
                net[id] = {'bytesOut': nic.bytes_sent, 'bytesIn': nic.bytes_recv}
            status['net'] = net
        except:
            logging.warning("status will be empty because 'psutil' package is not found.")
            pass
        return status

    def update(self):
        try:
            docker = self.docker
            info = docker.info()
            name = info['Name']
            id = name  # we use name as id, perhaps we should use full domain name
            path = '/discovery/nodes/' + id
            path += "?ttl=" + str(self.timeout)

            labels_arr = info['Labels']
            data = {
                'time': datetime.datetime.now().isoformat(),
                'id': id,
                'name': name,
                'address': docker.host + ":" + docker.port,
                'containers': list(map(self.process_container, docker.containers())),
                'labels': dict(map(lambda a: a.split('='), labels_arr)) if labels_arr else {},
                'system': self.get_status()
            }

            json_data = json.dumps(data)
            logging.debug("do registration with %s", json_data)

            def do_register():
                try:
                    # docker master close connection after each query, we must try with keep alive
                    self.open()
                    headers = {
                        'Content-Type': 'application/json'
                    }
                    if self.secret is not None:
                        headers['X-Auth-Node'] = self.secret
                    self.conn.request('POST', path, json_data, headers)
                    r = self.conn.getresponse()
                    rd = r.read()
                    if r.status != http.client.OK:
                        if r.status == http.client.UNAUTHORIZED:
                            raise Exception("Server http://{}:{}{}, require authorization, specify correct 'secret'. \n\n {}"
                                            .format(self.host, self.port, path,
                                                    rd))
                        else:
                            raise Exception("Invalid response: {} {} from http://{}:{}{} \n\n {}"
                                            .format(r.status,
                                                    r.reason,
                                                    self.host, self.port, path,
                                                    rd))
                    r.close()
                    logging.debug("update registration success")
                except:
                    self.conn.close()
                    self.conn = None
                    raise
            tries = 3
            while tries > 0:
                try:
                    tries -= 1
                    do_register()
                    break
                except Exception as e:
                    # we cannot find cause of BrokenPipeError and
                    # therefore need to immediately repeat at this error
                    if e is BrokenPipeError: # in python 3.4 no this exception or e is http.client.RemoteDisconnected:
                        continue
                    else:
                        logging.exception("Cannot update registration due to %s", e)
                        break
        except Exception as e:
            logging.exception("Cannot update registration due to %s", e)

class Bootstrap:
    def __init__(self):
        this_name = os.path.basename(__file__)
        self.this_name = (this_name[0:-len('.py')] if this_name.endswith('.py') else this_name)

        config_name = self.this_name + '.ini'
        self.config_path_global = '/etc/' + config_name

        config_home = os.getenv('XDG_CONFIG_HOME')
        if config_home is None:
            config_home = os.path.expanduser('~/.config/')
        cwd = os.getcwd()
        config_files = [cwd + '/' + config_name, config_home + config_name, self.config_path_global]

        self.sample_config = '''[main]
docker = 172.31.0.12:2375
master = 172.31.0.3:8762
timeout = 10
secret = secr3t
log_level = 2'''

        parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                         description='DockMaster node agent.',
                                         epilog='''Example:
  dockmaster-agent.py -d 172.31.0.12:2375 -m 172.31.0.3:8763 -t 2 -vv
Sample config:''' + self.sample_config + 'By default find config in:\n\t' + '\n\t'.join(config_files))
        parser.add_argument('command', choices=['daemon', 'install'], nargs='?', default='daemon',
                            help='''daemon - run agent daemon (it default)
install - this into OS startup scripts''')
        parser.add_argument('-d', '--docker', dest='docker', action='store',
                            help='ip and port of docker service')
        parser.add_argument('-m', '--master', dest='master', action='store',
                            help='ip and port of dockmaster service')
        parser.add_argument('-s', '--secret', dest='secret', action='store',
                            help='secret for auth on master server')
        parser.add_argument('-t', '--timeout', dest='timeout', action='store',
                            type=int,
                            help='timeout in sec between node registration updates')
        parser.add_argument('-v', '--verbose', dest='log_level', action='count',
                            help='logging level, -v is INFO, -vv is DEBUG')
        parser.add_argument('-f', '--config', dest='config', action='store',
                            help='path to config file')
        args = parser.parse_args()

        config = configparser.ConfigParser()
        if args.config is not None:
            config_files.append(args.config)
        # for backward capability
        config_files.extend([cwd + '/dm-agent.ini', config_home + 'dm-agent.ini', '/etc/dm-agent.ini'])
        self._readed_files = config.read(config_files, encoding='utf8')
        cm = config['main'] if 'main' in config else None

        def get(name, converter=None):
            res = getattr(args, name, None)
            if res is None and cm is not None:
                res = cm.get(name)
            if res is None:
                res = default_value.get(name)
                # when default value is not set, we ignore it
                if res is not None and len(res) > 2 and res[0] == '$' and res[-1] == '$':
                    res = None
            if res is not None and callable(converter):
                res = converter(res)
            return res

        self.log_level = get('log_level', int)
        self.docker = get('docker')
        self.timeout = get('timeout', int)
        self.master = get('master')
        self.secret = get('secret')
        self.command = get('command')
        if self.command == 'daemon':
            # also do another args validation here
            self.check_address(self.docker)
            self.check_address(self.master)

    def check_address(self, address):
        if address is None or (address.find(':') <= 0):
            raise ValueError("Address must be non null and has format: 'host:port', but: " + str(address))

    def print_config(self):
        args = {}
        for k in ['log_level', 'docker', 'timeout', 'master', 'secret']:
            args[k] = self.__dict__.get(k)
        logging.info('Configs: %s\nArguments: %s', ', '.join(self._readed_files), args)

def install(bs):
    script_path = __file__
    script_name = os.path.basename(script_path)
    bin_dir = '/usr/local/bin/'
    if not script_path.startswith(bin_dir):
        script_path = bin_dir + script_name
        logging.warning('Copy script to %s', script_path)
        shutil.copyfile(__file__, script_path)

    os.chmod(script_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH)
    service_name = bs.this_name + '.service'
    service_path = '/etc/systemd/system/' + service_name
    service_str = '''
[Unit]
Description=Haven node agent.
Documentation=https://github.com/codeabovelab/haven-platform
After=network.target docker.service

[Service]
Type=simple
ExecStart={} -v

[Install]
WantedBy=multi-user.target
'''.format(script_path)
    with open(service_path, 'w') as file:
        file.write(service_str)
    if not os.path.exists(bs.config_path_global):
        with open(bs.config_path_global, 'w') as file:
            file.write(bs.sample_config.replace("\n", "\n#"))
    logging.warning('Script at %s,\n\t unit %s at %s \n\t config at %s',
                    script_path, service_name, service_path, bs.config_path_global)

    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'daemon-reload')
    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'enable', service_name)
    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'start', service_name)
    logging.warning('Done')

def uninstall():
    # bin_dir = '/usr/local/bin/'
    # service_path = '/etc/systemd/system/' + service_name
    #
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'stop', service_name)
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'disable', service_name)
    # TODO remove service, config & script
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'daemon-reload')
    logging.warning('Done')

def main():
    bs = Bootstrap()

    logging.basicConfig(level=logging.INFO if bs.log_level == 1 else logging.DEBUG if bs.log_level == 2 else logging.WARNING,
                        format='%(asctime)s - %(levelname)s - %(message)s')
    bs.print_config()

    if bs.command == 'install':
        install(bs)
        return

    docker = Docker(bs.docker)

    ttl = bs.timeout * 2  # ttl is a least two of update interval in seconds
    master = DockerMaster(bs.master, docker, timeout=ttl, secret=bs.secret)
    while True:
        master.update()
        time.sleep(bs.timeout)

if __name__ == '__main__':
    main()
<commit_msg>node-agent.py polish doc<commit_after>#!/usr/bin/python3

import argparse
import configparser
import copy
import http.client
import json
import logging
import os
import sys
import shutil
import stat
import time
import datetime

# default values which is set by preprocessor on server
default_value = {
    "docker": "$DOCKER$",
    "master": "$MASTER$",
    "secret": "$SECRET$",
    "timeout": "60"
}

class CachedValue:

    def __init__(self, loader, time_after_write):
        self.__data = None
        self.__load_time = None
        self.__loader = loader
        self.__taw = time_after_write

    def get(self, *args, **kwargs):
        t = time.time()
        if not self.__data or self.__load_time + self.__taw >= t:
            self.__data = self.__loader(*args, **kwargs)
            self.__load_time = time.time()
        return self.__data

class Docker:

    def __init__(self, address):
        self.host, self.port = address.split(':')
        self.conn = None
        self.last_req_info = None
        self.__id = None
        self.__info = CachedValue(loader=lambda: self.__send("/info"),
                                  time_after_write=600)
        self.__containers = CachedValue(loader=lambda: self.__send("/containers/json?all=1"),
                                        time_after_write=600)

    def __open(self):
        if self.conn:
            return
        self.conn = http.client.HTTPConnection(self.host, self.port)

    def __send(self, path, method='GET', data=None):
        self.last_req_info = {'method': method, 'path': path, 'data': data}
        try:
            self.__open()
            self.conn.request(method, path)
        except:
            self.conn.close()
            self.conn = None
            logging.error("Can not connect to docker %s due to error: %s", self.gather_error_info(), sys.exc_info()[1])
            raise
        resp = self.conn.getresponse()
        if resp.status != http.client.OK:
            raise Exception("Invalid response: {} {} from {}"
                            .format(resp.status, resp.reason, self.gather_error_info()))
        return json.loads(resp.read().decode('utf8'))

    def gather_error_info(self):
        info = {}
        info.update(self.last_req_info)
        info['docker'] = self.host + ":" + self.port
        return info

    def info(self):
        return copy.deepcopy(self.__info.get())

    def containers(self):
        return copy.deepcopy(self.__containers.get())

    def id(self):
        if not self.__id:
            info = self.__info.get()
            self.__id = info["ID"].replace(":", "")
        return self.__id

class DockerMaster:

    def __init__(self, address, docker, timeout, secret):
        self.host, self.port = address.split(':')
        self.docker = docker
        self.timeout = timeout
        self.secret = secret
        self.conn = None
        self.open()

    def open(self):
        if self.conn:
            return
        self.conn = http.client.HTTPConnection(self.host, self.port, timeout=self.timeout*20)

    def process_container(self, c):
        # name is looked like /nodename/containername or /containername and we need to correct it
        name_src = c['Names'][0].split('/')
        name = name_src[len(name_src) - 1]
        return {
            'id': c['Id'],
            'name': name,
            'image': c['Image'],
            'labels': c['Labels']
        }

    def get_status(self):
        status = {
            # cpu load 1.0 - 100%, .5 - 50% and etc (float)
            'cpuLoad': 0.0,
            # memory in bytes (float)
            'memory': {
                'total': 0.0,
                'available': 0.0,
                'used': 0.0
            },
            # disk usage for each mount point, in bytes, (float)
            'disks': {
                '/': {'total': 0.0, 'used': 0.0},
                '/home': {'total': 0.0, 'used': 0.0},
            },
            'net': {
            }
        }
        try:
            import psutil
            status['cpuLoad'] = psutil.cpu_percent()
            mem = psutil.virtual_memory()
            status['memory'] = {
                'total': mem.total,
                'available': mem.available,
                'used': mem.used
            }
            disks = {}
            for part in psutil.disk_partitions():
                usage = psutil.disk_usage(part.mountpoint)
                disks[part.mountpoint] = {'total': usage.total, 'used': usage.used}
            status['disks'] = disks
            net = {}
            for id, nic in psutil.net_io_counters(pernic=True).items():
                if id == 'lo':
                    continue
                net[id] = {'bytesOut': nic.bytes_sent, 'bytesIn': nic.bytes_recv}
            status['net'] = net
        except:
            logging.warning("status will be empty because 'psutil' package is not found.")
            pass
        return status

    def update(self):
        try:
            docker = self.docker
            info = docker.info()
            name = info['Name']
            id = name  # we use name as id, perhaps we should use full domain name
            path = '/discovery/nodes/' + id
            path += "?ttl=" + str(self.timeout)

            labels_arr = info['Labels']
            data = {
                'time': datetime.datetime.now().isoformat(),
                'id': id,
                'name': name,
                'address': docker.host + ":" + docker.port,
                'containers': list(map(self.process_container, docker.containers())),
                'labels': dict(map(lambda a: a.split('='), labels_arr)) if labels_arr else {},
                'system': self.get_status()
            }

            json_data = json.dumps(data)
            logging.debug("do registration with %s", json_data)

            def do_register():
                try:
                    # docker master close connection after each query, we must try with keep alive
                    self.open()
                    headers = {
                        'Content-Type': 'application/json'
                    }
                    if self.secret is not None:
                        headers['X-Auth-Node'] = self.secret
                    self.conn.request('POST', path, json_data, headers)
                    r = self.conn.getresponse()
                    rd = r.read()
                    if r.status != http.client.OK:
                        if r.status == http.client.UNAUTHORIZED:
                            raise Exception("Server http://{}:{}{}, require authorization, specify correct 'secret'. \n\n {}"
                                            .format(self.host, self.port, path,
                                                    rd))
                        else:
                            raise Exception("Invalid response: {} {} from http://{}:{}{} \n\n {}"
                                            .format(r.status,
                                                    r.reason,
                                                    self.host, self.port, path,
                                                    rd))
                    r.close()
                    logging.debug("update registration success")
                except:
                    self.conn.close()
                    self.conn = None
                    raise
            tries = 3
            while tries > 0:
                try:
                    tries -= 1
                    do_register()
                    break
                except Exception as e:
                    # we cannot find cause of BrokenPipeError and
                    # therefore need to immediately repeat at this error
                    if e is BrokenPipeError: # in python 3.4 no this exception or e is http.client.RemoteDisconnected:
                        continue
                    else:
                        logging.exception("Cannot update registration due to %s", e)
                        break
        except Exception as e:
            logging.exception("Cannot update registration due to %s", e)

class Bootstrap:
    def __init__(self):
        this_name = os.path.basename(__file__)
        self.this_name = (this_name[0:-len('.py')] if this_name.endswith('.py') else this_name)

        config_name = self.this_name + '.ini'
        self.config_path_global = '/etc/' + config_name

        config_home = os.getenv('XDG_CONFIG_HOME')
        if config_home is None:
            config_home = os.path.expanduser('~/.config/')
        cwd = os.getcwd()
        config_files = [cwd + '/' + config_name, config_home + config_name, self.config_path_global]

        self.sample_config = '''[main]
 docker = 172.31.0.12:2375
 master = 172.31.0.3:8762
 timeout = 10
 secret = secr3t
 log_level = 2'''

        parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter,
                                         description='DockMaster node agent.',
                                         epilog='''Example:
  ''' + this_name + ''' -d 172.31.0.12:2375 -m 172.31.0.3:8763 -t 2 -vv
Sample config:

''' + self.sample_config + '\n\nBy default find config in:\n\t' + '\n\t'.join(config_files))
        parser.add_argument('command', choices=['daemon', 'install'], nargs='?', default='daemon',
                            help='''daemon - run agent daemon (it default)
install - this into OS startup scripts''')
        parser.add_argument('-d', '--docker', dest='docker', action='store',
                            help='ip and port of docker service')
        parser.add_argument('-m', '--master', dest='master', action='store',
                            help='ip and port of dockmaster service')
        parser.add_argument('-s', '--secret', dest='secret', action='store',
                            help='secret for auth on master server')
        parser.add_argument('-t', '--timeout', dest='timeout', action='store',
                            type=int,
                            help='timeout in sec between node registration updates')
        parser.add_argument('-v', '--verbose', dest='log_level', action='count',
                            help='logging level, -v is INFO, -vv is DEBUG')
        parser.add_argument('-f', '--config', dest='config', action='store',
                            help='path to config file')
        args = parser.parse_args()

        config = configparser.ConfigParser()
        if args.config is not None:
            config_files.append(args.config)
        # for backward capability
        config_files.extend([cwd + '/dm-agent.ini', config_home + 'dm-agent.ini', '/etc/dm-agent.ini'])
        self._readed_files = config.read(config_files, encoding='utf8')
        cm = config['main'] if 'main' in config else None

        def get(name, converter=None):
            res = getattr(args, name, None)
            if res is None and cm is not None:
                res = cm.get(name)
            if res is None:
                res = default_value.get(name)
                # when default value is not set, we ignore it
                if res is not None and len(res) > 2 and res[0] == '$' and res[-1] == '$':
                    res = None
            if res is not None and callable(converter):
                res = converter(res)
            return res

        self.log_level = get('log_level', int)
        self.docker = get('docker')
        self.timeout = get('timeout', int)
        self.master = get('master')
        self.secret = get('secret')
        self.command = get('command')
        if self.command == 'daemon':
            # also do another args validation here
            self.check_address(self.docker)
            self.check_address(self.master)

    def check_address(self, address):
        if address is None or (address.find(':') <= 0):
            raise ValueError("Address must be non null and has format: 'host:port', but: " + str(address))

    def print_config(self):
        args = {}
        for k in ['log_level', 'docker', 'timeout', 'master', 'secret']:
            args[k] = self.__dict__.get(k)
        logging.info('Configs: %s\nArguments: %s', ', '.join(self._readed_files), args)

def install(bs):
    script_path = __file__
    script_name = os.path.basename(script_path)
    bin_dir = '/usr/local/bin/'
    if not script_path.startswith(bin_dir):
        script_path = bin_dir + script_name
        logging.warning('Copy script to %s', script_path)
        shutil.copyfile(__file__, script_path)

    os.chmod(script_path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH)
    service_name = bs.this_name + '.service'
    service_path = '/etc/systemd/system/' + service_name
    service_str = '''
[Unit]
Description=Haven node agent.
Documentation=https://github.com/codeabovelab/haven-platform
After=network.target docker.service

[Service]
Type=simple
ExecStart={} -v

[Install]
WantedBy=multi-user.target
'''.format(script_path)
    with open(service_path, 'w') as file:
        file.write(service_str)
    if not os.path.exists(bs.config_path_global):
        with open(bs.config_path_global, 'w') as file:
            file.write(bs.sample_config.replace("\n", "\n#"))
    logging.warning('Script at %s,\n\t unit %s at %s \n\t config at %s',
                    script_path, service_name, service_path, bs.config_path_global)

    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'daemon-reload')
    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'enable', service_name)
    os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'start', service_name)
    logging.warning('Done')

def uninstall():
    # bin_dir = '/usr/local/bin/'
    # service_path = '/etc/systemd/system/' + service_name
    #
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'stop', service_name)
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'disable', service_name)
    # TODO remove service, config & script
    # os.spawnl(os.P_WAIT, '/bin/systemctl', '--system', 'daemon-reload')
    logging.warning('Done')

def main():
    bs = Bootstrap()

    logging.basicConfig(level=logging.INFO if bs.log_level == 1 else logging.DEBUG if bs.log_level == 2 else logging.WARNING,
                        format='%(asctime)s - %(levelname)s - %(message)s')
    bs.print_config()

    if bs.command == 'install':
        install(bs)
        return

    docker = Docker(bs.docker)

    ttl = bs.timeout * 2  # ttl is a least two of update interval in seconds
    master = DockerMaster(bs.master, docker, timeout=ttl, secret=bs.secret)
    while True:
        master.update()
        time.sleep(bs.timeout)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# polling_observer.py: Generic polling observer implementation.
#
# Copyright (C) 2010 Gora Khargosh <gora.khargosh@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from os.path import realpath, abspath, isdir as path_isdir
from Queue import Queue, Empty as QueueEmpty
from threading import Thread, Event as ThreadedEvent

from dirsnapshot import DirectorySnapshot
from decorator_utils import synchronized
from events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

#import logging
#logging.basicConfig(level=logging.DEBUG)

class _PollingEventEmitter(Thread):
    """Daemon thread that monitors a given path recursively and emits
    file system events.
    """
    def __init__(self, path, interval=1, out_event_queue=None, name=None):
        """Monitors a given path and appends file system modification
        events to the output queue."""
        Thread.__init__(self)
        self.interval = interval
        self.out_event_queue = out_event_queue
        self.stopped = ThreadedEvent()
        self.snapshot = None
        self.path = path
        if name is None:
            self.name = '%s(%s)' % (self.__class__.__name__, realpath(abspath(self.path)))
        else:
            self.name = name
        self.setDaemon(True)

    def stop(self):
        """Stops monitoring the given path by setting a flag to stop."""
        self.stopped.set()

    @synchronized()
    def _get_directory_snapshot_diff(self):
        """Obtains a diff of two directory snapshots."""
        if self.snapshot is None:
            self.snapshot = DirectorySnapshot(self.path)
            diff = None
        else:
            new_snapshot = DirectorySnapshot(self.path)
            diff = new_snapshot - self.snapshot
            self.snapshot = new_snapshot
        return diff

    def run(self):
        """
        Appends events to the output event queue
        based on the diff between two states of the same directory.

        """
        while not self.stopped.is_set():
            self.stopped.wait(self.interval)
            diff = self._get_directory_snapshot_diff()
            if diff and self.out_event_queue:
                q = self.out_event_queue

                for path in diff.files_deleted:
                    q.put((self.path, FileDeletedEvent(path)))

                for path in diff.files_modified:
                    q.put((self.path, FileModifiedEvent(path)))

                for path in diff.files_created:
                    q.put((self.path, FileCreatedEvent(path)))

                for path, new_path in diff.files_moved.items():
                    q.put((self.path, FileMovedEvent(path, new_path)))

                for path in diff.dirs_modified:
                    q.put((self.path, DirModifiedEvent(path)))

                for path in diff.dirs_deleted:
                    q.put((self.path, DirDeletedEvent(path)))

                for path in diff.dirs_created:
                    q.put((self.path, DirCreatedEvent(path)))

                for path, new_path in diff.dirs_moved.items():
                    q.put((self.path, DirMovedEvent(path, new_path)))

class _Rule(object):
    '''
    A rule object represents
    '''
    def __init__(self, path, event_handler, event_emitter):
        self._path = path
        self._event_handler = event_handler
        self._event_emitter = event_emitter

    @property
    def path(self):
        return self._path

    @property
    def event_handler(self):
        return self._event_handler

    @property
    def event_emitter(self):
        return self._event_emitter

class PollingObserver(Thread):
    """Observer daemon thread that spawns threads for each path to be monitored.
    """
    def __init__(self, interval=1, *args, **kwargs):
        Thread.__init__(self)
        self.interval = interval
        self.event_queue = Queue()
        self.event_emitters = set()
        self.rules = {}
        self.map_name_to_paths = {}
        self.setDaemon(True)
        self.stopped = ThreadedEvent()

    def _create_event_emitter(self, path):
        return _PollingEventEmitter(path=path, interval=self.interval, out_event_queue=self.event_queue)

    @synchronized()
    def schedule(self, name, event_handler, *paths):
        """Schedules monitoring specified paths and calls methods in the
        given callback handler based on events occurring in the file system.
        """
        self.map_name_to_paths[name] = set()
        for path in paths:
            if not isinstance(path, str):
                raise TypeError(
                    "Path must be string, not '%s'." % type(path).__name__)
            self._schedule_path(name, event_handler, path)

    @synchronized()
    def _schedule_path(self, name, event_handler, path):
        """Starts monitoring the given path for file system events."""
        if path_isdir(path) and not path in self.rules:
            event_emitter = self._create_event_emitter(path=path)
            self.event_emitters.add(event_emitter)
            self.rules[path] = _Rule(path=path,
                                    event_handler=event_handler,
                                    event_emitter=event_emitter)
            self.map_name_to_paths[name].add(path)
            event_emitter.start()

    @synchronized()
    def unschedule(self, *names):
        """Stops monitoring specified paths for file system events."""
        for name in names:
            for path in self.map_name_to_paths[name]:
                self._unschedule_path(path)
            del self.map_name_to_paths[name]

    @synchronized()
    def _unschedule_path(self, path):
        """Stops watching a given path if already being monitored."""
        if path in self.rules:
            rule = self.rules.pop(path)
            rule.event_emitter.stop()
            self.event_emitters.remove(rule.event_emitter)

    def run(self):
        """Dispatches events from the event queue to the callback handler."""
        while not self.stopped.is_set():
            #logging.debug('runloop')
            try:
                (rule_path, event) = self.event_queue.get(block=True, timeout=self.interval)
                rule = self.rules[rule_path]
                rule.event_handler.dispatch(event)
                self.event_queue.task_done()
            except QueueEmpty:
                #logging.debug('queue empty')
                continue

    def stop(self):
        """Stops all monitoring."""
        self.stopped.set()
        for event_emitter in self.event_emitters:
            event_emitter.stop()

if __name__ == '__main__':
    import time
    import sys
    from os.path import abspath, realpath, dirname
    from events import LoggingFileSystemEventHandler

    o = PollingObserver()
    event_handler = LoggingFileSystemEventHandler()
    o.schedule('arguments', event_handler, *sys.argv[1:])
    o.start()
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        o.unschedule('arguments')
        o.stop()
    o.join()
<commit_msg>Remove unnecessary import from polling_observer.py<commit_after># -*- coding: utf-8 -*-
# polling_observer.py: Generic polling observer implementation.
#
# Copyright (C) 2010 Gora Khargosh <gora.khargosh@gmail.com>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from os.path import realpath, abspath, isdir as path_isdir
from Queue import Queue, Empty as QueueEmpty
from threading import Thread, Event as ThreadedEvent

from dirsnapshot import DirectorySnapshot
from decorator_utils import synchronized
from events import DirMovedEvent, DirDeletedEvent, DirCreatedEvent, DirModifiedEvent, \
    FileMovedEvent, FileDeletedEvent, FileCreatedEvent, FileModifiedEvent

#import logging
#logging.basicConfig(level=logging.DEBUG)

class _PollingEventEmitter(Thread):
    """Daemon thread that monitors a given path recursively and emits
    file system events.
    """
    def __init__(self, path, interval=1, out_event_queue=None, name=None):
        """Monitors a given path and appends file system modification
        events to the output queue."""
        Thread.__init__(self)
        self.interval = interval
        self.out_event_queue = out_event_queue
        self.stopped = ThreadedEvent()
        self.snapshot = None
        self.path = path
        if name is None:
            self.name = '%s(%s)' % (self.__class__.__name__, realpath(abspath(self.path)))
        else:
            self.name = name
        self.setDaemon(True)

    def stop(self):
        """Stops monitoring the given path by setting a flag to stop."""
        self.stopped.set()

    @synchronized()
    def _get_directory_snapshot_diff(self):
        """Obtains a diff of two directory snapshots."""
        if self.snapshot is None:
            self.snapshot = DirectorySnapshot(self.path)
            diff = None
        else:
            new_snapshot = DirectorySnapshot(self.path)
            diff = new_snapshot - self.snapshot
            self.snapshot = new_snapshot
        return diff

    def run(self):
        """
        Appends events to the output event queue
        based on the diff between two states of the same directory.

        """
        while not self.stopped.is_set():
            self.stopped.wait(self.interval)
            diff = self._get_directory_snapshot_diff()
            if diff and self.out_event_queue:
                q = self.out_event_queue

                for path in diff.files_deleted:
                    q.put((self.path, FileDeletedEvent(path)))

                for path in diff.files_modified:
                    q.put((self.path, FileModifiedEvent(path)))

                for path in diff.files_created:
                    q.put((self.path, FileCreatedEvent(path)))

                for path, new_path in diff.files_moved.items():
                    q.put((self.path, FileMovedEvent(path, new_path)))

                for path in diff.dirs_modified:
                    q.put((self.path, DirModifiedEvent(path)))

                for path in diff.dirs_deleted:
                    q.put((self.path, DirDeletedEvent(path)))

                for path in diff.dirs_created:
                    q.put((self.path, DirCreatedEvent(path)))

                for path, new_path in diff.dirs_moved.items():
                    q.put((self.path, DirMovedEvent(path, new_path)))

class _Rule(object):
    '''
    A rule object represents
    '''
    def __init__(self, path, event_handler, event_emitter):
        self._path = path
        self._event_handler = event_handler
        self._event_emitter = event_emitter

    @property
    def path(self):
        return self._path

    @property
    def event_handler(self):
        return self._event_handler

    @property
    def event_emitter(self):
        return self._event_emitter

class PollingObserver(Thread):
    """Observer daemon thread that spawns threads for each path to be monitored.
    """
    def __init__(self, interval=1, *args, **kwargs):
        Thread.__init__(self)
        self.interval = interval
        self.event_queue = Queue()
        self.event_emitters = set()
        self.rules = {}
        self.map_name_to_paths = {}
        self.setDaemon(True)
        self.stopped = ThreadedEvent()

    def _create_event_emitter(self, path):
        return _PollingEventEmitter(path=path, interval=self.interval, out_event_queue=self.event_queue)

    @synchronized()
    def schedule(self, name, event_handler, *paths):
        """Schedules monitoring specified paths and calls methods in the
        given callback handler based on events occurring in the file system.
        """
        self.map_name_to_paths[name] = set()
        for path in paths:
            if not isinstance(path, str):
                raise TypeError(
                    "Path must be string, not '%s'." % type(path).__name__)
            self._schedule_path(name, event_handler, path)

    @synchronized()
    def _schedule_path(self, name, event_handler, path):
        """Starts monitoring the given path for file system events."""
        if path_isdir(path) and not path in self.rules:
            event_emitter = self._create_event_emitter(path=path)
            self.event_emitters.add(event_emitter)
            self.rules[path] = _Rule(path=path,
                                    event_handler=event_handler,
                                    event_emitter=event_emitter)
            self.map_name_to_paths[name].add(path)
            event_emitter.start()

    @synchronized()
    def unschedule(self, *names):
        """Stops monitoring specified paths for file system events."""
        for name in names:
            for path in self.map_name_to_paths[name]:
                self._unschedule_path(path)
            del self.map_name_to_paths[name]

    @synchronized()
    def _unschedule_path(self, path):
        """Stops watching a given path if already being monitored."""
        if path in self.rules:
            rule = self.rules.pop(path)
            rule.event_emitter.stop()
            self.event_emitters.remove(rule.event_emitter)

    def run(self):
        """Dispatches events from the event queue to the callback handler."""
        while not self.stopped.is_set():
            #logging.debug('runloop')
            try:
                (rule_path, event) = self.event_queue.get(block=True, timeout=self.interval)
                rule = self.rules[rule_path]
                rule.event_handler.dispatch(event)
                self.event_queue.task_done()
            except QueueEmpty:
                #logging.debug('queue empty')
                continue

    def stop(self):
        """Stops all monitoring."""
        self.stopped.set()
        for event_emitter in self.event_emitters:
            event_emitter.stop()

if __name__ == '__main__':
    import time
    import sys
    from events import LoggingFileSystemEventHandler

    o = PollingObserver()
    event_handler = LoggingFileSystemEventHandler()
    o.schedule('arguments', event_handler, *sys.argv[1:])
    o.start()
    try:
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        o.unschedule('arguments')
        o.stop()
    o.join()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# mammon - a useless ircd
#
# Copyright (c) 2015, William Pitcock <nenolod@dereferenced.org>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

import time

from mammon.server import eventmgr_core
from mammon.capability import Capability

cap_server_time = Capability('server-time')

@eventmgr_core.handler('outbound message postprocess', priority=1)
def m_server_time(m):
    if 'server-time' in m.client.caps:
        m.tags['time'] = time.strftime("%Y-%m-%dT%H:%M:%SZ")
<commit_msg>mammon.ext.ircv3.server_time: use eventloop timestamp<commit_after>#!/usr/bin/env python
# mammon - a useless ircd
#
# Copyright (c) 2015, William Pitcock <nenolod@dereferenced.org>
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

import time

from mammon.server import eventmgr_core
from mammon.capability import Capability

cap_server_time = Capability('server-time')

@eventmgr_core.handler('outbound message postprocess', priority=1)
def m_server_time(m):
    if 'server-time' in m.client.caps:
        m.tags['time'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(m.client.ctx.eventloop.time()))
<|endoftext|>


<commit_before>
from re import compile as regex

foi = {
  "*": [".*escription$","^(.+/)?[Tt]ext$","^(.+/)?[Dd]ialog/[^/]+/[^/]+/.*[0-9]+$",
    "^(.+/)?(sub)?[tT]itle(/value)?$", "^(.+/)+caption$", "^(.+/)?label$",
    "^(.+/)?message$", "^.+Name/value$", "^.*friendlyName$", ".*senderName$",
    ".*destinations/.+[Nn]ame$", ".+[lL]abel[a-zA-Z]*/value$", "bookmarkName"],
  ".object": ["^chatOptions/.+$",],
  ".codex": ["contentPages/[0-9]+$"],
  ".matherial": [],
  ".config": ["^(.+/)?lbl[a-zA-Z]+/value$", "^labels/.+$", "^otherDeed/.+$",
    "^enclosedArea/.+$", "^tagCriteria/.+$", "^hail/.+$", "^converse/.+$",
    "^follow/.+$", "^flee/.+$", "^encorage/.+$", "^severe/.+$", "^accuse/.+$",
    "^tout/.+$", "^rent/.+$", "^alert/.+$", "^welcome/.+$", "^beacon/.+$",
    "^safe/.+$", "^helpme/.+$", "^final/.+$", "^worldTypeToDescription/.+$",
    "^planetTypeToDescription/.+$", "^starmapTypeDescription/.+$", "^.+Text(/value)?$",
    "^gui.+/value$", ".+Title$", "^paneLayout/.+/value$", "areYouSure/value$",
    "^threatLevelToText/[0-9]+$",  "^blueprintUnlock$",
    "^blueprintAlreadyKnown$", "^rotTimeDescriptions/.+/1$", "^messages/[a-zA-Z]+$",
    ".+[mM]essage$", "^defaultItemConfig/category$", "^.+/hint$",
    "^defaultPetNameLabel$", ".*descriptions/[0-9]+$", "^(un)?trackLabel$",
     "^modeTypeTextAndColor/[0-9]+/[0-9]+$"],
  "themes.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "dungeonitems.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "treats.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "weapon.config":["^[0-9]/1/[0-9]/(0|(1/)?name)$"],
  "monsters.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "hatadjectives.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "cockpit.config": [".*/displayName$"],
  "statuses.config":["^statuses/.+$"],
  "help.config": ["^[a-z]+Commands/.+$"],
  "hunger.config": ["^.*$"],
  "locations.config": [".*/name$"],
  ".species": ["^charGenTextLabels/[0-9]+$"],
  ".tech": [],
  ".cinematic": [],
  ".liquid": [],
  ".biome": [],
  ".item": [],
  ".instrument": [],
  ".legs": [],
  ".chest": [],
  ".back": [],
  ".head": [],
  ".harvestingtool": [],
  ".flashlight": [],
  ".painttool": [],
  ".wiretool": [],
  ".tillingtool": [],
  ".miningtool": [],
  ".beamaxe": [],
  ".inspectiontool": ["^outOfRangeText/.+$", "^nothingThereText/.+$"],
  ".thrownitem": [],
  ".unlock": ["^unlockMessage$"],
  ".matitem": [],
  ".liqitem": [],
  ".augment": [],
  ".consumable": [],
  ".coinitem": [],
  ".activeitem": [ "^altAbility/name$"],
  ".namesource": ["^sourceNames/[0-9]+$"],
  ".particle": [],
  ".damage": [],
  ".statuseffect": [],
  ".stagehand": ["^radioMessages/[^/]+/(0|2)$"],
  ".material": [],
  ".matmod": [],
  ".npctype": ["^scriptConfig/crew/role/(name|field)$", "^scriptConfig/crew/ranks/[0-9]+$"],
  ".mat": [],
  ".radiomessages": [],
  ".bush": [],
  ".grass": [],
  ".monstertype": ["^(.+/)?dialog/.+$"],
  ".monsterskill": ["^label$"],
  ".aimission": [".*Text$"],
  ".questtemplate": ["^.+Text(/.+/[0-9])?$", "^scriptConfig/descriptions/.+$"],
  ".tooltip": [],
  ".itemdescription": [],
  "_metadata":[]

  }
files_of_interest = dict()
for ext, poi in foi.items():
  files_of_interest[ext] = list()
  for p in poi:
    #print(p)
    files_of_interest[ext].append(regex(p))
<commit_msg>Added pronouns and objective descriptions to parsing script<commit_after>
from re import compile as regex

foi = {
  "*": [".*escription$","^(.+/)?[Tt]ext$","^(.+/)?[Dd]ialog/[^/]+/[^/]+/.*[0-9]+$",
    "^(.+/)?(sub)?[tT]itle(/value)?$", "^(.+/)+caption$", "^(.+/)?label$",
    "^(.+/)?message$", "^.+Name/value$", "^.*friendlyName$", ".*senderName$",
    ".*destinations/.+[Nn]ame$", ".+[lL]abel[a-zA-Z]*/value$", "bookmarkName"],
  ".object": ["^chatOptions/.+$",],
  ".codex": ["contentPages/[0-9]+$"],
  ".matherial": [],
  ".config": ["^(.+/)?lbl[a-zA-Z]+/value$", "^labels/.+$", "^otherDeed/.+$",
    "^enclosedArea/.+$", "^tagCriteria/.+$", "^hail/.+$", "^converse/.+$",
    "^follow/.+$", "^flee/.+$", "^encorage/.+$", "^severe/.+$", "^accuse/.+$",
    "^tout/.+$", "^rent/.+$", "^alert/.+$", "^welcome/.+$", "^beacon/.+$",
    "^safe/.+$", "^helpme/.+$", "^final/.+$", "^worldTypeToDescription/.+$",
    "^planetTypeToDescription/.+$", "^starmapTypeDescription/.+$", "^.+Text(/value)?$",
    "^gui.+/value$", ".+Title$", "^paneLayout/.+/value$", "areYouSure/value$",
    "^threatLevelToText/[0-9]+$",  "^blueprintUnlock$",
    "^blueprintAlreadyKnown$", "^rotTimeDescriptions/.+/1$", "^messages/[a-zA-Z]+$",
    ".+[mM]essage$", "^defaultItemConfig/category$", "^.+/hint$",
    "^defaultPetNameLabel$", ".*descriptions/[0-9]+$", "^(un)?trackLabel$",
     "^modeTypeTextAndColor/[0-9]+/[0-9]+$"],
  "themes.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "dungeonitems.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "treats.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "weapon.config":["^[0-9]/1/[0-9]/(0|(1/)?name)$"],
  "monsters.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "hatadjectives.config":["^[0-9]+/1/[0-9]+/(0|(1/)?name)$"],
  "cockpit.config": [".*/displayName$"],
  "statuses.config":["^statuses/.+$"],
  "help.config": ["^[a-z]+Commands/.+$"],
  "hunger.config": ["^.*$"],
  "locations.config": [".*/name$"],
  "quests.config": ["^pronouns/.+$", "^objectiveDescriptions/.+"],
  ".species": ["^charGenTextLabels/[0-9]+$"],
  ".tech": [],
  ".cinematic": [],
  ".liquid": [],
  ".biome": [],
  ".item": [],
  ".instrument": [],
  ".legs": [],
  ".chest": [],
  ".back": [],
  ".head": [],
  ".harvestingtool": [],
  ".flashlight": [],
  ".painttool": [],
  ".wiretool": [],
  ".tillingtool": [],
  ".miningtool": [],
  ".beamaxe": [],
  ".inspectiontool": ["^outOfRangeText/.+$", "^nothingThereText/.+$"],
  ".thrownitem": [],
  ".unlock": ["^unlockMessage$"],
  ".matitem": [],
  ".liqitem": [],
  ".augment": [],
  ".consumable": [],
  ".coinitem": [],
  ".activeitem": [ "^altAbility/name$"],
  ".namesource": ["^sourceNames/[0-9]+$"],
  ".particle": [],
  ".damage": [],
  ".statuseffect": [],
  ".stagehand": ["^radioMessages/[^/]+/(0|2)$"],
  ".material": [],
  ".matmod": [],
  ".npctype": ["^scriptConfig/crew/role/(name|field)$", "^scriptConfig/crew/ranks/[0-9]+$"],
  ".mat": [],
  ".radiomessages": [],
  ".bush": [],
  ".grass": [],
  ".monstertype": ["^(.+/)?dialog/.+$"],
  ".monsterskill": ["^label$"],
  ".aimission": [".*Text$"],
  ".questtemplate": ["^.+Text(/.+/[0-9])?$", "^scriptConfig/descriptions/.+$"],
  ".tooltip": [],
  ".itemdescription": [],
  "_metadata":[]

  }
files_of_interest = dict()
for ext, poi in foi.items():
  files_of_interest[ext] = list()
  for p in poi:
    #print(p)
    files_of_interest[ext].append(regex(p))
<|endoftext|>


<commit_before>import abc

class BaseMetadata(metaclass=abc.ABCMeta):

    def __init__(self, raw):
        self.raw = raw

    def serialized(self):
        return {
            'provider': self.provider,
            'kind': self.kind,
            'name': self.name,
            'path': self.path,
            'extra': self.extra,
        }

    def _impute_slashes(self, path):
        if not path.startswith('/'):
            path = '/' + path
        if self.kind == 'folder' and not path.endswith('/'):
            path += '/'
        return path

    @abc.abstractproperty
    def provider(self):
        pass

    @abc.abstractproperty
    def kind(self):
        pass

    @abc.abstractproperty
    def name(self):
        pass

    @property
    def path(self):
        return self._impute_slashes(self._path)

    @abc.abstractproperty
    def _path(self):
        pass

    @property
    def extra(self):
        return {}

class BaseFileMetadata(BaseMetadata):

    def serialized(self):
        return dict(super().serialized(), **{
            'contentType': self.content_type,
            'modified': self.modified,
            'size': self.size,
        })

    @property
    def kind(self):
        return 'file'

    @abc.abstractproperty
    def content_type(self):
        pass

    @abc.abstractproperty
    def modified(self):
        pass

    @abc.abstractproperty
    def size(self):
        pass

class BaseFileRevisionMetadata(BaseFileMetadata):

    def serialized(self):
        return dict(super().serialized(), **{
            'revision': self.revision,
        })

    @abc.abstractproperty
    def revision(self):
        pass

class BaseFolderMetadata(BaseMetadata):

    @property
    def kind(self):
        return 'folder'
<commit_msg>back to normal<commit_after>import abc

class BaseMetadata(metaclass=abc.ABCMeta):

    def __init__(self, raw):
        self.raw = raw

    def serialized(self):
        return {
            'provider': self.provider,
            'kind': self.kind,
            'name': self.name,
            'path': self.path,
            'extra': self.extra,
        }

    def impute_slashes(self, path):
        if not path.startswith('/'):
            path = '/' + path
        if self.kind == 'folder' and not path.endswith('/'):
            path += '/'
        return path

    @abc.abstractproperty
    def provider(self):
        pass

    @abc.abstractproperty
    def kind(self):
        pass

    @abc.abstractproperty
    def name(self):
        pass

    @abc.abstractproperty
    def path(self):
        pass

    @property
    def extra(self):
        return {}

class BaseFileMetadata(BaseMetadata):

    def serialized(self):
        return dict(super().serialized(), **{
            'contentType': self.content_type,
            'modified': self.modified,
            'size': self.size,
        })

    @property
    def kind(self):
        return 'file'

    @abc.abstractproperty
    def content_type(self):
        pass

    @abc.abstractproperty
    def modified(self):
        pass

    @abc.abstractproperty
    def size(self):
        pass

class BaseFileRevisionMetadata(BaseFileMetadata):

    def serialized(self):
        return dict(super().serialized(), **{
            'revision': self.revision,
        })

    @abc.abstractproperty
    def revision(self):
        pass

class BaseFolderMetadata(BaseMetadata):

    @property
    def kind(self):
        return 'folder'
<|endoftext|>


<commit_before>import abc

class BaseMetadata(metaclass=abc.ABCMeta):

    def __init__(self, raw):
        self.raw = raw

    def serialized(self):
        return {
            'provider': self.provider,
            'kind': self.kind,
            'name': self.name,
            'size': self.size,
            'path': self.path,
            'modified': self.modified,
            'extra': self.extra,
        }

    @abc.abstractproperty
    def provider(self):
        pass

    @abc.abstractproperty
    def kind(self):
        pass

    @abc.abstractproperty
    def name(self):
        pass

    @abc.abstractproperty
    def path(self):
        pass

    @abc.abstractproperty
    def modified(self):
        pass

    @abc.abstractproperty
    def size(self):
        pass

    @property
    def extra(self):
        return {}

class BaseRevision(metaclass=abc.ABCMeta):

    def __init__(self, raw):
        self.raw = raw

    def serialized(self):
        return {
            'provider': self.provider,
            'size': self.size,
            'modified': self.modified,
            'revision': self.revision,
            'extra': self.extra,
        }

    @abc.abstractproperty
    def provider(self):
        pass

    @abc.abstractproperty
    def modified(self):
        pass

    @abc.abstractproperty
    def size(self):
        pass

    @abc.abstractproperty
    def revision(self):
        pass

    @property
    def extra(self):
        return {}
<commit_msg>base file/file revision/folder metadata classes<commit_after>import abc

class BaseMetadata(metaclass=abc.ABCMeta):

    def __init__(self, raw):
        self.raw = raw

    def serialized(self):
        return {
            'provider': self.provider,
            'kind': self.kind,
            'name': self.name,
            'path': self.path,
            'extra': self.extra,
        }

    @abc.abstractproperty
    def provider(self):
        pass

    @abc.abstractproperty
    def kind(self):
        pass

    @abc.abstractproperty
    def name(self):
        pass

    @abc.abstractproperty
    def path(self):
        pass

    @property
    def extra(self):
        return {}

class BaseFileMetadata(BaseMetadata, metaclass=abc.ABCMeta):

    def serialized(self):
        return super().serialized().update({
            'content_type': self.content_type,
            'modified': self.modified,
            'size': self.size,
        })

    @property
    def kind(self):
        return 'file'

    @abc.abstractproperty
    def content_type(self):
        pass

    @abc.abstractproperty
    def modified(self):
        pass

    @abc.abstractproperty
    def size(self):
        pass

class BaseFileRevisionMetadata(BaseFileMetadata, metaclass=abc.ABCMeta):

    def serialized(self):
        return super().serialized().update({
            'revision': self.revision,
        })

    @abc.abstractproperty
    def revision(self):
        pass

class BaseFolderMetadata(BaseMetadata, metaclass=abc.ABCMeta):

    @property
    def kind(self):
        return 'folder'
<|endoftext|>


<commit_before>import os
import pandas as pd
import pandas.util.testing as tm
import numpy as np
from castra import Castra
from castra.core import _safe_mkdir
import tempfile
import pickle
import shutil
import nose.tools as nt
import unittest

A = pd.DataFrame({'x': [1, 2],
                  'y': [1., 2.]},
                 columns=['x', 'y'],
                 index=[1, 2])

B = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.]},
                 columns=['x', 'y'],
                 index=[10, 20])

C = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.],
                  'z': [0, 1]},
                 columns=['x', 'y', 'z']).set_index('z')
C.columns.name = 'cols'

class Base(unittest.TestCase):

    def setUp(self):
        self.path = tempfile.mkdtemp(prefix='castra-')

    def tearDown(self):
        shutil.rmtree(self.path)

class TestSafeMkdir(Base):

    def test_safe_mkdir_with_new(self):
        path = os.path.join(self.path, 'db')
        _safe_mkdir(path)
        nt.assert_true(os.path.exists(path))
        nt.assert_true(os.path.isdir(path))

    def test_safe_mkdir_with_existing(self):
        # an existing path should not raise an exception
        _safe_mkdir(self.path)

class TestConstructorAndContextManager(Base):

    def test_create_with_random_directory(self):
        Castra(template=A)

    def test_create_with_non_existing_path(self):
        path = os.path.join(self.path, 'db')
        Castra(path=path, template=A)

    def test_create_with_existing_path(self):
        Castra(path=self.path, template=A)

    def test_exception_with_non_dir(self):
        file_ = os.path.join(self.path, 'file')
        with open(file_, 'w') as f:
            f.write('file')
        nt.assert_raises(ValueError, Castra, path=file_)

    def test_exception_with_existing_castra_and_template(self):
        with Castra(path=self.path, template=A) as c:
            c.extend(A)
        nt.assert_raises(ValueError, Castra, path=self.path, template=A)

    def test_exception_with_empty_dir_and_no_template(self):
        nt.assert_raises(ValueError, Castra, path=self.path)

    def test_load(self):
        with Castra(path=self.path, template=A) as c:
            c.extend(A)
            c.extend(B)

        loaded = Castra(path=self.path)
        tm.assert_frame_equal(pd.concat([A, B]), loaded[:])

    def test_del_with_random_dir(self):
        c = Castra(template=A)
        assert os.path.exists(c.path)
        c.__del__()
        assert not os.path.exists(c.path)

    def test_context_manager_with_random_dir(self):
        with Castra(template=A) as c:
            assert os.path.exists(c.path)
        assert not os.path.exists(c.path)

    def test_context_manager_with_specific_dir(self):
        with Castra(path=self.path, template=A) as c:
            assert os.path.exists(c.path)
        assert os.path.exists(c.path)

def test_timeseries():
    indices = [pd.DatetimeIndex(start=str(i), end=str(i+1), freq='w')
                for i in range(2000, 2015)]
    dfs = [pd.DataFrame({'x': list(range(len(ind)))}, ind)
                for ind in indices]

    with Castra(template=dfs[0]) as c:
        for df in dfs:
            c.extend(df)
        df = c['2010-05': '2013-02']
        assert len(df) > 100

def test_Castra():
    c = Castra(template=A)
    c.extend(A)
    c.extend(B)

    assert c.columns == ['x', 'y']

    tm.assert_frame_equal(c[0:100], pd.concat([A, B]))
    tm.assert_frame_equal(c[:5], A)
    tm.assert_frame_equal(c[5:], B)

    tm.assert_frame_equal(c[2:5], A[1:])
    tm.assert_frame_equal(c[2:15], pd.concat([A[1:], B[:1]]))

def test_pickle_Castra():
    path = tempfile.mkdtemp(prefix='castra-')
    c = Castra(path=path, template=A)
    c.extend(A)
    c.extend(B)

    dumped = pickle.dumps(c)
    undumped = pickle.loads(dumped)

    tm.assert_frame_equal(pd.concat([A, B]), undumped[:])

def test_text():
    df = pd.DataFrame({'name': ['Alice', 'Bob'],
                       'balance': [100, 200]}, columns=['name', 'balance'])
    with Castra(template=df) as c:
        c.extend(df)

        tm.assert_frame_equal(c[:], df)

def test_column_access():
    with Castra(template=A) as c:
        c.extend(A)
        c.extend(B)
        df = c[:, ['x']]

        tm.assert_frame_equal(df, pd.concat([A[['x']], B[['x']]]))

        df = c[:, 'x']
        tm.assert_series_equal(df, pd.concat([A.x, B.x]))

def test_reload():
    path = tempfile.mkdtemp(prefix='castra-')
    try:
        c = Castra(template=A, path=path)
        c.extend(A)

        d = Castra(path=path)

        assert c.columns == d.columns
        assert (c.partitions == d.partitions).all()
        assert c.minimum == d.minimum
    finally:
        shutil.rmtree(path)

def test_index_dtype_matches_template():
    with Castra(template=A) as c:
        assert c.partitions.index.dtype == A.index.dtype

def test_to_dask_dataframe():
    try:
        import dask.dataframe as dd
    except ImportError:
        return

    with Castra(template=A) as c:
        c.extend(A)
        c.extend(B)

        df = c.to_dask()
        assert isinstance(df, dd.DataFrame)
        assert list(df.divisions) == [1, 2, 20]
        tm.assert_frame_equal(df.compute(), c[:])

        df = c.to_dask('x')
        assert isinstance(df, dd.Series)
        assert list(df.divisions) == [1, 2, 20]
        tm.assert_series_equal(df.compute(), c[:, 'x'])

def test_categorize():
    A = pd.DataFrame({'x': [1, 2, 3], 'y': ['A', None, 'A']},
                     columns=['x', 'y'], index=[0, 10, 20])
    B = pd.DataFrame({'x': [4, 5, 6], 'y': ['C', None, 'A']},
                     columns=['x', 'y'], index=[30, 40, 50])

    with Castra(template=A, categories=['y']) as c:
        c.extend(A)
        assert c[:].dtypes['y'] == 'category'
        assert c[:]['y'].cat.codes.dtype == np.dtype('i1')
        assert list(c[:, 'y'].cat.categories) == ['A', None]

        c.extend(B)
        assert list(c[:, 'y'].cat.categories) == ['A', None, 'C']

        assert c.load_partition(c.partitions.iloc[0], 'y').dtype == 'category'

        c.flush()

        d = Castra(path=c.path)
        tm.assert_frame_equal(c[:], d[:])

def test_save_axis_names():
    with Castra(template=C) as c:
        c.extend(C)
        assert c[:].index.name == 'z'
        assert c[:].columns.name == 'cols'
        tm.assert_frame_equal(c[:], C)
<commit_msg>Add ordering test<commit_after>import os
import pandas as pd
import pandas.util.testing as tm
import numpy as np
from castra import Castra
from castra.core import _safe_mkdir
import tempfile
import pickle
import shutil
import nose.tools as nt
import unittest

A = pd.DataFrame({'x': [1, 2],
                  'y': [1., 2.]},
                 columns=['x', 'y'],
                 index=[1, 2])

B = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.]},
                 columns=['x', 'y'],
                 index=[10, 20])

C = pd.DataFrame({'x': [10, 20],
                  'y': [10., 20.],
                  'z': [0, 1]},
                 columns=['x', 'y', 'z']).set_index('z')
C.columns.name = 'cols'

class Base(unittest.TestCase):

    def setUp(self):
        self.path = tempfile.mkdtemp(prefix='castra-')

    def tearDown(self):
        shutil.rmtree(self.path)

class TestSafeMkdir(Base):

    def test_safe_mkdir_with_new(self):
        path = os.path.join(self.path, 'db')
        _safe_mkdir(path)
        nt.assert_true(os.path.exists(path))
        nt.assert_true(os.path.isdir(path))

    def test_safe_mkdir_with_existing(self):
        # an existing path should not raise an exception
        _safe_mkdir(self.path)

class TestConstructorAndContextManager(Base):

    def test_create_with_random_directory(self):
        Castra(template=A)

    def test_create_with_non_existing_path(self):
        path = os.path.join(self.path, 'db')
        Castra(path=path, template=A)

    def test_create_with_existing_path(self):
        Castra(path=self.path, template=A)

    def test_exception_with_non_dir(self):
        file_ = os.path.join(self.path, 'file')
        with open(file_, 'w') as f:
            f.write('file')
        nt.assert_raises(ValueError, Castra, path=file_)

    def test_exception_with_existing_castra_and_template(self):
        with Castra(path=self.path, template=A) as c:
            c.extend(A)
        nt.assert_raises(ValueError, Castra, path=self.path, template=A)

    def test_exception_with_empty_dir_and_no_template(self):
        nt.assert_raises(ValueError, Castra, path=self.path)

    def test_load(self):
        with Castra(path=self.path, template=A) as c:
            c.extend(A)
            c.extend(B)

        loaded = Castra(path=self.path)
        tm.assert_frame_equal(pd.concat([A, B]), loaded[:])

    def test_del_with_random_dir(self):
        c = Castra(template=A)
        assert os.path.exists(c.path)
        c.__del__()
        assert not os.path.exists(c.path)

    def test_context_manager_with_random_dir(self):
        with Castra(template=A) as c:
            assert os.path.exists(c.path)
        assert not os.path.exists(c.path)

    def test_context_manager_with_specific_dir(self):
        with Castra(path=self.path, template=A) as c:
            assert os.path.exists(c.path)
        assert os.path.exists(c.path)

def test_timeseries():
    indices = [pd.DatetimeIndex(start=str(i), end=str(i+1), freq='w')
                for i in range(2000, 2015)]
    dfs = [pd.DataFrame({'x': list(range(len(ind)))}, ind)
                for ind in indices]

    with Castra(template=dfs[0]) as c:
        for df in dfs:
            c.extend(df)
        df = c['2010-05': '2013-02']
        assert len(df) > 100

def test_Castra():
    c = Castra(template=A)
    c.extend(A)
    c.extend(B)

    assert c.columns == ['x', 'y']

    tm.assert_frame_equal(c[0:100], pd.concat([A, B]))
    tm.assert_frame_equal(c[:5], A)
    tm.assert_frame_equal(c[5:], B)

    tm.assert_frame_equal(c[2:5], A[1:])
    tm.assert_frame_equal(c[2:15], pd.concat([A[1:], B[:1]]))

def test_pickle_Castra():
    path = tempfile.mkdtemp(prefix='castra-')
    c = Castra(path=path, template=A)
    c.extend(A)
    c.extend(B)

    dumped = pickle.dumps(c)
    undumped = pickle.loads(dumped)

    tm.assert_frame_equal(pd.concat([A, B]), undumped[:])

def test_text():
    df = pd.DataFrame({'name': ['Alice', 'Bob'],
                       'balance': [100, 200]}, columns=['name', 'balance'])
    with Castra(template=df) as c:
        c.extend(df)

        tm.assert_frame_equal(c[:], df)

def test_column_access():
    with Castra(template=A) as c:
        c.extend(A)
        c.extend(B)
        df = c[:, ['x']]

        tm.assert_frame_equal(df, pd.concat([A[['x']], B[['x']]]))

        df = c[:, 'x']
        tm.assert_series_equal(df, pd.concat([A.x, B.x]))

def test_reload():
    path = tempfile.mkdtemp(prefix='castra-')
    try:
        c = Castra(template=A, path=path)
        c.extend(A)

        d = Castra(path=path)

        assert c.columns == d.columns
        assert (c.partitions == d.partitions).all()
        assert c.minimum == d.minimum
    finally:
        shutil.rmtree(path)

def test_index_dtype_matches_template():
    with Castra(template=A) as c:
        assert c.partitions.index.dtype == A.index.dtype

def test_to_dask_dataframe():
    try:
        import dask.dataframe as dd
    except ImportError:
        return

    with Castra(template=A) as c:
        c.extend(A)
        c.extend(B)

        df = c.to_dask()
        assert isinstance(df, dd.DataFrame)
        assert list(df.divisions) == [1, 2, 20]
        tm.assert_frame_equal(df.compute(), c[:])

        df = c.to_dask('x')
        assert isinstance(df, dd.Series)
        assert list(df.divisions) == [1, 2, 20]
        tm.assert_series_equal(df.compute(), c[:, 'x'])

def test_categorize():
    A = pd.DataFrame({'x': [1, 2, 3], 'y': ['A', None, 'A']},
                     columns=['x', 'y'], index=[0, 10, 20])
    B = pd.DataFrame({'x': [4, 5, 6], 'y': ['C', None, 'A']},
                     columns=['x', 'y'], index=[30, 40, 50])

    with Castra(template=A, categories=['y']) as c:
        c.extend(A)
        assert c[:].dtypes['y'] == 'category'
        assert c[:]['y'].cat.codes.dtype == np.dtype('i1')
        assert list(c[:, 'y'].cat.categories) == ['A', None]

        c.extend(B)
        assert list(c[:, 'y'].cat.categories) == ['A', None, 'C']

        assert c.load_partition(c.partitions.iloc[0], 'y').dtype == 'category'

        c.flush()

        d = Castra(path=c.path)
        tm.assert_frame_equal(c[:], d[:])

def test_save_axis_names():
    with Castra(template=C) as c:
        c.extend(C)
        assert c[:].index.name == 'z'
        assert c[:].columns.name == 'cols'
        tm.assert_frame_equal(c[:], C)

def test_same_categories_when_already_categorized():
    A = pd.DataFrame({'x': [1, 2] * 1000,
                      'y': [1., 2.] * 1000,
                      'z': np.random.choice(list('abc'), size=2000)},
                     columns=list('xyz'))
    A['z'] = A.z.astype('category')
    with Castra(template=A, categories=['z']) as c:
        c.extend(A)
        assert c.categories['z'] == A.z.cat.categories.tolist()
<|endoftext|>