<commit_before>#!/usr/bin/python

import logging
import json
import util
import os
import neo4j_util
import argparse
import db_controller as dbc
import flask
import crypt_util
import re

from flask import Flask
from flask import Response
from flask import session
from flask import redirect
from flask import request
from flask import send_from_directory

from functools import wraps
from rz_mesh import init_ws_interface
from rz_kernel import RZ_Kernel
import rz_api
import rz_api_rest

class Config(object):
    """
    rhizi-server configuration

    TODO: config option documentation

        htpasswd_path
        listen_address
        listen_port
        neo4j_url
        root_path
    """

    @staticmethod
    def init_from_file(file_path):

        if False == os.path.exists(file_path):
            raise Exception('config file not found: ' + file_path)

        # apply defaults
        cfg = {}
        cfg['access_control'] = True
        cfg['config_dir'] = os.path.abspath(os.path.dirname(file_path))  # bypass prop restriction
        cfg['development_mode'] = False
        cfg['listen_address'] = '127.0.0.1'
        cfg['listen_port'] = 8080
        cfg['root_path'] = os.getcwd()
        cfg['static_url_path'] = '/static'

        # Flask keys
        cfg['SECRET_KEY'] = ''

        with open(file_path, 'r') as f:
            for line in f:
                if re.match('(^#)|(\s+$)', line):
                    continue

                kv_arr = line.split('=')
                if 2 != len(kv_arr):
                    raise Exception('failed to parse config line: ' + line)

                k, v = map(str.strip, kv_arr)

                if None != cfg.get(k):
                    # apply type conversion based on default value type
                    type_f = type(cfg[k])
                    if bool == type_f:
                        v = v in ("True", "true")  # workaround bool('false') = True
                    else:
                        v = type_f(v)

                # [!] we can't use k.lower() as we are loading Flask configuration
                # keys which are expected to be capitalized
                cfg[k] = v

        ret = Config()
        ret.__dict__ = cfg  # allows setting of @property attributes

        # validate config
        if False == os.path.isabs(ret.root_path):
            ret.root_path = os.path.abspath(ret.root_path)

        return ret

    def __str__(self):
        return '\n'.join('%s: %s' % (k, v) for k, v in self.__dict__.items())

    @property
    def db_base_url(self):
        return self.neo4j_url

    @property
    def tx_api_path(self):
        return '/db/data/transaction'

    @property
    def config_dir_path(self):
        return self.config_dir

    @property
    def secret_key(self):
        return self.SECRET_KEY

class FlaskExt(Flask):
    """
    Flask server customization
    """

    def __init__(self, import_name, *args, **kwargs):
        """
        reserved for future use
        """
        super(FlaskExt, self).__init__(import_name, *args, **kwargs)

    def before_request(self, *args, **kwargs):
        # TODO impl
        pass

    def make_default_options_response(self):
        ret = Flask.make_default_options_response(self)

        ret.headers['Access-Control-Allow-Origin'] = 'http://rhizi.net'
        ret.headers['Access-Control-Allow-Headers'] = "Accept, Authorization, Content-Type, Origin"
        ret.headers['Access-Control-Allow-Credentials'] = 'true'

        # ret.headers['Access-Control-Allow-Methods'] = ', '.join(m_list)
        return ret

def init_log(cfg):
    """
    init log file, location derived from configuration
    """
    log = logging.getLogger('rhizi')
    log.setLevel(logging.DEBUG)
    log_handler_c = logging.StreamHandler()
    log_handler_f = logging.FileHandler(cfg.log_path)

    log.addHandler(log_handler_c)
    log.addHandler(log_handler_f)
    return log

def init_rest_interface(cfg, flask_webapp):
    """
    Initialize REST interface
    """

    def rest_entry(path, f, flask_args={'methods': ['POST']}):
        return (path, f, flask_args)

    def redirect_entry(path, path_to, flask_args):
        def redirector():
            return redirect(path_to, code=302)
        return (path, redirector, flask_args)

    def dev_mode__resend_from_static(static_url):
        """
        redirect static links while in dev mode:
           - /res/<path> -> <path>
        """
        static_folder = flask.current_app.static_folder

        new_req_path = None
        if request.path.startswith('/res/'):
            # turn absolute/res/... URLs to static-folder relative
            new_req_path = request.path.replace('/res/', '')
        return send_from_directory(static_folder, new_req_path)

    def login_decorator(f):
        """
        [!] security boundary: asserd logged-in user before executing REST api call
        """
        @wraps(f)
        def wrapped_function(*args, **kw):
            if not 'username' in session:
                return redirect('/login')
            return f(*args, **kw)

        return wrapped_function

    rest_entry_set = [
                      redirect_entry('/', '/index', {'methods': ['GET']}),
                      rest_entry('/add/node-set' , rhizi_api.add_node_set),
                      rest_entry('/graph/clone', rhizi_api.rz_clone),
                      rest_entry('/graph/diff-commit-set', rhizi_api.diff_commit__set),
                      rest_entry('/graph/diff-commit-topo', rhizi_api.diff_commit__topo),
                      rest_entry('/graph/diff-commit-attr', rhizi_api.diff_commit__attr),
                      rest_entry('/graph/diff-commit-vis', rhizi_api.diff_commit__vis),
                      rest_entry('/index', rhizi_api.index, {'methods': ['GET']}),
                      rest_entry('/load/node-set-by-id', rhizi_api.load_node_set_by_id_attr),
                      rest_entry('/load/link-set/by_link_ptr_set', rhizi_api.load_link_set_by_link_ptr_set),
                      rest_entry('/login', rhizi_api.login, {'methods': ['GET', 'POST']}),
                      rest_entry('/logout', rhizi_api.logout, {'methods': ['GET', 'POST']}),
                      rest_entry('/match/node-set', rhizi_api.match_node_set_by_attr_filter_map),
                      rest_entry('/monitor/server-info', rhizi_api.monitor__server_info),
                  ]

    if cfg.development_mode:
        dev_path_set = ['/static', '/res']
        rest_dev_entry_set = []
        for dev_path in dev_path_set:
            rest_dev_entry_set.append(rest_entry(dev_path + '/<path:static_url>',
                                                 dev_mode__resend_from_static,
                                                 {'methods': ['GET']}))
        rest_entry_set += rest_dev_entry_set

    for re_entry in rest_entry_set:
        rest_path, f, flask_args = re_entry

        if cfg.access_control and '/login' != rest_path:
            # currently require login on all but /login paths
            f = login_decorator(f)

        # [!] order seems important - apply route decorator last
        route_dec = flask_webapp.route(rest_path, **flask_args)
        f = route_dec(f)

        flask_webapp.f = f  # assign decorated function

def init_webapp(cfg, kernel, db_ctl=None):
    """
    Initialize webapp:
       - call init_rest_interface()
    """
    root_path = cfg.root_path

    webapp = FlaskExt(__name__,
                      static_folder='static',
                      template_folder=os.path.join(root_path, 'templates'),
                      static_url_path=cfg.static_url_path)
    webapp.config.from_object(cfg)
    webapp.root_path = root_path  # for some reason calling config.from_xxx() does not have effect

    if None == db_ctl:
        db_ctl = dbc.DB_Controller(cfg)
    rz_api.db_ctl = db_ctl
    rz_api_rest.db_ctl = db_ctl

    webapp.rz_config = cfg
    webapp.kernel = kernel

    init_rest_interface(cfg, webapp)
    return webapp

def init_config(cfg_dir):
    cfg_path = os.path.join(cfg_dir, 'rhizi-server.conf')
    cfg = Config.init_from_file(cfg_path)
    return cfg

def init_pw_db(cfg, user_pw_list_file):
    pass  # TODO: stub, will move to tools/

if __name__ == "__main__":

    p = argparse.ArgumentParser(description='rhizi-server')
    p.add_argument('--config-dir', help='path to Rhizi config dir', default='res/etc')
    p.add_argument('--init-htpasswd-db', help='init login htpasswd db', action='store_const', const=True)
    p.add_argument('--htpasswd-init-file', help='htpasswd db initialization file in \'user,pw\' format')
    args = p.parse_args()

    cfg = init_config(args.config_dir)
    log = init_log(cfg)

    cfg_indent_str = '   ' + str(cfg).replace('\n', '\n   ')
    log.debug('loaded configuration:\n%s' % cfg_indent_str)  # print indented
    if False == cfg.access_control:
        log.warn('access control disabled, all-granted access set on all URLs')

    if args.init_htpasswd_db:
        init_pw_db(cfg, args.htpasswd_init_file)
        exit(0)

    kernel = RZ_Kernel()
    webapp = init_webapp(cfg, kernel)
    ws_srv = init_ws_interface(cfg, kernel, webapp)

    ws_srv.serve_forever()
<commit_msg>server: fix missing renames from rz_api_rest/rz_api and db_op/db_controller plus rhizi_server->rz_server changes<commit_after>#!/usr/bin/python

import logging
import json
import util
import os
import neo4j_util
import argparse
import db_controller as dbc
import flask
import crypt_util
import re

from flask import Flask
from flask import Response
from flask import session
from flask import redirect
from flask import request
from flask import send_from_directory

from functools import wraps
from rz_mesh import init_ws_interface
from rz_kernel import RZ_Kernel
import rz_api
import rz_api_rest

class Config(object):
    """
    rhizi-server configuration

    TODO: config option documentation

        htpasswd_path
        listen_address
        listen_port
        neo4j_url
        root_path
    """

    @staticmethod
    def init_from_file(file_path):

        if False == os.path.exists(file_path):
            raise Exception('config file not found: ' + file_path)

        # apply defaults
        cfg = {}
        cfg['access_control'] = True
        cfg['config_dir'] = os.path.abspath(os.path.dirname(file_path))  # bypass prop restriction
        cfg['development_mode'] = False
        cfg['listen_address'] = '127.0.0.1'
        cfg['listen_port'] = 8080
        cfg['root_path'] = os.getcwd()
        cfg['static_url_path'] = '/static'

        # Flask keys
        cfg['SECRET_KEY'] = ''

        with open(file_path, 'r') as f:
            for line in f:
                if re.match('(^#)|(\s+$)', line):
                    continue

                kv_arr = line.split('=')
                if 2 != len(kv_arr):
                    raise Exception('failed to parse config line: ' + line)

                k, v = map(str.strip, kv_arr)

                if None != cfg.get(k):
                    # apply type conversion based on default value type
                    type_f = type(cfg[k])
                    if bool == type_f:
                        v = v in ("True", "true")  # workaround bool('false') = True
                    else:
                        v = type_f(v)

                # [!] we can't use k.lower() as we are loading Flask configuration
                # keys which are expected to be capitalized
                cfg[k] = v

        ret = Config()
        ret.__dict__ = cfg  # allows setting of @property attributes

        # validate config
        if False == os.path.isabs(ret.root_path):
            ret.root_path = os.path.abspath(ret.root_path)

        return ret

    def __str__(self):
        return '\n'.join('%s: %s' % (k, v) for k, v in self.__dict__.items())

    @property
    def db_base_url(self):
        return self.neo4j_url

    @property
    def tx_api_path(self):
        return '/db/data/transaction'

    @property
    def config_dir_path(self):
        return self.config_dir

    @property
    def secret_key(self):
        return self.SECRET_KEY

class FlaskExt(Flask):
    """
    Flask server customization
    """

    def __init__(self, import_name, *args, **kwargs):
        """
        reserved for future use
        """
        super(FlaskExt, self).__init__(import_name, *args, **kwargs)

    def before_request(self, *args, **kwargs):
        # TODO impl
        pass

    def make_default_options_response(self):
        ret = Flask.make_default_options_response(self)

        ret.headers['Access-Control-Allow-Origin'] = 'http://rhizi.net'
        ret.headers['Access-Control-Allow-Headers'] = "Accept, Authorization, Content-Type, Origin"
        ret.headers['Access-Control-Allow-Credentials'] = 'true'

        # ret.headers['Access-Control-Allow-Methods'] = ', '.join(m_list)
        return ret

def init_log(cfg):
    """
    init log file, location derived from configuration
    """
    log = logging.getLogger('rhizi')
    log.setLevel(logging.DEBUG)
    log_handler_c = logging.StreamHandler()
    log_handler_f = logging.FileHandler(cfg.log_path)

    log.addHandler(log_handler_c)
    log.addHandler(log_handler_f)
    return log

def init_rest_interface(cfg, flask_webapp):
    """
    Initialize REST interface
    """

    def rest_entry(path, f, flask_args={'methods': ['POST']}):
        return (path, f, flask_args)

    def redirect_entry(path, path_to, flask_args):
        def redirector():
            return redirect(path_to, code=302)
        return (path, redirector, flask_args)

    def dev_mode__resend_from_static(static_url):
        """
        redirect static links while in dev mode:
           - /res/<path> -> <path>
        """
        static_folder = flask.current_app.static_folder

        new_req_path = None
        if request.path.startswith('/res/'):
            # turn absolute/res/... URLs to static-folder relative
            new_req_path = request.path.replace('/res/', '')
        return send_from_directory(static_folder, new_req_path)

    def login_decorator(f):
        """
        [!] security boundary: asserd logged-in user before executing REST api call
        """
        @wraps(f)
        def wrapped_function(*args, **kw):
            if not 'username' in session:
                return redirect('/login')
            return f(*args, **kw)

        return wrapped_function

    rest_entry_set = [
                      redirect_entry('/', '/index', {'methods': ['GET']}),
                      rest_entry('/add/node-set' , rz_api.add_node_set),
                      rest_entry('/graph/clone', rz_api.rz_clone),
                      rest_entry('/graph/diff-commit-set', rz_api.diff_commit__set),
                      rest_entry('/graph/diff-commit-topo', rz_api_rest.diff_commit__topo),
                      rest_entry('/graph/diff-commit-attr', rz_api_rest.diff_commit__attr),
                      rest_entry('/graph/diff-commit-vis', rz_api_rest.diff_commit__vis),
                      rest_entry('/index', rz_api.index, {'methods': ['GET']}),
                      rest_entry('/load/node-set-by-id', rz_api.load_node_set_by_id_attr),
                      rest_entry('/load/link-set/by_link_ptr_set', rz_api.load_link_set_by_link_ptr_set),
                      rest_entry('/login', rz_api.login, {'methods': ['GET', 'POST']}),
                      rest_entry('/logout', rz_api.logout, {'methods': ['GET', 'POST']}),
                      rest_entry('/match/node-set', rz_api.match_node_set_by_attr_filter_map),
                      rest_entry('/monitor/server-info', rz_api.monitor__server_info),
                  ]

    if cfg.development_mode:
        dev_path_set = ['/static', '/res']
        rest_dev_entry_set = []
        for dev_path in dev_path_set:
            rest_dev_entry_set.append(rest_entry(dev_path + '/<path:static_url>',
                                                 dev_mode__resend_from_static,
                                                 {'methods': ['GET']}))
        rest_entry_set += rest_dev_entry_set

    for re_entry in rest_entry_set:
        rest_path, f, flask_args = re_entry

        if cfg.access_control and '/login' != rest_path:
            # currently require login on all but /login paths
            f = login_decorator(f)

        # [!] order seems important - apply route decorator last
        route_dec = flask_webapp.route(rest_path, **flask_args)
        f = route_dec(f)

        flask_webapp.f = f  # assign decorated function

def init_webapp(cfg, kernel, db_ctl=None):
    """
    Initialize webapp:
       - call init_rest_interface()
    """
    root_path = cfg.root_path

    webapp = FlaskExt(__name__,
                      static_folder='static',
                      template_folder=os.path.join(root_path, 'templates'),
                      static_url_path=cfg.static_url_path)
    webapp.config.from_object(cfg)
    webapp.root_path = root_path  # for some reason calling config.from_xxx() does not have effect

    if None == db_ctl:
        db_ctl = dbc.DB_Controller(cfg)
    rz_api.db_ctl = db_ctl
    rz_api_rest.db_ctl = db_ctl

    webapp.rz_config = cfg
    webapp.kernel = kernel

    init_rest_interface(cfg, webapp)
    return webapp

def init_config(cfg_dir):
    cfg_path = os.path.join(cfg_dir, 'rhizi-server.conf')
    cfg = Config.init_from_file(cfg_path)
    return cfg

def init_pw_db(cfg, user_pw_list_file):
    pass  # TODO: stub, will move to tools/

if __name__ == "__main__":

    p = argparse.ArgumentParser(description='rhizi-server')
    p.add_argument('--config-dir', help='path to Rhizi config dir', default='res/etc')
    p.add_argument('--init-htpasswd-db', help='init login htpasswd db', action='store_const', const=True)
    p.add_argument('--htpasswd-init-file', help='htpasswd db initialization file in \'user,pw\' format')
    args = p.parse_args()

    cfg = init_config(args.config_dir)
    log = init_log(cfg)

    cfg_indent_str = '   ' + str(cfg).replace('\n', '\n   ')
    log.debug('loaded configuration:\n%s' % cfg_indent_str)  # print indented
    if False == cfg.access_control:
        log.warn('access control disabled, all-granted access set on all URLs')

    if args.init_htpasswd_db:
        init_pw_db(cfg, args.htpasswd_init_file)
        exit(0)

    kernel = RZ_Kernel()
    webapp = init_webapp(cfg, kernel)
    ws_srv = init_ws_interface(cfg, kernel, webapp)

    ws_srv.serve_forever()
<|endoftext|>


<commit_before>"""
SkCode tools code.
"""

import re
import unicodedata

from html import escape as escape_html
from urllib.parse import urlsplit, urlunsplit, urljoin

# URL charset regex for cleaning URL
URL_CHARSET_SUB = re.compile(r'[^a-zA-Z0-9-~+_.?#=!&;,/:%@$\|*\'()\[\]\x80-\xff]')

def escape_attribute_value(value):
    """
    Escape the given value and return it escaped and wrapped in simple or double quotes.
    Try to avoid escape sequence as much as possible to make the value more user-friendly.
    :param value: The input attribute value (unescaped).
    :return The attribute value escaped, with surrounding quotes if required.
    """
    if "'" in value and '"' in value:
        return '"%s"' % value.replace('\\', '\\\\').replace('"', '\\"')
    elif "'" in value:
        return '"%s"' % value
    elif '"' in value:
        return "'%s'" % value
    else:
        return '"%s"' % value

def sanitize_url(url, default_scheme='http',
                 allowed_schemes=('http', 'https', 'ftp', 'ftps', 'mailto'),
                 encode_html_entities=True, force_default_scheme=False,
                 force_remove_scheme=False, fix_non_local_urls=True,
                 absolute_base_url=''):
    """
    Sanitize the given URL. Avoid XSS by filtering-out forbidden protocol and characters.
    Allowed protocols by default are: ``http``, ``https``, ``ftp``, ``ftps`` and ``mailto``.
    If no protocol scheme is specified, all non-local URL will be tied to the default scheme.
    :param url: The user-supplied URL to be sanitized.
    :param default_scheme: The default scheme to use (default to ``http``).
    :param allowed_schemes: The list of allowed schemes (see defaults above).
    :param encode_html_entities: If set to ``True``, the output URL will be encoded to avoid raw HTML entities
    (default is ``True``).
    :param force_default_scheme: Set to ``True`` to force the default scheme to be used in all case
    (default is``False``).
    :param force_remove_scheme: Set to ``True`` to remove the scheme if set (default is ``False``).
    N.B. The ``force_default_scheme`` and ``force_remove_scheme`` are mutually exclusive for obvious reasons.
    :param fix_non_local_urls: Set to ``True`` to fix non local URL with netloc in path (default is ``True``).
    Example: with ``fix_non_local_urls`` set to ``True``, ``google.com`` will become ``http://google.com/``.
    :param absolute_base_url: The base URL for the relative-to-absolute conversion.
    If set, relative URLs will be expanded as absolute URLs using the given base URL. Example: with
    ``absolute_base_url`` set to ``http://example.com/``, ``/forum/`` will become ``http://example.com/forum/``.
    :return: The sanitized URL as string, or an empty string if erroneous.
    """
    assert default_scheme, "A default scheme is mandatory to avoid XSS."
    assert len(allowed_schemes) > 0, "You need to allow at least one scheme to get a result."
    assert not (force_default_scheme and
                force_remove_scheme), "You cannot force the default scheme and also force-remove the scheme."

    # Shortcut for empty string
    if not url:
        return ''

    # Remove dangerous stuff
    url = URL_CHARSET_SUB.sub('', url)

    # Split the URL
    try:
        scheme, netloc, path, query, fragment = urlsplit(url)
    except ValueError:

        # Handle malformed URL
        return ''

    # Check the scheme against the white list
    if scheme and scheme not in allowed_schemes:
        return ''

    # Detect and fix non local URL without // at beginning (not supported by  ``urlsplit``)
    if not netloc and path and not path.startswith('/') and fix_non_local_urls:
        parts = path.split('/', 1)
        if len(parts) == 2:
            netloc, path = parts
        else:
            netloc = parts[0]
            path = ''

    # Add scheme to any non-local URL if required
    if (not scheme and netloc) or force_default_scheme:
        scheme = default_scheme

    # Remove the scheme if requested
    if force_remove_scheme:
        scheme = ''

    # Build the final URL
    if netloc or not absolute_base_url:
        result = urlunsplit((scheme, netloc, path, query, fragment))
    else:
        result = urlunsplit(('', '', path, query, fragment))
        result = urljoin(absolute_base_url, result)

    # Escape HTML if requested
    if encode_html_entities:
        result = escape_html(result)

    # Return the sanitized URL
    return result

def slugify(value):
    """
    Convert the given value to a plain-text ASCII slug. Spaces are converted to hyphens.
    Characters that aren't alphanumerics, underscores, or hyphens are removed.
    The resulting string is then converted to lowercase. Leading and trailing whitespace are also stripped.
    :param value: The string to be turned into a slug.
    """
    value = value.strip()
    if not value:
        return ''
    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
    value = re.sub(r'[^\w\s-]', '', value).strip().lower()
    return re.sub(r'[-\s]+', '-', value).strip('-')
<commit_msg>Fix if-else<commit_after>"""
SkCode tools code.
"""

import re
import unicodedata

from html import escape as escape_html
from urllib.parse import urlsplit, urlunsplit, urljoin

# URL charset regex for cleaning URL
URL_CHARSET_SUB = re.compile(r'[^a-zA-Z0-9-~+_.?#=!&;,/:%@$\|*\'()\[\]\x80-\xff]')

def escape_attribute_value(value):
    """
    Escape the given value and return it escaped and wrapped in simple or double quotes.
    Try to avoid escape sequence as much as possible to make the value more user-friendly.
    :param value: The input attribute value (unescaped).
    :return The attribute value escaped, with surrounding quotes if required.
    """
    if "'" in value and '"' in value:
        return '"%s"' % value.replace('\\', '\\\\').replace('"', '\\"')
    elif '"' in value:
        return "'%s'" % value
    else:
        return '"%s"' % value

def sanitize_url(url, default_scheme='http',
                 allowed_schemes=('http', 'https', 'ftp', 'ftps', 'mailto'),
                 encode_html_entities=True, force_default_scheme=False,
                 force_remove_scheme=False, fix_non_local_urls=True,
                 absolute_base_url=''):
    """
    Sanitize the given URL. Avoid XSS by filtering-out forbidden protocol and characters.
    Allowed protocols by default are: ``http``, ``https``, ``ftp``, ``ftps`` and ``mailto``.
    If no protocol scheme is specified, all non-local URL will be tied to the default scheme.
    :param url: The user-supplied URL to be sanitized.
    :param default_scheme: The default scheme to use (default to ``http``).
    :param allowed_schemes: The list of allowed schemes (see defaults above).
    :param encode_html_entities: If set to ``True``, the output URL will be encoded to avoid raw HTML entities
    (default is ``True``).
    :param force_default_scheme: Set to ``True`` to force the default scheme to be used in all case
    (default is``False``).
    :param force_remove_scheme: Set to ``True`` to remove the scheme if set (default is ``False``).
    N.B. The ``force_default_scheme`` and ``force_remove_scheme`` are mutually exclusive for obvious reasons.
    :param fix_non_local_urls: Set to ``True`` to fix non local URL with netloc in path (default is ``True``).
    Example: with ``fix_non_local_urls`` set to ``True``, ``google.com`` will become ``http://google.com/``.
    :param absolute_base_url: The base URL for the relative-to-absolute conversion.
    If set, relative URLs will be expanded as absolute URLs using the given base URL. Example: with
    ``absolute_base_url`` set to ``http://example.com/``, ``/forum/`` will become ``http://example.com/forum/``.
    :return: The sanitized URL as string, or an empty string if erroneous.
    """
    assert default_scheme, "A default scheme is mandatory to avoid XSS."
    assert len(allowed_schemes) > 0, "You need to allow at least one scheme to get a result."
    assert not (force_default_scheme and
                force_remove_scheme), "You cannot force the default scheme and also force-remove the scheme."

    # Shortcut for empty string
    if not url:
        return ''

    # Remove dangerous stuff
    url = URL_CHARSET_SUB.sub('', url)

    # Split the URL
    try:
        scheme, netloc, path, query, fragment = urlsplit(url)
    except ValueError:

        # Handle malformed URL
        return ''

    # Check the scheme against the white list
    if scheme and scheme not in allowed_schemes:
        return ''

    # Detect and fix non local URL without // at beginning (not supported by  ``urlsplit``)
    if not netloc and path and not path.startswith('/') and fix_non_local_urls:
        parts = path.split('/', 1)
        if len(parts) == 2:
            netloc, path = parts
        else:
            netloc = parts[0]
            path = ''

    # Add scheme to any non-local URL if required
    if (not scheme and netloc) or force_default_scheme:
        scheme = default_scheme

    # Remove the scheme if requested
    if force_remove_scheme:
        scheme = ''

    # Build the final URL
    if netloc or not absolute_base_url:
        result = urlunsplit((scheme, netloc, path, query, fragment))
    else:
        result = urlunsplit(('', '', path, query, fragment))
        result = urljoin(absolute_base_url, result)

    # Escape HTML if requested
    if encode_html_entities:
        result = escape_html(result)

    # Return the sanitized URL
    return result

def slugify(value):
    """
    Convert the given value to a plain-text ASCII slug. Spaces are converted to hyphens.
    Characters that aren't alphanumerics, underscores, or hyphens are removed.
    The resulting string is then converted to lowercase. Leading and trailing whitespace are also stripped.
    :param value: The string to be turned into a slug.
    """
    value = value.strip()
    if not value:
        return ''
    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
    value = re.sub(r'[^\w\s-]', '', value).strip().lower()
    return re.sub(r'[-\s]+', '-', value).strip('-')
<|endoftext|>


<commit_before>'''
.. module:: skrf.circuit
========================================
circuit (:mod:`skrf.circuit`)
========================================

Provides a class representing a circuit of arbitrary topology,
consisting of an arbitrary number of N-ports networks or impedance elements.

The results are returned in :class:`~skrf.network.Circuit` objects

Circuit Class
================

.. autosummary::
   :toctree: generated/

   Circuit

'''
from . network import Network
from . media import media

import numpy as np
import networkx as nx

from itertools import chain, product
from scipy.linalg import block_diag

class Circuit():
    '''
    Notes
    -----
    The algorithm used to calculate the resultant network can be found in [#]_.

    References
    ----------
    .. [#] P. Hallbjörner, Microw. Opt. Technol. Lett. 38, 99 (2003).

    '''
    def __init__(self, connections):
        '''
        Circuit constructor.

        Creates a circuit made of a set of N-ports networks.

        Parameters
        -----------
        connections : list of list
            List of the intersections of the circuit.
            Each intersection is a list of tuples containing the network
            Example of an intersection between two network:
            [ [(network1, network1_port_nb), (network2, network2_port_nb)],
              [(network1, network1_port_nb), (network2, network2_port_nb)]

        ! The external ports indexing is defined by the order of appearance of
        the ports in the connections list.

        '''
        self.connections = connections

        # check if all networks have a name
        for cnx in self.connections:
            for (ntw, _) in cnx:
                if not self._is_named(ntw):
                    raise AttributeError('All Networks must have a name.')

        # create a list of networks for initial checks
        self.ntws = self.networks_list(self.connections)

        # check if all networks have same frequency
        ref_freq = self.ntws[0].frequency
        for ntw in self.ntws:
            if ntw.frequency != ref_freq:
                raise AttributeError('All Networks must have same frequencies')
        # All frequencies are the same, Circuit frequency can be any of the ntw
        self.frequency = self.ntws[0].frequency

    def _is_named(self, ntw):
        '''
        Return True is the network has a name, False otherwise
        '''
        if not ntw.name or ntw.name == '':
            return False
        else:
            return True

    @classmethod
    def Port(cls, frequency, name, z0=50+0*1j):
        '''
        Return a port Network. Passing the frequency and name is mendatory

        Parameters
        -----------
        frequency : :class:`~skrf.frequency.Frequency`
            Frequency common to all other networks in the circuit
        name : string
            Name of the port.
            Must inlude the word 'port' inside. (ex: 'Port1' or 'port_3')
        Z0 : complex
            Characteristic impedance of the port. Default is 50 Ohm.

        Returns
        --------
        port : :class:`~skrf.network.Network` object
            (External) 1-port network

        Examples
        -----------
        .. ipython::

            @suppress
            In [16]: import skrf as rf

            In [17]: freq = rf.Frequency(start=1, stop=2, npoints=101)

            In [18]: port1 = rf.Circuit.Port(freq, name='Port1')
        '''
        _media = media.DefinedGammaZ0(frequency, z0=z0)
        return _media.match(name=name)

    def networks_dict(self, connections=None, min_nports=1):
        '''
        Return the dictionnary of Networks from the connection setup X
        '''
        if not connections:
            connections = self.connections

        ntws = []
        for cnx in connections:
            for (ntw, port) in cnx:
                ntws.append(ntw)
        return {ntw.name: ntw for ntw in ntws  if ntw.nports >= min_nports}

    def networks_list(self, connections=None, min_nports=1):
        '''
        return a list of unique networks (sorted by appearing order in connections)
        '''
        if not connections:
            connections = self.connections

        ntw_dict = self.networks_dict(connections)
        return [ntw for ntw in ntw_dict.values() if ntw.nports >= min_nports]

    @property
    def connections_nb(self):
        '''
        Returns the number of intersections in the circuit.
        '''
        return len(self.connections)

    @property
    def networks_nb(self):
        '''
        Return the number of connected networks (port excluded)
        '''
        return len(self.networks_list(self.connections))

    @property
    def nodes_nb(self):
        '''
        Return the number of nodes in the circuit.
        '''
        return self.connections_nb + self.networks_nb

    @property
    def dim(self):
        '''
        Return the dimension of the C, X and global S matrices.

        It correspond to the sum of all connections
        '''
        return np.sum([len(cnx) for cnx in self.connections])

    @property
    def G(self):
        return self.graph()

    def graph(self):
        '''
        Generate the graph of the circuit
        '''
        G = nx.Graph()
        # Adding network nodes
        G.add_nodes_from([it for it in self.networks_dict(self.connections)])

        # Adding edges in the graph between connections and networks
        for (idx, cnx) in enumerate(self.connections):
            cnx_name = f'$X_{idx}$'
            # Adding connection nodes and edges
            G.add_node(cnx_name)
            for (ntw, ntw_port) in cnx:
                ntw_name = ntw.name
                G.add_edge(cnx_name, ntw_name)
        return G

    def is_connected(self):
        '''
        Check if the circuit's graph is connected, that is if
        if every pair of vertices in the graph is connected.
        '''
        return nx.algorithms.components.is_connected(self.G)

    def plot(self):
        '''
        Plot the graph of the circuit
        '''
        nx.draw(self.G, with_labels=True)

    def _Y_k(self, cnx):
        '''
        Return the sum of the system admittances of each intersection
        '''
        y_ns = []
        for (ntw, ntw_port) in cnx:
            # formula (2)
            y_ns.append(1/ntw.z0[:,ntw_port] )

        y_k = np.array(y_ns).sum(axis=0)  # shape: (nb_freq,)
        return y_k

    def _Xnn_k(self, cnx_k):
        '''
        Return the reflection coefficients x_nn of the connection matrix [X]_k
        '''
        X_nn = []
        y_k = self._Y_k(cnx_k)

        for (ntw, ntw_port) in cnx_k:
            # formula (1)
            X_nn.append( 2/(ntw.z0[:,ntw_port]*y_k) - 1)

        return np.array(X_nn).T  # shape: (nb_freq, nb_n)

    def _Xmn_k(self, cnx_k):
        '''
        Return the transmission coefficient x_mn of the mth column of
        intersection scattering matrix matrix [X]_k
        '''
        # get the char.impedance of the n
        X_mn = []
        y_k = self._Y_k(cnx_k)

        # There is a problem in the case of two-ports connexion:
        # the formula (3) in P. Hallbjörner (2003) seems incorrect.
        # Instead of Z_n one should have sqrt(Z_1 x Z_2).
        # The formula works with respect to the example given in the paper
        # (3 port connection), but not with 2-port connections made with skrf
        if len(cnx_k) == 2:
            z0s = []
            for (ntw, ntw_port) in cnx_k:
                z0s.append(ntw.z0[:,ntw_port])

            z0eq = np.array(z0s).prod(axis=0)

            for (ntw, ntw_port) in cnx_k:
                X_mn.append( 2/(np.sqrt(z0eq) *y_k) )
        else:
            # formula (3)
            for (ntw, ntw_port) in cnx_k:
                X_mn.append( 2/(ntw.z0[:,ntw_port]*y_k) )

        return np.array(X_mn).T  # shape: (nb_freq, nb_n)

    def _Xk(self, cnx_k):
        '''
        Return the scattering matrices [X]_k of the individual intersections k
        '''
        Xnn = self._Xnn_k(cnx_k)
        Xmn = self._Xmn_k(cnx_k)
        # repeat Xmn along the lines
        # TODO : avoid the loop?
        Xs = []
        for (_Xnn, _Xmn) in zip(Xnn, Xmn):  # for all frequencies
            _X = np.tile(_Xmn, (len(_Xmn), 1))
            _X[np.diag_indices(len(_Xmn))] = _Xnn
            Xs.append(_X)

        return np.array(Xs) # shape : nb_freq, nb_n, nb_n

        # TEST : Could we use media.splitter() instead ? -> does not work
        # _media = media.DefinedGammaZ0(frequency=self.frequency)
        # Xs = _media.splitter(len(cnx_k), z0=self._cnx_z0(cnx_k))
        # return Xs.s

    @property
    def X(self):
        '''
        Return the concatenated intersection matrix [X] of the circuit.

        It is composed of the individual intersection matrices [X]_k assembled
        by bloc diagonal.
        '''
        Xk = []
        for cnx in self.connections:
            Xk.append(self._Xk(cnx))
        Xk = np.array(Xk)

        #X = np.zeros(len(C.frequency), )
        Xf = []
        for (idx, f) in enumerate(self.frequency):
            Xf.append(block_diag(*Xk[:,idx,:]))
        return np.array(Xf)  # shape: (nb_frequency, nb_inter*nb_n, nb_inter*nb_n)

    @property
    def C(self):
        '''
        Return the global scattering matrix of the networks
        '''
        # list all networks with at least two ports
        ntws = self.networks_dict(min_nports=2)

        # generate the port reordering indexes from each connections
        ntws_ports_reordering = {ntw:[] for ntw in ntws}
        for (idx_cnx, cnx) in enumerate(chain.from_iterable(self.connections)):
            ntw, ntw_port = cnx
            if ntw.name in ntws.keys():
                ntws_ports_reordering[ntw.name].append([ntw_port, idx_cnx])

        # re-ordering scattering parameters
        S = np.zeros((len(self.frequency), self.dim, self.dim), dtype='complex' )

        for (ntw_name, ntw_ports) in ntws_ports_reordering.items():
            # get the port re-ordering indexes (from -> to)
            ntw_ports = np.array(ntw_ports)
            # create the port permutations
            from_port = list(product(ntw_ports[:,0], repeat=2))
            to_port = list(product(ntw_ports[:,1], repeat=2))

            #print(ntw_name, from_port, to_port)
            for (_from, _to) in zip(from_port, to_port):
                #print(f'{_from} --> {_to}')
                S[:, _to[0], _to[1]] = ntws[ntw_name].s[:, _from[0], _from[1]]

        return S  # shape (nb_frequency, nb_inter*nb_n, nb_inter*nb_n)

    @property
    def S(self):
        '''
        Return the global scattering parameter of the circuit, that is with
        both "inner" and "outer" ports
        '''
        # transpose is necessary to get expected result
        return np.transpose(self.X @ np.linalg.inv(np.identity(self.dim) - self.C @ self.X), axes=(0,2,1))

    @property
    def port_indexes(self):
        '''
        Return the indexes of the "external" ports. These must be labelled "port"
        '''
        port_indexes = []
        for (idx_cnx, cnx) in enumerate(chain.from_iterable(self.connections)):
            ntw, ntw_port = cnx
            if 'port' in str.lower(ntw.name):
                port_indexes.append(idx_cnx)
        return port_indexes

    def _cnx_z0(self, cnx_k):
        '''
        Return the characteristic impedances of a specific connection
        '''
        z0s = []
        for (ntw, ntw_port) in cnx_k:
            z0s.append(ntw.z0[:,ntw_port])

        return np.array(z0s).T  # shape (nb_freq, nb_ports_at_cnx)

    @property
    def port_z0(self):
        '''
        Return the external port impedances
        '''
        z0s = []
        for cnx in self.connections:
            for (ntw, ntw_port) in cnx:
                z0s.append(ntw.z0[:,ntw_port])

        return np.array(z0s)[self.port_indexes, :].T  # shape (nb_freq, nb_ports)

    @property
    def S_external(self):
        '''
        Return the scattering parameter for the external ports
        '''
        port_indexes = self.port_indexes
        a, b = np.meshgrid(port_indexes, port_indexes)
        S_ext = self.S[:, a, b]
        return S_ext  # shape (nb_frequency, nb_ports, nb_ports)

    @property
    def network(self):
        '''
        Return the Network associated to external ports
        '''
        ntw = Network()
        ntw.frequency = self.frequency
        ntw.z0 = self.port_z0
        ntw.s = self.S_external
        return ntw
<commit_msg>removed a f-string for older Python 3 compatibility and added encoding comment for Python 2<commit_after># -*- coding: utf-8 -*-
'''
.. module:: skrf.circuit
========================================
circuit (:mod:`skrf.circuit`)
========================================

Provides a class representing a circuit of arbitrary topology,
consisting of an arbitrary number of N-ports networks or impedance elements.

The results are returned in :class:`~skrf.network.Circuit` objects

Circuit Class
================

.. autosummary::
   :toctree: generated/

   Circuit

'''
from . network import Network
from . media import media

import numpy as np
import networkx as nx

from itertools import chain, product
from scipy.linalg import block_diag

class Circuit():
    '''
    Notes
    -----
    The algorithm used to calculate the resultant network can be found in [#]_.

    References
    ----------
    .. [#] P. Hallbjörner, Microw. Opt. Technol. Lett. 38, 99 (2003).

    '''
    def __init__(self, connections):
        '''
        Circuit constructor.

        Creates a circuit made of a set of N-ports networks.

        Parameters
        -----------
        connections : list of list
            List of the intersections of the circuit.
            Each intersection is a list of tuples containing the network
            Example of an intersection between two network:
            [ [(network1, network1_port_nb), (network2, network2_port_nb)],
              [(network1, network1_port_nb), (network2, network2_port_nb)]

        ! The external ports indexing is defined by the order of appearance of
        the ports in the connections list.

        '''
        self.connections = connections

        # check if all networks have a name
        for cnx in self.connections:
            for (ntw, _) in cnx:
                if not self._is_named(ntw):
                    raise AttributeError('All Networks must have a name.')

        # create a list of networks for initial checks
        self.ntws = self.networks_list(self.connections)

        # check if all networks have same frequency
        ref_freq = self.ntws[0].frequency
        for ntw in self.ntws:
            if ntw.frequency != ref_freq:
                raise AttributeError('All Networks must have same frequencies')
        # All frequencies are the same, Circuit frequency can be any of the ntw
        self.frequency = self.ntws[0].frequency

    def _is_named(self, ntw):
        '''
        Return True is the network has a name, False otherwise
        '''
        if not ntw.name or ntw.name == '':
            return False
        else:
            return True

    @classmethod
    def Port(cls, frequency, name, z0=50+0*1j):
        '''
        Return a port Network. Passing the frequency and name is mendatory

        Parameters
        -----------
        frequency : :class:`~skrf.frequency.Frequency`
            Frequency common to all other networks in the circuit
        name : string
            Name of the port.
            Must inlude the word 'port' inside. (ex: 'Port1' or 'port_3')
        Z0 : complex
            Characteristic impedance of the port. Default is 50 Ohm.

        Returns
        --------
        port : :class:`~skrf.network.Network` object
            (External) 1-port network

        Examples
        -----------
        .. ipython::

            @suppress
            In [16]: import skrf as rf

            In [17]: freq = rf.Frequency(start=1, stop=2, npoints=101)

            In [18]: port1 = rf.Circuit.Port(freq, name='Port1')
        '''
        _media = media.DefinedGammaZ0(frequency, z0=z0)
        return _media.match(name=name)

    def networks_dict(self, connections=None, min_nports=1):
        '''
        Return the dictionnary of Networks from the connection setup X
        '''
        if not connections:
            connections = self.connections

        ntws = []
        for cnx in connections:
            for (ntw, port) in cnx:
                ntws.append(ntw)
        return {ntw.name: ntw for ntw in ntws  if ntw.nports >= min_nports}

    def networks_list(self, connections=None, min_nports=1):
        '''
        return a list of unique networks (sorted by appearing order in connections)
        '''
        if not connections:
            connections = self.connections

        ntw_dict = self.networks_dict(connections)
        return [ntw for ntw in ntw_dict.values() if ntw.nports >= min_nports]

    @property
    def connections_nb(self):
        '''
        Returns the number of intersections in the circuit.
        '''
        return len(self.connections)

    @property
    def networks_nb(self):
        '''
        Return the number of connected networks (port excluded)
        '''
        return len(self.networks_list(self.connections))

    @property
    def nodes_nb(self):
        '''
        Return the number of nodes in the circuit.
        '''
        return self.connections_nb + self.networks_nb

    @property
    def dim(self):
        '''
        Return the dimension of the C, X and global S matrices.

        It correspond to the sum of all connections
        '''
        return np.sum([len(cnx) for cnx in self.connections])

    @property
    def G(self):
        return self.graph()

    def graph(self):
        '''
        Generate the graph of the circuit
        '''
        G = nx.Graph()
        # Adding network nodes
        G.add_nodes_from([it for it in self.networks_dict(self.connections)])

        # Adding edges in the graph between connections and networks
        for (idx, cnx) in enumerate(self.connections):
            cnx_name = '$X_'+str(idx)+'$'
            # Adding connection nodes and edges
            G.add_node(cnx_name)
            for (ntw, ntw_port) in cnx:
                ntw_name = ntw.name
                G.add_edge(cnx_name, ntw_name)
        return G

    def is_connected(self):
        '''
        Check if the circuit's graph is connected, that is if
        if every pair of vertices in the graph is connected.
        '''
        return nx.algorithms.components.is_connected(self.G)

    def plot(self):
        '''
        Plot the graph of the circuit
        '''
        nx.draw(self.G, with_labels=True)

    def _Y_k(self, cnx):
        '''
        Return the sum of the system admittances of each intersection
        '''
        y_ns = []
        for (ntw, ntw_port) in cnx:
            # formula (2)
            y_ns.append(1/ntw.z0[:,ntw_port] )

        y_k = np.array(y_ns).sum(axis=0)  # shape: (nb_freq,)
        return y_k

    def _Xnn_k(self, cnx_k):
        '''
        Return the reflection coefficients x_nn of the connection matrix [X]_k
        '''
        X_nn = []
        y_k = self._Y_k(cnx_k)

        for (ntw, ntw_port) in cnx_k:
            # formula (1)
            X_nn.append( 2/(ntw.z0[:,ntw_port]*y_k) - 1)

        return np.array(X_nn).T  # shape: (nb_freq, nb_n)

    def _Xmn_k(self, cnx_k):
        '''
        Return the transmission coefficient x_mn of the mth column of
        intersection scattering matrix matrix [X]_k
        '''
        # get the char.impedance of the n
        X_mn = []
        y_k = self._Y_k(cnx_k)

        # There is a problem in the case of two-ports connexion:
        # the formula (3) in P. Hallbjörner (2003) seems incorrect.
        # Instead of Z_n one should have sqrt(Z_1 x Z_2).
        # The formula works with respect to the example given in the paper
        # (3 port connection), but not with 2-port connections made with skrf
        if len(cnx_k) == 2:
            z0s = []
            for (ntw, ntw_port) in cnx_k:
                z0s.append(ntw.z0[:,ntw_port])

            z0eq = np.array(z0s).prod(axis=0)

            for (ntw, ntw_port) in cnx_k:
                X_mn.append( 2/(np.sqrt(z0eq) *y_k) )
        else:
            # formula (3)
            for (ntw, ntw_port) in cnx_k:
                X_mn.append( 2/(ntw.z0[:,ntw_port]*y_k) )

        return np.array(X_mn).T  # shape: (nb_freq, nb_n)

    def _Xk(self, cnx_k):
        '''
        Return the scattering matrices [X]_k of the individual intersections k
        '''
        Xnn = self._Xnn_k(cnx_k)
        Xmn = self._Xmn_k(cnx_k)
        # repeat Xmn along the lines
        # TODO : avoid the loop?
        Xs = []
        for (_Xnn, _Xmn) in zip(Xnn, Xmn):  # for all frequencies
            _X = np.tile(_Xmn, (len(_Xmn), 1))
            _X[np.diag_indices(len(_Xmn))] = _Xnn
            Xs.append(_X)

        return np.array(Xs) # shape : nb_freq, nb_n, nb_n

        # TEST : Could we use media.splitter() instead ? -> does not work
        # _media = media.DefinedGammaZ0(frequency=self.frequency)
        # Xs = _media.splitter(len(cnx_k), z0=self._cnx_z0(cnx_k))
        # return Xs.s

    @property
    def X(self):
        '''
        Return the concatenated intersection matrix [X] of the circuit.

        It is composed of the individual intersection matrices [X]_k assembled
        by bloc diagonal.
        '''
        Xk = []
        for cnx in self.connections:
            Xk.append(self._Xk(cnx))
        Xk = np.array(Xk)

        #X = np.zeros(len(C.frequency), )
        Xf = []
        for (idx, f) in enumerate(self.frequency):
            Xf.append(block_diag(*Xk[:,idx,:]))
        return np.array(Xf)  # shape: (nb_frequency, nb_inter*nb_n, nb_inter*nb_n)

    @property
    def C(self):
        '''
        Return the global scattering matrix of the networks
        '''
        # list all networks with at least two ports
        ntws = self.networks_dict(min_nports=2)

        # generate the port reordering indexes from each connections
        ntws_ports_reordering = {ntw:[] for ntw in ntws}
        for (idx_cnx, cnx) in enumerate(chain.from_iterable(self.connections)):
            ntw, ntw_port = cnx
            if ntw.name in ntws.keys():
                ntws_ports_reordering[ntw.name].append([ntw_port, idx_cnx])

        # re-ordering scattering parameters
        S = np.zeros((len(self.frequency), self.dim, self.dim), dtype='complex' )

        for (ntw_name, ntw_ports) in ntws_ports_reordering.items():
            # get the port re-ordering indexes (from -> to)
            ntw_ports = np.array(ntw_ports)
            # create the port permutations
            from_port = list(product(ntw_ports[:,0], repeat=2))
            to_port = list(product(ntw_ports[:,1], repeat=2))

            #print(ntw_name, from_port, to_port)
            for (_from, _to) in zip(from_port, to_port):
                #print(f'{_from} --> {_to}')
                S[:, _to[0], _to[1]] = ntws[ntw_name].s[:, _from[0], _from[1]]

        return S  # shape (nb_frequency, nb_inter*nb_n, nb_inter*nb_n)

    @property
    def S(self):
        '''
        Return the global scattering parameter of the circuit, that is with
        both "inner" and "outer" ports
        '''
        # transpose is necessary to get expected result
        return np.transpose(self.X @ np.linalg.inv(np.identity(self.dim) - self.C @ self.X), axes=(0,2,1))

    @property
    def port_indexes(self):
        '''
        Return the indexes of the "external" ports. These must be labelled "port"
        '''
        port_indexes = []
        for (idx_cnx, cnx) in enumerate(chain.from_iterable(self.connections)):
            ntw, ntw_port = cnx
            if 'port' in str.lower(ntw.name):
                port_indexes.append(idx_cnx)
        return port_indexes

    def _cnx_z0(self, cnx_k):
        '''
        Return the characteristic impedances of a specific connection
        '''
        z0s = []
        for (ntw, ntw_port) in cnx_k:
            z0s.append(ntw.z0[:,ntw_port])

        return np.array(z0s).T  # shape (nb_freq, nb_ports_at_cnx)

    @property
    def port_z0(self):
        '''
        Return the external port impedances
        '''
        z0s = []
        for cnx in self.connections:
            for (ntw, ntw_port) in cnx:
                z0s.append(ntw.z0[:,ntw_port])

        return np.array(z0s)[self.port_indexes, :].T  # shape (nb_freq, nb_ports)

    @property
    def S_external(self):
        '''
        Return the scattering parameter for the external ports
        '''
        port_indexes = self.port_indexes
        a, b = np.meshgrid(port_indexes, port_indexes)
        S_ext = self.S[:, a, b]
        return S_ext  # shape (nb_frequency, nb_ports, nb_ports)

    @property
    def network(self):
        '''
        Return the Network associated to external ports
        '''
        ntw = Network()
        ntw.frequency = self.frequency
        ntw.z0 = self.port_z0
        ntw.s = self.S_external
        return ntw
<|endoftext|>


<commit_before># -*- coding: utf8 -*-

from glob import glob
import imp
import importlib
import logging
import os
import re
import thread, threading
import time

from slackbot import settings
from slackbot.slackclient import SlackClient
from slackbot.utils import to_utf8
from slackbot.dispatcher import MessageDispatcher

logger = logging.getLogger(__name__)

class Bot(object):
    def __init__(self, api_token=None):
      if api_token is None:
        api_token = settings.API_TOKEN
      self._client = SlackClient(
        api_token,
        bot_icon = settings.BOT_ICON if hasattr(settings, 'BOT_ICON') else None,
        bot_emoji = settings.BOT_EMOJI if hasattr(settings, 'BOT_EMOJI') else None
      )
      self._plugins = PluginsManager()
      self._dispatcher = MessageDispatcher(self._client, self._plugins)
      self._stop = threading.Event()

    def run(self):
      self._plugins.init_plugins()
      self._dispatcher.start()
      self._client.rtm_connect()
      thread.start_new_thread(self._keepactive, tuple())
      logger.info('Connected to slack RTM api')
      self._dispatcher.loop()

    def stop(self):
      self._stop.set()
      self._dispatcher.stop()

    def send_message(self, channel, message, **kwargs):
      """
      Send a message using the web API
      """
      logger.info("Send to %s: %s" % (channel,message))
      self._client.webapi.chat.post_message(channel, message, as_user=True, **kwargs)

    def _keepactive(self):
      logger.info('Start heartbeat thread')
      while not self._stop.isSet():
        self._stop.wait(30.0 * 60)
        self._client.ping()
      logger.info("Stop heartbeat thread")

class PluginsManager(object):
    commands = {
        'respond_to': {},
        'listen_to': {}
    }

    def __init__(self):
        pass

    def init_plugins(self):
        if hasattr(settings, 'PLUGINS'):
            plugins = settings.PLUGINS
        else:
            plugins = 'slackbot.plugins'

        for plugin in plugins:
            self._load_plugins(plugin)

    def _load_plugins(self, plugin):
        logger.info('Loading plugin "%s"', plugin)
        path_name = None
        for mod in plugin.split('.'):
            if path_name is not None:
                path_name = [path_name]
            _, path_name, _ = imp.find_module(mod, path_name)
        logger.info("[%s] Found module path '%s'" % (plugin,path_name))
        for pyfile in glob('{}/[!_]*.py'.format(path_name)):
            module = '.'.join((plugin, os.path.split(pyfile)[-1][:-3]))
            try:
                logger.info("[%s] Loading module '%s' from '%s'" % (plugin,module,pyfile))
                importlib.import_module(module)
            except:
                logger.exception('Failed to import %s', module)

    def get_plugins(self, category, text):
        logger.info("[%s] Getting available plugins" % category)
        import inspect
        f = lambda v: "%s.%s (%s)" % (v.__module__, v.__name__, inspect.getsourcefile(v))
        pretty = { k.pattern:f(v) for k,v in self.commands[category].items() }
        logger.info("[%s] Got commands: \n%s" % (category, pretty))
        has_matching_plugin = False
        for matcher in self.commands[category]:
            m = matcher.search(text)
            if m:
                has_matching_plugin = True
                yield self.commands[category][matcher], to_utf8(m.groups())

        if not has_matching_plugin:
            yield None, None

def respond_to(matchstr, flags=0):
  def wrapper(func):
    PluginsManager.commands['respond_to'][re.compile(matchstr, flags)] = func
    logger.info('Registered respond_to plugin "%s" to "%s"', func.__name__, matchstr)
    return func
  return wrapper

def listen_to(matchstr, flags=0):
  def wrapper(func):
    PluginsManager.commands['listen_to'][re.compile(matchstr, flags)] = func
    logger.info('Registered listen_to plugin "%s" to "%s"', func.__name__, matchstr)
    return func
  return wrapper
<commit_msg>Remove logging<commit_after># -*- coding: utf8 -*-

from glob import glob
import imp
import importlib
import logging
import os
import re
import thread, threading
import time

from slackbot import settings
from slackbot.slackclient import SlackClient
from slackbot.utils import to_utf8
from slackbot.dispatcher import MessageDispatcher

logger = logging.getLogger(__name__)

class Bot(object):
    def __init__(self, api_token=None):
      if api_token is None:
        api_token = settings.API_TOKEN
      self._client = SlackClient(
        api_token,
        bot_icon = settings.BOT_ICON if hasattr(settings, 'BOT_ICON') else None,
        bot_emoji = settings.BOT_EMOJI if hasattr(settings, 'BOT_EMOJI') else None
      )
      self._plugins = PluginsManager()
      self._dispatcher = MessageDispatcher(self._client, self._plugins)
      self._stop = threading.Event()

    def run(self):
      self._plugins.init_plugins()
      self._dispatcher.start()
      self._client.rtm_connect()
      thread.start_new_thread(self._keepactive, tuple())
      logger.info('Connected to slack RTM api')
      self._dispatcher.loop()

    def stop(self):
      self._stop.set()
      self._dispatcher.stop()

    def send_message(self, channel, message, **kwargs):
      """
      Send a message using the web API
      """
      logger.info("Send to %s: %s" % (channel,message))
      self._client.webapi.chat.post_message(channel, message, as_user=True, **kwargs)

    def _keepactive(self):
      logger.info('Start heartbeat thread')
      while not self._stop.isSet():
        self._stop.wait(30.0 * 60)
        self._client.ping()
      logger.info("Stop heartbeat thread")

class PluginsManager(object):
    commands = {
        'respond_to': {},
        'listen_to': {}
    }

    def __init__(self):
        pass

    def init_plugins(self):
        if hasattr(settings, 'PLUGINS'):
            plugins = settings.PLUGINS
        else:
            plugins = 'slackbot.plugins'

        for plugin in plugins:
            self._load_plugins(plugin)

    def _load_plugins(self, plugin):
        logger.info('Loading plugin "%s"', plugin)
        path_name = None
        for mod in plugin.split('.'):
            if path_name is not None:
                path_name = [path_name]
            _, path_name, _ = imp.find_module(mod, path_name)
        for pyfile in glob('{}/[!_]*.py'.format(path_name)):
            module = '.'.join((plugin, os.path.split(pyfile)[-1][:-3]))
            try:
                importlib.import_module(module)
            except:
                logger.exception('Failed to import %s', module)

    def get_plugins(self, category, text):
        has_matching_plugin = False
        for matcher in self.commands[category]:
            m = matcher.search(text)
            if m:
                has_matching_plugin = True
                yield self.commands[category][matcher], to_utf8(m.groups())

        if not has_matching_plugin:
            yield None, None

def respond_to(matchstr, flags=0):
  def wrapper(func):
    PluginsManager.commands['respond_to'][re.compile(matchstr, flags)] = func
    logger.info('Registered respond_to plugin "%s" to "%s"', func.__name__, matchstr)
    return func
  return wrapper

def listen_to(matchstr, flags=0):
  def wrapper(func):
    PluginsManager.commands['listen_to'][re.compile(matchstr, flags)] = func
    logger.info('Registered listen_to plugin "%s" to "%s"', func.__name__, matchstr)
    return func
  return wrapper
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import threading
from abc import abstractmethod, ABCMeta

__author__ = "Romain Mormont <romainmormont@hotmail.com>"
__version__ = "0.1"

class Logger(object):
    """A base class loggers. A logger is an object which print or not the messages it is passed according to
    the verbosity level. Base class are responsible for defining the printing policy.
    """
    SILENT = 0
    ERROR = 1
    WARNING = 2
    INFO = 3
    DEBUG = 4

    def __init__(self, level, prefix=True):
        """Build a logger object with the given level of verbosity
        Parameters
        ----------
        level: int
            Verbosity level
        """
        self._level = level
        self._prefix = prefix
        self._lock = threading.Lock()

    @property
    def level(self):
        return self._level

    @level.setter
    def level(self, level):
        self._level = level

    def d(self, msg):
        """Alias for self.debug
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.debug(msg)

    def debug(self, msg):
        """Logs a information message if the level of verbosity is above or equal DEBUG
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.DEBUG, msg)

    def i(self, msg):
        """Alias for self.info
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.info(msg)

    def info(self, msg):
        """Logs a information message if the level of verbosity is above or equal INFO
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.INFO, msg)

    def w(self, msg):
        """Alias for self.warning
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.warning(msg)

    def warning(self, msg):
        """Logs a information message if the level of verbosity is above or equal WARNING
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.WARNING, msg)

    def e(self, msg):
        """Alias for self.error
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.error(msg)

    def error(self, msg):
        """Logs a information message if the level of verbosity is above or equal ERROR
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.ERROR, msg)

    def _log(self, level, msg):
        """Check the verbosity level, if it is above, actually prints the message
        Parameters
        ----------
        level: int
            Verbosity level of the message
        msg: string
            The message
        """
        if self._level >= level:
            formatted = self._format_msg(level, msg)
            self._lock.acquire()
            self._print(formatted)
            self._lock.release()

    @abstractmethod
    def _print(self, formatted_msg):
        pass

    @classmethod
    def prefix(cls, level):
        from datetime import datetime
        now = datetime.now().isoformat()
        tid = "{}".format(threading.current_thread().ident).zfill(6)
        return "[tid:{}][{}][{}]".format(tid, now, cls.level2str(level))

    @classmethod
    def level2str(cls, level):
        if level == cls.DEBUG:
            return "DEBUG"
        elif level == cls.WARNING:
            return "WARN "
        elif level == cls.ERROR:
            return "ERROR"
        else:  # info
            return "INFO "

    def _format_msg(self, level, msg):
        if self._prefix:
            rows = ["{} {}".format(self.prefix(level), row) for row in msg.split(os.linesep)]
            return os.linesep.join(rows)
        else:
            return msg

class StandardOutputLogger(Logger):
    """A logger printing the messages on the standard output
    """
    def _print(self, formatted_msg):
        print formatted_msg

class FileLogger(Logger):
    """A logger printing the messages into a file
    """
    def __init__(self, filepath, level, prefix=True):
        """Create a FileLogger
        Parameters
        ----------
        filepath: string
            Path to the logging file
        level: int
            Verbosity level
        prefix: bool
            True for adding a prefix for the logger
        """
        Logger.__init__(self, level, prefix=prefix)
        self._file = open(filepath, "w+")

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def _print(self, formatted_msg):
        self._file.write(formatted_msg)

    def close(self):
        """Close the logging file
        """
        self._file.close()

class SilentLogger(Logger):
    """A logger that ignore messages
    """
    def __init__(self, prefix=True):
        Logger.__init__(self, Logger.SILENT, prefix=prefix)

    def _print(self, formatted_msg):
        pass

class Loggable(object):
    """A base class to be implemented by any component that can produce logs through a logger.
    By default the logger object is set with a SilentLogger.
    """
    __metaclass__ = ABCMeta

    def __init__(self, logger=SilentLogger()):
        self._logger = logger

    @property
    def logger(self):
        """Get the logger of the loggable object
        Returns
        -------
        logger: Logger
            The logger object
        """
        return self._logger

    @logger.setter
    def logger(self, logger):
        """Set the logger of the loggable object
        Parameters
        ----------
        logger: Logger
            The logger object to set
        """
        self._logger = logger
<commit_msg>use multiprocessing lock instead of thread lock<commit_after># -*- coding: utf-8 -*-
import os
import multiprocessing
import threading
from abc import abstractmethod, ABCMeta

__author__ = "Romain Mormont <romainmormont@hotmail.com>"
__version__ = "0.1"

class Logger(object):
    """A base class loggers. A logger is an object which print or not the messages it is passed according to
    the verbosity level. Base class are responsible for defining the printing policy.
    """
    SILENT = 0
    ERROR = 1
    WARNING = 2
    INFO = 3
    DEBUG = 4

    def __init__(self, level, prefix=True):
        """Build a logger object with the given level of verbosity
        Parameters
        ----------
        level: int
            Verbosity level
        """
        self._level = level
        self._prefix = prefix
        self._lock = multiprocessing.Lock()

    @property
    def level(self):
        return self._level

    @level.setter
    def level(self, level):
        self._level = level

    def d(self, msg):
        """Alias for self.debug
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.debug(msg)

    def debug(self, msg):
        """Logs a information message if the level of verbosity is above or equal DEBUG
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.DEBUG, msg)

    def i(self, msg):
        """Alias for self.info
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.info(msg)

    def info(self, msg):
        """Logs a information message if the level of verbosity is above or equal INFO
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.INFO, msg)

    def w(self, msg):
        """Alias for self.warning
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.warning(msg)

    def warning(self, msg):
        """Logs a information message if the level of verbosity is above or equal WARNING
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.WARNING, msg)

    def e(self, msg):
        """Alias for self.error
        Parameters
        ----------
        msg: string
            The message to log
        """
        self.error(msg)

    def error(self, msg):
        """Logs a information message if the level of verbosity is above or equal ERROR
        Parameters
        ----------
        msg: string
            The message to log
        """
        self._log(Logger.ERROR, msg)

    def _log(self, level, msg):
        """Check the verbosity level, if it is above, actually prints the message
        Parameters
        ----------
        level: int
            Verbosity level of the message
        msg: string
            The message
        """
        if self._level >= level:
            formatted = self._format_msg(level, msg)
            self._lock.acquire()
            self._print(formatted)
            self._lock.release()

    @abstractmethod
    def _print(self, formatted_msg):
        pass

    @classmethod
    def prefix(cls, level):
        from datetime import datetime
        now = datetime.now().isoformat()
        tid = "{}".format(threading.current_thread().ident).zfill(6)
        return "[tid:{}][{}][{}]".format(tid, now, cls.level2str(level))

    @classmethod
    def level2str(cls, level):
        if level == cls.DEBUG:
            return "DEBUG"
        elif level == cls.WARNING:
            return "WARN "
        elif level == cls.ERROR:
            return "ERROR"
        else:  # info
            return "INFO "

    def _format_msg(self, level, msg):
        if self._prefix:
            rows = ["{} {}".format(self.prefix(level), row) for row in msg.split(os.linesep)]
            return os.linesep.join(rows)
        else:
            return msg

class StandardOutputLogger(Logger):
    """A logger printing the messages on the standard output
    """
    def _print(self, formatted_msg):
        print formatted_msg

class FileLogger(Logger):
    """A logger printing the messages into a file
    """
    def __init__(self, filepath, level, prefix=True):
        """Create a FileLogger
        Parameters
        ----------
        filepath: string
            Path to the logging file
        level: int
            Verbosity level
        prefix: bool
            True for adding a prefix for the logger
        """
        Logger.__init__(self, level, prefix=prefix)
        self._file = open(filepath, "w+")

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.close()

    def _print(self, formatted_msg):
        self._file.write(formatted_msg)

    def close(self):
        """Close the logging file
        """
        self._file.close()

class SilentLogger(Logger):
    """A logger that ignore messages
    """
    def __init__(self, prefix=True):
        Logger.__init__(self, Logger.SILENT, prefix=prefix)

    def _print(self, formatted_msg):
        pass

class Loggable(object):
    """A base class to be implemented by any component that can produce logs through a logger.
    By default the logger object is set with a SilentLogger.
    """
    __metaclass__ = ABCMeta

    def __init__(self, logger=SilentLogger()):
        self._logger = logger

    @property
    def logger(self):
        """Get the logger of the loggable object
        Returns
        -------
        logger: Logger
            The logger object
        """
        return self._logger

    @logger.setter
    def logger(self, logger):
        """Set the logger of the loggable object
        Parameters
        ----------
        logger: Logger
            The logger object to set
        """
        self._logger = logger
<|endoftext|>


<commit_before>#! /usr/bin/env python3.4

import os
import sqlite3
from datetime import datetime

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "data.settings")

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

from data.models import File as DB_FILE, Path as DB_PATH
from django.db.utils import OperationalError

class InterfaceHelper(object):
    """Handles the messaging to the user."""

    def __init__(self, progress_bar, progress_var, update_idletasks, status_config):
        progress_bar.configure(maximum=100)
        self.progress_bar = progress_bar
        self.progress_var = progress_var
        self.update_idletasks = update_idletasks
        self.status_config = status_config

    def message_user(self, through='status', msg='Ready', weight=0, value=0):
        """Show a message to the user."""
        if through == 'status':
            self._use_status(msg, weight)
        if through == 'progress_bar':
            self._use_progress_bar(weight, value)
        if through == 'both':
            self._use_status(msg, weight)
            self._use_progress_bar(weight, value)

    def _use_status(self, msg, weight):
        if weight == 0:
            self.status_config(foreground='black', text=' {}'.format(str(msg)))
        if weight == 1:
            self.status_config(foreground='blue', text=' {}'.format(str(msg)))
        if weight == 2:
            self.status_config(foreground='red',
                               text=' {}'.format(str(msg)))

    def _use_progress_bar(self, weight, value):
        color = 'blue'
        if weight == 1:
            color = 'green'
        self.progress_var.set(value)
        self.progress_bar.configure(
            style="{}.Horizontal.TProgressbar".format(color))
        self.update_idletasks()

class DatabaseHelper(object):
    """A helper class that interacts with the database.

    data attributes:
        DB_NAME - the path to the database
        db_path_objects - DB_PATH.objects
        db_file_objects - DB_FILE.objects
        db_ready - True if database table have been created

    methods:
        initialise_db
        get_start_value
        get_report
        update
        alter_value
        get_history
    """

    def __init__(self, db_name):
        self.DB_NAME = db_name
        self.db_file_objects = DB_FILE.objects
        self.db_path_objects = DB_PATH.objects
        self.db_ready = False

    def initialise_db(self, test=False):
        """Initialise database, set self.db_ready to True.

        If test is True (for tests), drop tables first.
        """
        if not self.db_ready:
            connection = sqlite3.connect(self.DB_NAME)
            cursor = connection.cursor()

            if test:
                query1 = 'DROP TABLE IF EXISTS data_file'
                query2 = 'DROP TABLE IF EXISTS data_path'
                query3 = 'DROP TABLE IF EXISTS data_path_filename_id_1d40e5f2'
                cursor.execute(query1)
                cursor.execute(query2)
                cursor.execute(query3)
                connection.commit()
            # Create tables
            # Do not alter the queries, may not work with subsequent database
            # operations
            query1 = """CREATE TABLE IF NOT EXISTS "data_file" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "filename" text NOT NULL, "filepath_hash" text NOT NULL, "last_modified" datetime NOT NULL, "added_at" datetime NOT NULL);"""
            query2 = """CREATE TABLE IF NOT EXISTS "data_path" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "source" text NOT NULL, "destination" text NOT NULL, "accepted" bool NOT NULL, "added_at" datetime NOT NULL, "filename_id" integer NOT NULL REFERENCES "data_file" ("id"));"""
            query3 = """CREATE INDEX IF NOT EXISTS "data_path_filename_id_1d40e5f2" ON "data_path" ("filename_id");"""
            cursor.execute(query1)
            cursor.execute(query2)
            cursor.execute(query3)
            connection.commit()
            connection.close()
            self.db_ready = True

        return self.db_ready

    def get_start_value(self):
        """Return the index of the last database entry in the
        db_file_objects table.

        Return 0 if table has no entry.
        """
        try:
            start_value = self.db_file_objects.last().id
        except (AttributeError, OperationalError):
            start_value = 0
        finally:
            return start_value

    def get_report(self, start_value):
        """Return the objects created by the current operations."""
        paths = self.db_path_objects.filter(
            id__gte=start_value).order_by('-pk')

        report = []
        for path in paths:
            row_tup = (path.filename.filename, path.source,
                       path.destination, path.added_at)
            report.append(row_tup)
        return report

    def update(self, database_dict):
        """Insert values of the current operations into the database."""
        now = datetime.now()
        item_count = len(database_dict)

        run_list = []
        for filename in database_dict.keys():
            filename_dict = database_dict[filename]
            this_file = DB_FILE(added_at=now, **filename_dict['file'])
            run_list.append(this_file)
        self.db_file_objects.bulk_create(run_list)

        file_objects = DB_FILE.objects.filter(added_at=now)
        for file_ in file_objects:
            this_path = self.db_path_objects.create(
                filename=file_, added_at=now, **database_dict[file_.filename]['path'])

    def alter_path(self, alter_value, finders):
        """Alter the value of db_file_objects instance in the database.

        finders - the key,value pairs to use to search for the entry in the
            table
        alter_value - the key,value pair of the value to alter in the database
            table
        """
        updated = self.db_path_objects.filter(**finders).update(**alter_value)

    def get_history(self, count):
        """Return the number of db_file_objects instances as specified by count."""
        files = self.db_file_objects.all().order_by(
            '-pk').select_related().filter(filename_path__accepted=True)[:count]

        try:
            length = len(files)
        except OperationalError:
            pass
        finally:
            return files
<commit_msg>Fix: catch OperationalError from Django and sqlite3<commit_after>#! /usr/bin/env python3.4

import os
import sqlite3
from datetime import datetime

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "data.settings")

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

from data.models import File as DB_FILE, Path as DB_PATH
from django.db.utils import OperationalError as DjangoOperationalError

class InterfaceHelper(object):
    """Handles the messaging to the user."""

    def __init__(self, progress_bar, progress_var, update_idletasks, status_config):
        progress_bar.configure(maximum=100)
        self.progress_bar = progress_bar
        self.progress_var = progress_var
        self.update_idletasks = update_idletasks
        self.status_config = status_config

    def message_user(self, through='status', msg='Ready', weight=0, value=0):
        """Show a message to the user."""
        if through == 'status':
            self._use_status(msg, weight)
        if through == 'progress_bar':
            self._use_progress_bar(weight, value)
        if through == 'both':
            self._use_status(msg, weight)
            self._use_progress_bar(weight, value)

    def _use_status(self, msg, weight):
        if weight == 0:
            self.status_config(foreground='black', text=' {}'.format(str(msg)))
        if weight == 1:
            self.status_config(foreground='blue', text=' {}'.format(str(msg)))
        if weight == 2:
            self.status_config(foreground='red',
                               text=' {}'.format(str(msg)))

    def _use_progress_bar(self, weight, value):
        color = 'blue'
        if weight == 1:
            color = 'green'
        self.progress_var.set(value)
        self.progress_bar.configure(
            style="{}.Horizontal.TProgressbar".format(color))
        self.update_idletasks()

class DatabaseHelper(object):
    """A helper class that interacts with the database.

    data attributes:
        DB_NAME - the path to the database
        db_path_objects - DB_PATH.objects
        db_file_objects - DB_FILE.objects
        db_ready - True if database table have been created

    methods:
        initialise_db
        get_start_value
        get_report
        update
        alter_value
        get_history
    """

    def __init__(self, db_name):
        self.DB_NAME = db_name
        self.db_file_objects = DB_FILE.objects
        self.db_path_objects = DB_PATH.objects
        self.db_ready = False

    def initialise_db(self, test=False):
        """Initialise database, set self.db_ready to True.

        If test is True (for tests), drop tables first.
        """
        if not self.db_ready:
            connection = sqlite3.connect(self.DB_NAME)
            cursor = connection.cursor()

            if test:
                query1 = 'DROP TABLE IF EXISTS data_file'
                query2 = 'DROP TABLE IF EXISTS data_path'
                query3 = 'DROP TABLE IF EXISTS data_path_filename_id_1d40e5f2'
                cursor.execute(query1)
                cursor.execute(query2)
                cursor.execute(query3)
                connection.commit()
            # Create tables
            # Do not alter the queries, may not work with subsequent database
            # operations
            query1 = """CREATE TABLE IF NOT EXISTS "data_file" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "filename" text NOT NULL, "filepath_hash" text NOT NULL, "last_modified" datetime NOT NULL, "added_at" datetime NOT NULL);"""
            query2 = """CREATE TABLE IF NOT EXISTS "data_path" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "source" text NOT NULL, "destination" text NOT NULL, "accepted" bool NOT NULL, "added_at" datetime NOT NULL, "filename_id" integer NOT NULL REFERENCES "data_file" ("id"));"""
            query3 = """CREATE INDEX IF NOT EXISTS "data_path_filename_id_1d40e5f2" ON "data_path" ("filename_id");"""
            cursor.execute(query1)
            cursor.execute(query2)
            cursor.execute(query3)
            connection.commit()
            connection.close()
            self.db_ready = True

        return self.db_ready

    def get_start_value(self):
        """Return the index of the last database entry in the
        db_file_objects table.

        Return 0 if table has no entry.
        """
        try:
            start_value = self.db_file_objects.last().id
        except (AttributeError, DjangoOperationalError):
            start_value = 0
        finally:
            return start_value

    def get_report(self, start_value):
        """Return the objects created by the current operations."""
        paths = self.db_path_objects.filter(
            id__gte=start_value).order_by('-pk')

        report = []
        for path in paths:
            row_tup = (path.filename.filename, path.source,
                       path.destination, path.added_at)
            report.append(row_tup)
        return report

    def update(self, database_dict):
        """Insert values of the current operations into the database."""
        now = datetime.now()
        item_count = len(database_dict)

        run_list = []
        for filename in database_dict.keys():
            filename_dict = database_dict[filename]
            this_file = DB_FILE(added_at=now, **filename_dict['file'])
            run_list.append(this_file)
        self.db_file_objects.bulk_create(run_list)

        file_objects = DB_FILE.objects.filter(added_at=now)
        for file_ in file_objects:
            this_path = self.db_path_objects.create(
                filename=file_, added_at=now, **database_dict[file_.filename]['path'])

    def alter_path(self, alter_value, finders):
        """Alter the value of db_file_objects instance in the database.

        finders - the key,value pairs to use to search for the entry in the
            table
        alter_value - the key,value pair of the value to alter in the database
            table
        """
        updated = self.db_path_objects.filter(**finders).update(**alter_value)

    def get_history(self, count):
        """Return the number of db_file_objects instances as specified by count."""
        try:
            files = self.db_file_objects.all().order_by(
                '-pk').select_related().filter(filename_path__accepted=True)[:count]
        except (DjangoOperationalError, sqlite3.OperationalError):
            files = ''
        finally:
            return files
<|endoftext|>