<commit_before>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
from urllib.parse import quote, urlparse
import uuid

from tornado import web
from tornado.auth import OAuth2Mixin
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List, default

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol, host=host, path=url_path_join(hub_server_url, 'oauth_callback')
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(json_state.encode('utf8')).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(OAuth2Mixin, BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    # these URLs are part of the OAuth2Mixin API
    # get them from the Authenticator object
    @property
    def _OAUTH_AUTHORIZE_URL(self):
        return self.authenticator.authorize_url

    @property
    def _OAUTH_ACCESS_TOKEN_URL(self):
        return self.authenticator.token_url

    @property
    def _OAUTH_USERINFO_URL(self):
        return self.authenticator.userdata_url

    def set_state_cookie(self, state):
        self.set_secure_cookie(STATE_COOKIE_NAME, state, expires_days=1, httponly=True)

    _state = None

    def get_state(self):
        next_url = original_next_url = self.get_argument('next', None)
        if next_url:
            # avoid browsers treating \ as /
            next_url = next_url.replace('\\', quote('\\'))
            # disallow hostname-having urls,
            # force absolute path redirect
            urlinfo = urlparse(next_url)
            next_url = urlinfo._replace(
                scheme='', netloc='', path='/' + urlinfo.path.lstrip('/')
            ).geturl()
            if next_url != original_next_url:
                self.log.warning(
                    "Ignoring next_url %r, using %r", original_next_url, next_url
                )
        if self._state is None:
            self._state = _serialize_state(
                {'state_id': uuid.uuid4().hex, 'next_url': next_url}
            )
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params={'state': state},
            response_type='code',
        )

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (
                self.get_secure_cookie(STATE_COOKIE_NAME) or b''
            ).decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_error(self):
        """Check the OAuth code"""
        error = self.get_argument("error", False)
        if error:
            message = self.get_argument("error_description", error)
            raise web.HTTPError(400, "OAuth error: %s" % message)

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check for oauth-standard error, error_description arguments
        - check that there's a code
        - check that state matches
        """
        self.check_error()
        self.check_code()
        self.check_state()

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    async def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = await self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    async def get(self):
        self.check_arguments()
        user = await self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    login_handler (likely a subclass of OAuthLoginHandler)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    authorize_url = Unicode(
        config=True, help="""The authenticate url for initiating oauth"""
    )
    @default("authorize_url")
    def _authorize_url_default(self):
        return os.environ.get("OAUTH2_AUTHORIZE_URL", "")

    token_url = Unicode(
        config=True,
        help="""The url retrieving an access token at the completion of oauth""",
    )
    @default("token_url")
    def _token_url_default(self):
        return os.environ.get("OAUTH2_TOKEN_URL", "")

    userdata_url = Unicode(
        config=True,
        help="""The url for retrieving user data with a completed access token""",
    )
    @default("userdata_url")
    def _userdata_url_default(self):
        return os.environ.get("OAUTH2_USERDATA_URL", "")

    scope = List(
        Unicode(),
        config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """,
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`""",
    )

    client_id_env = ''
    client_id = Unicode(config=True)

    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)

    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)

    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    login_handler = "Specify login handler class in subclass"
    callback_handler = OAuthCallbackHandler

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url,
            )
        else:
            raise ValueError(
                "Specify callback oauth_callback_url or give me a handler to guess with"
            )

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    async def authenticate(self, handler, data=None):
        raise NotImplementedError()
<commit_msg>make sure we actually use the new login handler class<commit_after>"""
Base classes for Custom Authenticator to use OAuth with JupyterHub

Most of the code c/o Kyle Kelley (@rgbkrk)
"""

import base64
import json
import os
from urllib.parse import quote, urlparse
import uuid

from tornado import web
from tornado.auth import OAuth2Mixin
from tornado.log import app_log

from jupyterhub.handlers import BaseHandler
from jupyterhub.auth import Authenticator
from jupyterhub.utils import url_path_join

from traitlets import Unicode, Bool, List, default

def guess_callback_uri(protocol, host, hub_server_url):
    return '{proto}://{host}{path}'.format(
        proto=protocol, host=host, path=url_path_join(hub_server_url, 'oauth_callback')
    )

STATE_COOKIE_NAME = 'oauthenticator-state'

def _serialize_state(state):
    """Serialize OAuth state to a base64 string after passing through JSON"""
    json_state = json.dumps(state)
    return base64.urlsafe_b64encode(json_state.encode('utf8')).decode('ascii')

def _deserialize_state(b64_state):
    """Deserialize OAuth state as serialized in _serialize_state"""
    if isinstance(b64_state, str):
        b64_state = b64_state.encode('ascii')
    try:
        json_state = base64.urlsafe_b64decode(b64_state).decode('utf8')
    except ValueError:
        app_log.error("Failed to b64-decode state: %r", b64_state)
        return {}
    try:
        return json.loads(json_state)
    except ValueError:
        app_log.error("Failed to json-decode state: %r", json_state)
        return {}

class OAuthLoginHandler(OAuth2Mixin, BaseHandler):
    """Base class for OAuth login handler

    Typically subclasses will need
    """

    # these URLs are part of the OAuth2Mixin API
    # get them from the Authenticator object
    @property
    def _OAUTH_AUTHORIZE_URL(self):
        return self.authenticator.authorize_url

    @property
    def _OAUTH_ACCESS_TOKEN_URL(self):
        return self.authenticator.token_url

    @property
    def _OAUTH_USERINFO_URL(self):
        return self.authenticator.userdata_url

    def set_state_cookie(self, state):
        self.set_secure_cookie(STATE_COOKIE_NAME, state, expires_days=1, httponly=True)

    _state = None

    def get_state(self):
        next_url = original_next_url = self.get_argument('next', None)
        if next_url:
            # avoid browsers treating \ as /
            next_url = next_url.replace('\\', quote('\\'))
            # disallow hostname-having urls,
            # force absolute path redirect
            urlinfo = urlparse(next_url)
            next_url = urlinfo._replace(
                scheme='', netloc='', path='/' + urlinfo.path.lstrip('/')
            ).geturl()
            if next_url != original_next_url:
                self.log.warning(
                    "Ignoring next_url %r, using %r", original_next_url, next_url
                )
        if self._state is None:
            self._state = _serialize_state(
                {'state_id': uuid.uuid4().hex, 'next_url': next_url}
            )
        return self._state

    def get(self):
        redirect_uri = self.authenticator.get_callback_url(self)
        self.log.info('OAuth redirect: %r', redirect_uri)
        state = self.get_state()
        self.set_state_cookie(state)
        self.authorize_redirect(
            redirect_uri=redirect_uri,
            client_id=self.authenticator.client_id,
            scope=self.authenticator.scope,
            extra_params={'state': state},
            response_type='code',
        )

class OAuthCallbackHandler(BaseHandler):
    """Basic handler for OAuth callback. Calls authenticator to verify username."""

    _state_cookie = None

    def get_state_cookie(self):
        """Get OAuth state from cookies

        To be compared with the value in redirect URL
        """
        if self._state_cookie is None:
            self._state_cookie = (
                self.get_secure_cookie(STATE_COOKIE_NAME) or b''
            ).decode('utf8', 'replace')
            self.clear_cookie(STATE_COOKIE_NAME)
        return self._state_cookie

    def get_state_url(self):
        """Get OAuth state from URL parameters

        to be compared with the value in cookies
        """
        return self.get_argument("state")

    def check_state(self):
        """Verify OAuth state

        compare value in cookie with redirect url param
        """
        cookie_state = self.get_state_cookie()
        url_state = self.get_state_url()
        if not cookie_state:
            raise web.HTTPError(400, "OAuth state missing from cookies")
        if not url_state:
            raise web.HTTPError(400, "OAuth state missing from URL")
        if cookie_state != url_state:
            self.log.warning("OAuth state mismatch: %s != %s", cookie_state, url_state)
            raise web.HTTPError(400, "OAuth state mismatch")

    def check_error(self):
        """Check the OAuth code"""
        error = self.get_argument("error", False)
        if error:
            message = self.get_argument("error_description", error)
            raise web.HTTPError(400, "OAuth error: %s" % message)

    def check_code(self):
        """Check the OAuth code"""
        if not self.get_argument("code", False):
            raise web.HTTPError(400, "OAuth callback made without a code")

    def check_arguments(self):
        """Validate the arguments of the redirect

        Default:

        - check for oauth-standard error, error_description arguments
        - check that there's a code
        - check that state matches
        """
        self.check_error()
        self.check_code()
        self.check_state()

    def get_next_url(self, user=None):
        """Get the redirect target from the state field"""
        state = self.get_state_url()
        if state:
            next_url = _deserialize_state(state).get('next_url')
            if next_url:
                return next_url
        # JupyterHub 0.8 adds default .get_next_url for a fallback
        if hasattr(BaseHandler, 'get_next_url'):
            return super().get_next_url(user)
        return url_path_join(self.hub.server.base_url, 'home')

    async def _login_user_pre_08(self):
        """login_user simplifies the login+cookie+auth_state process in JupyterHub 0.8

        _login_user_07 is for backward-compatibility with JupyterHub 0.7
        """
        user_info = await self.authenticator.get_authenticated_user(self, None)
        if user_info is None:
            return
        if isinstance(user_info, dict):
            username = user_info['name']
        else:
            username = user_info
        user = self.user_from_username(username)
        self.set_login_cookie(user)
        return user

    if not hasattr(BaseHandler, 'login_user'):
        # JupyterHub 0.7 doesn't have .login_user
        login_user = _login_user_pre_08

    async def get(self):
        self.check_arguments()
        user = await self.login_user()
        if user is None:
            # todo: custom error page?
            raise web.HTTPError(403)
        self.redirect(self.get_next_url(user))

class OAuthenticator(Authenticator):
    """Base class for OAuthenticators

    Subclasses must override:

    login_service (string identifying the service provider)
    authenticate (method takes one arg - the request handler handling the oauth callback)
    """

    login_handler = OAuthLoginHandler

    authorize_url = Unicode(
        config=True, help="""The authenticate url for initiating oauth"""
    )
    @default("authorize_url")
    def _authorize_url_default(self):
        return os.environ.get("OAUTH2_AUTHORIZE_URL", "")

    token_url = Unicode(
        config=True,
        help="""The url retrieving an access token at the completion of oauth""",
    )
    @default("token_url")
    def _token_url_default(self):
        return os.environ.get("OAUTH2_TOKEN_URL", "")

    userdata_url = Unicode(
        config=True,
        help="""The url for retrieving user data with a completed access token""",
    )
    @default("userdata_url")
    def _userdata_url_default(self):
        return os.environ.get("OAUTH2_USERDATA_URL", "")

    scope = List(
        Unicode(),
        config=True,
        help="""The OAuth scopes to request.
        See the OAuth documentation of your OAuth provider for options.
        For GitHub in particular, you can see github_scopes.md in this repo.
        """,
    )

    login_service = 'override in subclass'
    oauth_callback_url = Unicode(
        os.getenv('OAUTH_CALLBACK_URL', ''),
        config=True,
        help="""Callback URL to use.
        Typically `https://{host}/hub/oauth_callback`""",
    )

    client_id_env = ''
    client_id = Unicode(config=True)

    def _client_id_default(self):
        if self.client_id_env:
            client_id = os.getenv(self.client_id_env, '')
            if client_id:
                return client_id
        return os.getenv('OAUTH_CLIENT_ID', '')

    client_secret_env = ''
    client_secret = Unicode(config=True)

    def _client_secret_default(self):
        if self.client_secret_env:
            client_secret = os.getenv(self.client_secret_env, '')
            if client_secret:
                return client_secret
        return os.getenv('OAUTH_CLIENT_SECRET', '')

    validate_server_cert_env = 'OAUTH_TLS_VERIFY'
    validate_server_cert = Bool(config=True)

    def _validate_server_cert_default(self):
        env_value = os.getenv(self.validate_server_cert_env, '')
        if env_value == '0':
            return False
        else:
            return True

    def login_url(self, base_url):
        return url_path_join(base_url, 'oauth_login')

    login_handler = "Specify login handler class in subclass"
    callback_handler = OAuthCallbackHandler

    def get_callback_url(self, handler=None):
        """Get my OAuth redirect URL

        Either from config or guess based on the current request.
        """
        if self.oauth_callback_url:
            return self.oauth_callback_url
        elif handler:
            return guess_callback_uri(
                handler.request.protocol,
                handler.request.host,
                handler.hub.server.base_url,
            )
        else:
            raise ValueError(
                "Specify callback oauth_callback_url or give me a handler to guess with"
            )

    def get_handlers(self, app):
        return [
            (r'/oauth_login', self.login_handler),
            (r'/oauth_callback', self.callback_handler),
        ]

    async def authenticate(self, handler, data=None):
        raise NotImplementedError()
<|endoftext|>


<commit_before>__author__ = 'pat'

import re
import sys
import getopt
import pickle
from collections import defaultdict

def process_line(chars, ent_map, ep_map, line, rel_map, token_counter, double_vocab, replace_digits):
    e1_str, e2_str, rel_str, label = line.strip().split('\t')
    ep_str = e1_str + '\t' + e2_str
    tokens = list(rel_str.replace(' ', '<SPACE>')) if chars else rel_str.split(' ')

    if replace_digits:
        # replace digits except for ARG and log distance tokens
        tokens = [re.sub(r'[0-9]', '#', token) if '$ARG' not in token and not re.match('\[\d\]', token) else token for
                  token in tokens]

    # have seperate vocabularies for when arg1 proceeds arg2 and vice-versa
    if double_vocab and len(tokens) > 1 and "$ARG2" in tokens and tokens.index("$ARG1") > tokens.index("$ARG2"):
        tokens = [token + '_ARG2' for token in tokens]

    for token in tokens:
        token_counter[token] += 1

    # if not chars:
    rel_str = ' '.join(tokens)

    # add 1 for 1 indexing
    ent_map.setdefault(e1_str, str(len(ent_map) + 1))
    ent_map.setdefault(e2_str, str(len(ent_map) + 1))
    ep_map.setdefault(ep_str, str(len(ep_map) + 1))
    rel_map.setdefault(rel_str, str(len(rel_map) + 1))
    return e1_str, e2_str, ep_str, rel_str, tokens, label

def export_line(e1_str, e2_str, ep_str, rel_str, tokens, ent_map, ep_map, rel_map, token_map, label, out):
    # map tokens
    token_ids = [str(token_map[token]) if token in token_map else '1' for token in tokens]
    e1 = ent_map[e1_str]
    e2 = ent_map[e2_str]
    ep = ep_map[ep_str]
    rel = rel_map[rel_str]
    out.write('\t'.join([e1, e2, ep, rel, ' '.join(token_ids), label]) + '\n')

def export_map(file_name, vocab_map):
    with open(file_name, 'w') as fp:
        vocab_map = {token: int(id) for token, id in vocab_map.iteritems()}
        for token in sorted(vocab_map, key=vocab_map.get, reverse=False):
            fp.write(token + '\t' + str(vocab_map[token]) + '\n')

def main(argv):
    in_file = ''
    out_file = ''
    save_vocab_file = ''
    load_vocab_file = ''
    chars = False
    min_count = 0
    max_seq = sys.maxint
    double_vocab = False
    reset_tokens = False
    replace_digits = False

    help_msg = 'test.py -i <inFile> -o <outputfile> -m <throw away tokens seen less than this many times> \
-s <throw away relations longer than this> -c <use char tokens (default is use words)> -d <double vocab depending on if [A1 rel A2] or [A2 rel A1]>'
    try:
        opts, args = getopt.getopt(argv, "hi:o:dcm:s:l:v:rn", ["inFile=", "outFile=", "saveVocab=", "loadVocab=",
                                                               "chars", "doubleVocab", "minCount=", "maxSeq=",
                                                               "resetVocab", "noNumbers"])
    except getopt.GetoptError:
        print help_msg
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print help_msg
            sys.exit()
        elif opt in ("-i", "--inFile"):
            in_file = arg
        elif opt in ("-o", "--outFile"):
            out_file = arg
        elif opt in ("-m", "--minCount"):
            min_count = int(arg)
        elif opt in ("-s", "--maxSeq"):
            max_seq = int(arg)
        elif opt in ("-c", "--chars"):
            chars = True
        elif opt in ("-v", "--saveVocab"):
            save_vocab_file = arg
        elif opt in ("-l", "--loadVocab"):
            load_vocab_file = arg
        elif opt in ("-d", "--doubleVocab"):
            double_vocab = True
        elif opt in ("-r", "--resetVocab"):
            reset_tokens = True
        elif opt in ("-n", "--noNumbers"):
            replace_digits = True
    print 'Input file is :', in_file
    print 'Output file is :', out_file
    print ('Exporting char tokens' if chars else 'Exporting word tokens')

    # load memory maps from file or initialize new ones
    if load_vocab_file:
        with open(load_vocab_file, 'rb') as fp:
            [ent_map, ep_map, rel_map, token_map, token_counter] = pickle.load(fp)
        if reset_tokens:
            # this should probably be a different flag
            rel_map = {}
            token_map = {}
            token_counter = defaultdict(int)
    else:
        ent_map = {}
        ep_map = {}
        rel_map = {}
        token_map = {}
        token_counter = defaultdict(int)

    # memory map all the data and return processed lines
    print 'Processing lines and getting token counts'
    data = [process_line(chars, ent_map, ep_map, line, rel_map, token_counter, double_vocab, replace_digits)
            for line in open(in_file, 'r')]

    # prune infrequent tokens
    if reset_tokens or not load_vocab_file:
        filtered_tokens = {token: count for token, count in token_counter.iteritems() if count > min_count}
        sorted_tokens = [token for token in sorted(filtered_tokens, key=filtered_tokens.get, reverse=True)]
        token_map = {token: i + 1 for i, token in enumerate(sorted_tokens)}

    print 'Exporting processed lines to file'
    # export processed data
    out = open(out_file, 'w')
    [export_line(e1_str, e2_str, ep_str, rel_str, tokens, ent_map, ep_map, rel_map, token_map, label, out)
     for e1_str, e2_str, ep_str, rel_str, tokens, label in data if len(tokens) <= max_seq]
    out.close()
    print 'Num ents: ', len(ent_map), 'Num eps: ', len(ep_map), \
        'Num rels: ', len(rel_map), 'Num tokens: ', len(token_map)

    if save_vocab_file:
        with open(save_vocab_file, 'wb') as fp:
            pickle.dump([ent_map, ep_map, rel_map, token_map, token_counter], fp)
        export_map(save_vocab_file + '-tokens.txt', token_map)
        export_map(save_vocab_file + '-relations.txt', rel_map)
        export_map(save_vocab_file + '-entities.txt', ent_map)
        export_map(save_vocab_file + '-entpairs.txt', ep_map)

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>fix bug<commit_after>__author__ = 'pat'

import re
import sys
import getopt
import pickle
from collections import defaultdict

def process_line(chars, ent_map, ep_map, line, rel_map, token_counter, double_vocab, replace_digits):
    e1_str, e2_str, rel_str, label = line.strip().split('\t')
    ep_str = e1_str + '\t' + e2_str
    tokens = list(rel_str.replace(' ', '<SPACE>')) if chars else rel_str.split(' ')

    if replace_digits:
        # replace digits except for ARG and log distance tokens
        tokens = [re.sub(r'[0-9]', '#', token) if '$ARG' not in token and not re.match('\[\d\]', token) else token for
                  token in tokens]

    # have seperate vocabularies for when arg1 proceeds arg2 and vice-versa
    if double_vocab and len(tokens) > 1 \
            and "$ARG1" in tokens and "$ARG2" in tokens \
            and tokens.index("$ARG1") > tokens.index("$ARG2"):
        tokens = [token + '_ARG2' for token in tokens]

    for token in tokens:
        token_counter[token] += 1

    # if not chars:
    rel_str = ' '.join(tokens)

    # add 1 for 1 indexing
    ent_map.setdefault(e1_str, str(len(ent_map) + 1))
    ent_map.setdefault(e2_str, str(len(ent_map) + 1))
    ep_map.setdefault(ep_str, str(len(ep_map) + 1))
    rel_map.setdefault(rel_str, str(len(rel_map) + 1))
    return e1_str, e2_str, ep_str, rel_str, tokens, label

def export_line(e1_str, e2_str, ep_str, rel_str, tokens, ent_map, ep_map, rel_map, token_map, label, out):
    # map tokens
    token_ids = [str(token_map[token]) if token in token_map else '1' for token in tokens]
    e1 = ent_map[e1_str]
    e2 = ent_map[e2_str]
    ep = ep_map[ep_str]
    rel = rel_map[rel_str]
    out.write('\t'.join([e1, e2, ep, rel, ' '.join(token_ids), label]) + '\n')

def export_map(file_name, vocab_map):
    with open(file_name, 'w') as fp:
        vocab_map = {token: int(id) for token, id in vocab_map.iteritems()}
        for token in sorted(vocab_map, key=vocab_map.get, reverse=False):
            fp.write(token + '\t' + str(vocab_map[token]) + '\n')

def main(argv):
    in_file = ''
    out_file = ''
    save_vocab_file = ''
    load_vocab_file = ''
    chars = False
    min_count = 0
    max_seq = sys.maxint
    double_vocab = False
    reset_tokens = False
    replace_digits = False

    help_msg = 'test.py -i <inFile> -o <outputfile> -m <throw away tokens seen less than this many times> \
-s <throw away relations longer than this> -c <use char tokens (default is use words)> -d <double vocab depending on if [A1 rel A2] or [A2 rel A1]>'
    try:
        opts, args = getopt.getopt(argv, "hi:o:dcm:s:l:v:rn", ["inFile=", "outFile=", "saveVocab=", "loadVocab=",
                                                               "chars", "doubleVocab", "minCount=", "maxSeq=",
                                                               "resetVocab", "noNumbers"])
    except getopt.GetoptError:
        print help_msg
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print help_msg
            sys.exit()
        elif opt in ("-i", "--inFile"):
            in_file = arg
        elif opt in ("-o", "--outFile"):
            out_file = arg
        elif opt in ("-m", "--minCount"):
            min_count = int(arg)
        elif opt in ("-s", "--maxSeq"):
            max_seq = int(arg)
        elif opt in ("-c", "--chars"):
            chars = True
        elif opt in ("-v", "--saveVocab"):
            save_vocab_file = arg
        elif opt in ("-l", "--loadVocab"):
            load_vocab_file = arg
        elif opt in ("-d", "--doubleVocab"):
            double_vocab = True
        elif opt in ("-r", "--resetVocab"):
            reset_tokens = True
        elif opt in ("-n", "--noNumbers"):
            replace_digits = True
    print 'Input file is :', in_file
    print 'Output file is :', out_file
    print ('Exporting char tokens' if chars else 'Exporting word tokens')

    # load memory maps from file or initialize new ones
    if load_vocab_file:
        with open(load_vocab_file, 'rb') as fp:
            [ent_map, ep_map, rel_map, token_map, token_counter] = pickle.load(fp)
        if reset_tokens:
            # this should probably be a different flag
            rel_map = {}
            token_map = {}
            token_counter = defaultdict(int)
    else:
        ent_map = {}
        ep_map = {}
        rel_map = {}
        token_map = {}
        token_counter = defaultdict(int)

    # memory map all the data and return processed lines
    print 'Processing lines and getting token counts'
    data = [process_line(chars, ent_map, ep_map, line, rel_map, token_counter, double_vocab, replace_digits)
            for line in open(in_file, 'r')]

    # prune infrequent tokens
    if reset_tokens or not load_vocab_file:
        filtered_tokens = {token: count for token, count in token_counter.iteritems() if count > min_count}
        sorted_tokens = [token for token in sorted(filtered_tokens, key=filtered_tokens.get, reverse=True)]
        token_map = {token: i + 1 for i, token in enumerate(sorted_tokens)}

    print 'Exporting processed lines to file'
    # export processed data
    out = open(out_file, 'w')
    [export_line(e1_str, e2_str, ep_str, rel_str, tokens, ent_map, ep_map, rel_map, token_map, label, out)
     for e1_str, e2_str, ep_str, rel_str, tokens, label in data if len(tokens) <= max_seq]
    out.close()
    print 'Num ents: ', len(ent_map), 'Num eps: ', len(ep_map), \
        'Num rels: ', len(rel_map), 'Num tokens: ', len(token_map)

    if save_vocab_file:
        with open(save_vocab_file, 'wb') as fp:
            pickle.dump([ent_map, ep_map, rel_map, token_map, token_counter], fp)
        export_map(save_vocab_file + '-tokens.txt', token_map)
        export_map(save_vocab_file + '-relations.txt', rel_map)
        export_map(save_vocab_file + '-entities.txt', ent_map)
        export_map(save_vocab_file + '-entpairs.txt', ep_map)

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
A simple client to create a CLA anomaly detection model for geospatial data.
"""

import csv
import datetime
import sys

from nupic.data.datasethelpers import findDataset
from nupic.frameworks.opf.modelfactory import ModelFactory

import model_params

DEFAULT_DATA_PATH = "data/commute.csv"
DEFAULT_OUTPUT_PATH = "anomaly_scores.csv"

ACCURACY_THRESHOLD = 80  # meters
INTERVAL_THRESHOLD = 30  # seconds

def addTimeEncoders(params):
  params["modelParams"]["sensorParams"]["encoders"]["timestamp_timeOfDay"] = {
    "fieldname": u"timestamp",
    "name": u"timestamp_timeOfDay",
    "timeOfDay": (51, 9.5),
    "type": "DateEncoder"
  }
  return params

def setEncoderScale(params, scale):
  params["modelParams"]["sensorParams"]["encoders"]["vector"]["scale"] = \
    int(scale)
  return params

def createModel(useTimeEncoders, scale, verbose):
  params = model_params.MODEL_PARAMS
  if useTimeEncoders:
    params = addTimeEncoders(params)
  if scale:
    params = setEncoderScale(params, scale)
  if verbose:
    print "Model parameters:"
    print params
  return ModelFactory.create(params)

def runGeospatialAnomaly(dataPath, outputPath,
                         scale=False,
                         autoSequence=True,
                         useTimeEncoders=False,
                         verbose=False):

  model = createModel(useTimeEncoders, scale, verbose)

  with open (findDataset(dataPath)) as fin:
    reader = csv.reader(fin)
    csvWriter = csv.writer(open(outputPath,"wb"))
    csvWriter.writerow(["timestamp",
                       "longitude",
                       "latitude",
                       "speed",
                       "anomaly_score",
                       "new_sequence"])

    reader.next()
    reader.next()
    reader.next()

    lastTimestamp = None
    lastTrackName = None
    outputFormat = "%Y-%m-%dT%H:%M:%S"

    for _, record in enumerate(reader, start=1):
      trackName = record[0]
      timestamp = datetime.datetime.fromtimestamp(int(record[1]) / 1e3)
      longitude = float(record[2])
      latitude = float(record[3])
      speed = float(record[5])
      accuracy = float(record[7])

      if accuracy > ACCURACY_THRESHOLD:
        continue

      newSequence = False
      # Handle the automatic sequence creation
      if autoSequence:
        if lastTimestamp and (
          (timestamp - lastTimestamp).total_seconds() > INTERVAL_THRESHOLD):
          newSequence = True
      # Manual sequence resets depend on the track name
      else:
        if trackName != lastTrackName:
          newSequence = True

      lastTimestamp = timestamp
      lastTrackName = trackName

      if newSequence:
        if verbose:
          print "Starting new sequence..."
        model.resetSequenceStates()

      modelInput = {
        "vector": (longitude, latitude, speed)
      }

      if useTimeEncoders:
        modelInput["timestamp"] = timestamp

      result = model.run(modelInput)
      anomalyScore = result.inferences['anomalyScore']

      csvWriter.writerow([timestamp.strftime(outputFormat),
                          longitude,
                          latitude,
                          speed,
                          anomalyScore,
                          1 if newSequence else 0])
      if verbose:
        print "[{0}] - Anomaly score: {1}.".format(timestamp, anomalyScore)

  print "Anomaly scores have been written to {0}".format(outputPath)

if __name__ == "__main__":
  dataPath = DEFAULT_DATA_PATH
  outputPath = DEFAULT_OUTPUT_PATH

  if len(sys.argv) > 1:
    dataPath = sys.argv[1]

  if len(sys.argv) > 2:
    outputPath = sys.argv[2]

  runGeospatialAnomaly(dataPath, outputPath)
<commit_msg>Fixed args order to match new GCE interface<commit_after>#!/usr/bin/env python
# ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
A simple client to create a CLA anomaly detection model for geospatial data.
"""

import csv
import datetime
import sys

from nupic.data.datasethelpers import findDataset
from nupic.frameworks.opf.modelfactory import ModelFactory

import model_params

DEFAULT_DATA_PATH = "data/commute.csv"
DEFAULT_OUTPUT_PATH = "anomaly_scores.csv"

ACCURACY_THRESHOLD = 80  # meters
INTERVAL_THRESHOLD = 30  # seconds

def addTimeEncoders(params):
  params["modelParams"]["sensorParams"]["encoders"]["timestamp_timeOfDay"] = {
    "fieldname": u"timestamp",
    "name": u"timestamp_timeOfDay",
    "timeOfDay": (51, 9.5),
    "type": "DateEncoder"
  }
  return params

def setEncoderScale(params, scale):
  params["modelParams"]["sensorParams"]["encoders"]["vector"]["scale"] = \
    int(scale)
  return params

def createModel(useTimeEncoders, scale, verbose):
  params = model_params.MODEL_PARAMS
  if useTimeEncoders:
    params = addTimeEncoders(params)
  if scale:
    params = setEncoderScale(params, scale)
  if verbose:
    print "Model parameters:"
    print params
  return ModelFactory.create(params)

def runGeospatialAnomaly(dataPath, outputPath,
                         scale=False,
                         autoSequence=True,
                         useTimeEncoders=False,
                         verbose=False):

  model = createModel(useTimeEncoders, scale, verbose)

  with open (findDataset(dataPath)) as fin:
    reader = csv.reader(fin)
    csvWriter = csv.writer(open(outputPath,"wb"))
    csvWriter.writerow(["timestamp",
                       "longitude",
                       "latitude",
                       "speed",
                       "anomaly_score",
                       "new_sequence"])

    reader.next()
    reader.next()
    reader.next()

    lastTimestamp = None
    lastTrackName = None
    outputFormat = "%Y-%m-%dT%H:%M:%S"

    for _, record in enumerate(reader, start=1):
      trackName = record[0]
      timestamp = datetime.datetime.fromtimestamp(int(record[1]) / 1e3)
      longitude = float(record[2])
      latitude = float(record[3])
      speed = float(record[5])
      accuracy = float(record[7])

      if accuracy > ACCURACY_THRESHOLD:
        continue

      newSequence = False
      # Handle the automatic sequence creation
      if autoSequence:
        if lastTimestamp and (
          (timestamp - lastTimestamp).total_seconds() > INTERVAL_THRESHOLD):
          newSequence = True
      # Manual sequence resets depend on the track name
      else:
        if trackName != lastTrackName:
          newSequence = True

      lastTimestamp = timestamp
      lastTrackName = trackName

      if newSequence:
        if verbose:
          print "Starting new sequence..."
        model.resetSequenceStates()

      modelInput = {
        "vector": (speed, longitude, latitude)
      }

      if useTimeEncoders:
        modelInput["timestamp"] = timestamp

      result = model.run(modelInput)
      anomalyScore = result.inferences['anomalyScore']

      csvWriter.writerow([timestamp.strftime(outputFormat),
                          longitude,
                          latitude,
                          speed,
                          anomalyScore,
                          1 if newSequence else 0])
      if verbose:
        print "[{0}] - Anomaly score: {1}.".format(timestamp, anomalyScore)

  print "Anomaly scores have been written to {0}".format(outputPath)

if __name__ == "__main__":
  dataPath = DEFAULT_DATA_PATH
  outputPath = DEFAULT_OUTPUT_PATH

  if len(sys.argv) > 1:
    dataPath = sys.argv[1]

  if len(sys.argv) > 2:
    outputPath = sys.argv[2]

  runGeospatialAnomaly(dataPath, outputPath)
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2022 The Deeplab2 Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Pixel Decoder used in K-Means Mask Transformer.

The pixel decoder in the k-means Mask Transformer (k-MaX) [1] employs a simple
decoder structure, similar to MaX-DeepLab-S model [2].

We support using axial-block [3] and bottleneck-block [4] in the decoder, along
with skip connections from the pixel encoder (i.e., backbone).

[1] TODO(qihangyu): Add k-max reference.

[2] MaX-DeepLab: End-to-End Panoptic Segmentation with Mask Transformers,
    CVPR 2021.
      Huiyu Wang, Yukun Zhu, Hartwig Adam, Alan Yuille, Liang-Chieh Chen.

[3] Axial-Deeplab: Stand-Alone Axial-Attention for Panoptic Segmentation,
    ECCV 2020.
      Huiyu Wang, Yukun Zhu, Bradley Green, Hartwig Adam, Alan Yuille,
      Liang-Chieh Chen.

[4] Deep residual learning for image recognition.
    CVPR 2016.
      Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun.
"""

import functools

import tensorflow as tf

from deeplab2.model.layers import axial_block_groups
from deeplab2.model.layers import resized_fuse

axial_block = functools.partial(
    axial_block_groups.BlockGroup,
    original_resnet_stride=1,
    original_resnet_input_stride=32,
    use_axial_beyond_stride=16,
    use_transformer_beyond_stride=0,
    drop_path_keep_prob=1.0,
    activation='gelu',
    axial_use_recompute_grad=False)

bottleneck_block = functools.partial(
    axial_block_groups.BlockGroup,
    original_resnet_stride=1,
    original_resnet_input_stride=32,
    use_axial_beyond_stride=0,
    use_transformer_beyond_stride=0,
    drop_path_keep_prob=1.0,
    activation='gelu')

class KMaXPixelDecoder(tf.keras.Model):
  """Pixel Decoder for K-MaX.

  The pixel decoder in the k-means Mask Transformer (k-MaX) employs a simple
  decoder structure, similar to MaX-DeepLab-S model. We support using
  axial-block and bottleneck-block in the decoder, along with skip connections
  from the pixel encoder (i.e., backbone).

  TODO(qihangyu): Add k-max reference.
  """

  def __init__(self,
               name,
               norm_layer=tf.keras.layers.BatchNormalization,
               dims=(512, 256, 128, 64),
               num_blocks=(1, 5, 1, 1),
               block_type=('axial', 'axial', 'bottleneck', 'bottleneck')):
    """Initializes a PixelDecoder for K-MaX.

    Args:
      name: A string, the name of the model.
      norm_layer: An optional tf.keras.layers.Layer that computes the
        normalization (default: tf.keras.layers.BatchNormalization).
      dims: A list of integers specifying number of channels for each stage. The
        stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.
      num_blocks: A list of integers specifying number of blocks for each stage.
        The stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.
      block_type: A list of integers specifying the block type for each stage.
        'axial' for axial block, 'bottleneck' for bottleneck block. The
        stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.

    Raises:
      ValueError: If the length of dims, num_blocks, block_type are not equal to
        each other.
      ValueError: If num_stages is not 4 or 5.
    """
    super().__init__(name=name)

    num_stages = len(dims)
    if num_stages != len(num_blocks) or num_stages != len(block_type):
      raise ValueError('Number of stages should be equal in dims, num_blocks,'
                       ' and block_type!')
    # We have 4/5 stages for os32, 16, 8, 4, (2) respectively.
    if num_stages not in [4, 5]:
      raise ValueError('Number of stages should be 4 or 5!')

    # Define layer norm appended to encoder features at each output stride.
    self._backbone_norms = []
    for i in range(num_stages):
      # TODO(qihangyu): Experiment whether it matters of using layer norm or
      # batch norm.
      self._backbone_norms.append(
          tf.keras.layers.LayerNormalization(epsilon=1e-6))

    # Define decoder blocks.
    self._stages = []
    for i in range(num_stages):
      if block_type[i].lower() == 'axial':
        block_group_fn = axial_block(filters=dims[i],
                                     num_blocks=num_blocks[i],
                                     name=f'kmax_pixeldecoder_{i}',
                                     bn_layer=norm_layer)
      elif block_type[i].lower() == 'bottleneck':
        block_group_fn = bottleneck_block(filters=dims[i],
                                          num_blocks=num_blocks[i],
                                          name=f'kmax_pixeldecoder_{i}',
                                          bn_layer=norm_layer)
      else:
        raise ValueError('Unsupported block_type!')

      self._stages.append(block_group_fn)

    self._num_stages = num_stages
    self._norm_layer = norm_layer
    self._dims = dims
    self._stage_to_backbone_endpoint = {0: 'stage5',
                                        1: 'stage4',
                                        2: 'stage3',
                                        3: 'stage2',
                                        4: 'stage1',}

  def build(self, input_shape_dict):
    # Define skip-connections.
    self._skip_connections = []
    # Stage5 does not need any skip-connection.
    for i in range(self._num_stages - 1):
      decoder_height, decoder_width = (
          input_shape_dict[self._stage_to_backbone_endpoint[i + 1]][-3:-1])
      skip_connection_fn = resized_fuse.ResizedFuse(
          name=f'kmax_pixel_decoder_skip_connection_{i}',
          height=decoder_height,
          width=decoder_width,
          num_channels=self._dims[i + 1],
          activation='gelu',
          bn_layer=self._norm_layer)
      self._skip_connections.append(skip_connection_fn)

  def call(self, endpoints, training=False):
    # Make a copy so that input argument will not be modified, per requirements
    # from exporting a saved model.
    endpoints = dict(endpoints)

    x = self._backbone_norms[0](
        endpoints[self._stage_to_backbone_endpoint[0]], training=training)
    for i in range(self._num_stages - 1):
      x, _ = self._stages[i]((x, None), training=training)
      endpoints['decoder_stage{}'.format(i + 1)] = x
      x = self._skip_connections[i]([
          x,
          self._backbone_norms[i + 1](
              endpoints[self._stage_to_backbone_endpoint[i + 1]],
              training=training)], training=training)

    x, _ = self._stages[-1]((x, None), training=training)
    endpoints['decoder_output'] = x
    return endpoints
<commit_msg>Update docstring for pixel decoder.<commit_after># coding=utf-8
# Copyright 2022 The Deeplab2 Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Pixel Decoder used in K-Means Mask Transformer.

The pixel decoder in the k-means Mask Transformer (k-MaX) [1] employs a simple
decoder structure, similar to MaX-DeepLab-S model [2].

We support using axial-block [3] and bottleneck-block [4] in the decoder, along
with skip connections from the pixel encoder (i.e., backbone). When
self-attention operations are used (e.g., axial-blocks), it is equivalent to
incorporating the transformer encoder to the pixel decoder.

[1] TODO(qihangyu): Add k-max reference.

[2] MaX-DeepLab: End-to-End Panoptic Segmentation with Mask Transformers,
    CVPR 2021.
      Huiyu Wang, Yukun Zhu, Hartwig Adam, Alan Yuille, Liang-Chieh Chen.

[3] Axial-Deeplab: Stand-Alone Axial-Attention for Panoptic Segmentation,
    ECCV 2020.
      Huiyu Wang, Yukun Zhu, Bradley Green, Hartwig Adam, Alan Yuille,
      Liang-Chieh Chen.

[4] Deep residual learning for image recognition.
    CVPR 2016.
      Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun.
"""

import functools

import tensorflow as tf

from deeplab2.model.layers import axial_block_groups
from deeplab2.model.layers import resized_fuse

axial_block = functools.partial(
    axial_block_groups.BlockGroup,
    original_resnet_stride=1,
    original_resnet_input_stride=32,
    use_axial_beyond_stride=16,
    use_transformer_beyond_stride=0,
    drop_path_keep_prob=1.0,
    activation='gelu',
    axial_use_recompute_grad=False)

bottleneck_block = functools.partial(
    axial_block_groups.BlockGroup,
    original_resnet_stride=1,
    original_resnet_input_stride=32,
    use_axial_beyond_stride=0,
    use_transformer_beyond_stride=0,
    drop_path_keep_prob=1.0,
    activation='gelu')

class KMaXPixelDecoder(tf.keras.Model):
  """Pixel Decoder for K-MaX.

  The pixel decoder in the k-means Mask Transformer (k-MaX) employs a simple
  decoder structure, similar to MaX-DeepLab-S model. We support using
  axial-block and bottleneck-block in the decoder, along with skip connections
  from the pixel encoder (i.e., backbone). When self-attention operations are
  used (e.g., axial-blocks), it is equivalent to incorporating the transformer
  encoder to the pixel decoder.

  TODO(qihangyu): Add k-max reference.
  """

  def __init__(self,
               name,
               norm_layer=tf.keras.layers.BatchNormalization,
               dims=(512, 256, 128, 64),
               num_blocks=(1, 5, 1, 1),
               block_type=('axial', 'axial', 'bottleneck', 'bottleneck')):
    """Initializes a PixelDecoder for K-MaX.

    Args:
      name: A string, the name of the model.
      norm_layer: An optional tf.keras.layers.Layer that computes the
        normalization (default: tf.keras.layers.BatchNormalization).
      dims: A list of integers specifying number of channels for each stage. The
        stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.
      num_blocks: A list of integers specifying number of blocks for each stage.
        The stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.
      block_type: A list of integers specifying the block type for each stage.
        'axial' for axial block, 'bottleneck' for bottleneck block. The
        stage is counted from backwards, i.e., output stride 32 to output
        stride 4/2.

    Raises:
      ValueError: If the length of dims, num_blocks, block_type are not equal to
        each other.
      ValueError: If num_stages is not 4 or 5.
    """
    super().__init__(name=name)

    num_stages = len(dims)
    if num_stages != len(num_blocks) or num_stages != len(block_type):
      raise ValueError('Number of stages should be equal in dims, num_blocks,'
                       ' and block_type!')
    # We have 4/5 stages for os32, 16, 8, 4, (2) respectively.
    if num_stages not in [4, 5]:
      raise ValueError('Number of stages should be 4 or 5!')

    # Define layer norm appended to encoder features at each output stride.
    self._backbone_norms = []
    for i in range(num_stages):
      # TODO(qihangyu): Experiment whether it matters of using layer norm or
      # batch norm.
      self._backbone_norms.append(
          tf.keras.layers.LayerNormalization(epsilon=1e-6))

    # Define decoder blocks.
    self._stages = []
    for i in range(num_stages):
      if block_type[i].lower() == 'axial':
        block_group_fn = axial_block(filters=dims[i],
                                     num_blocks=num_blocks[i],
                                     name=f'kmax_pixeldecoder_{i}',
                                     bn_layer=norm_layer)
      elif block_type[i].lower() == 'bottleneck':
        block_group_fn = bottleneck_block(filters=dims[i],
                                          num_blocks=num_blocks[i],
                                          name=f'kmax_pixeldecoder_{i}',
                                          bn_layer=norm_layer)
      else:
        raise ValueError('Unsupported block_type!')

      self._stages.append(block_group_fn)

    self._num_stages = num_stages
    self._norm_layer = norm_layer
    self._dims = dims
    self._stage_to_backbone_endpoint = {0: 'stage5',
                                        1: 'stage4',
                                        2: 'stage3',
                                        3: 'stage2',
                                        4: 'stage1',}

  def build(self, input_shape_dict):
    # Define skip-connections.
    self._skip_connections = []
    # Stage5 does not need any skip-connection.
    for i in range(self._num_stages - 1):
      decoder_height, decoder_width = (
          input_shape_dict[self._stage_to_backbone_endpoint[i + 1]][-3:-1])
      skip_connection_fn = resized_fuse.ResizedFuse(
          name=f'kmax_pixel_decoder_skip_connection_{i}',
          height=decoder_height,
          width=decoder_width,
          num_channels=self._dims[i + 1],
          activation='gelu',
          bn_layer=self._norm_layer)
      self._skip_connections.append(skip_connection_fn)

  def call(self, endpoints, training=False):
    # Make a copy so that input argument will not be modified, per requirements
    # from exporting a saved model.
    endpoints = dict(endpoints)

    x = self._backbone_norms[0](
        endpoints[self._stage_to_backbone_endpoint[0]], training=training)
    for i in range(self._num_stages - 1):
      x, _ = self._stages[i]((x, None), training=training)
      endpoints['decoder_stage{}'.format(i + 1)] = x
      x = self._skip_connections[i]([
          x,
          self._backbone_norms[i + 1](
              endpoints[self._stage_to_backbone_endpoint[i + 1]],
              training=training)], training=training)

    x, _ = self._stages[-1]((x, None), training=training)
    endpoints['decoder_output'] = x
    return endpoints
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Implementation of Apriori algorithm.
"""

import sys
import argparse
import json
from collections import namedtuple
from itertools import combinations
from itertools import chain

__version__ = '0.1.0'
__author__ = 'ymoch'
__author_email__ = 'ymoch@githib.com'

# Ignore name errors because these names are namedtuples.
SupportRecord = namedtuple( # pylint: disable=C0103
    'SupportRecord', ('items', 'support'))
RelationRecord = namedtuple( # pylint: disable=C0103
    'RelationRecord', SupportRecord._fields + ('ordered_statistics',))
OrderedStatistic = namedtuple( # pylint: disable=C0103
    'OrderedStatistic', ('items_base', 'items_add', 'confidence', 'lift',))

class TransactionManager(object):
    """
    Transaction manager class.
    """

    def __init__(self, transactions):
        """
        Initialize.

        @param  transactions    A transaction list.
        """
        self.__num_transaction = 0
        self.__items = []
        self.__transaction_index_map = {}

        for transaction in transactions:
            self.add_transaction(transaction)

    def add_transaction(self, transaction):
        """
        Add a transaction.

        @param  transaction A transaction.
        """
        for item in transaction:
            if item not in self.__transaction_index_map:
                self.__items.append(item)
                self.__transaction_index_map[item] = set()
            self.__transaction_index_map[item].add(self.__num_transaction)
        self.__num_transaction += 1

    def calc_support(self, items):
        """
        Returns a support for items.
        """
        # Empty items is supported by all transactions.
        if not items:
            return 1.0

        # Create the transaction index intersection.
        sum_indexes = None
        for item in items:
            indexes = self.__transaction_index_map.get(item)
            if indexes is None:
                # No support for any set that contains a not existing item.
                return 0.0

            if sum_indexes is None:
                sum_indexes = indexes
            else:
                sum_indexes = sum_indexes.intersection(indexes)

        # Calculate the support.
        return float(len(sum_indexes)) / self.__num_transaction

    def initial_candidates(self):
        """
        Returns the initial candidates.
        """
        return [frozenset([item]) for item in self.__items]

    @property
    def num_transaction(self):
        """
        Returns the number of transactions.
        """
        return self.__num_transaction

    @property
    def items(self):
        """
        Returns the item list that the transaction is consisted of.
        """
        return self.__items

    @staticmethod
    def create(transactions):
        """
        Create the TransactionManager with a transaction instance.
        If the given instance is a TransactionManager, this returns itself.
        """
        if isinstance(transactions, TransactionManager):
            return transactions
        return TransactionManager(transactions)

def create_next_candidates(prev_candidates, length):
    """
    Returns the apriori candidates.
    """
    # Solve the items.
    items = set()
    for candidate in prev_candidates:
        for item in candidate:
            items.add(item)

    def check_subsets(candidate):
        """
        Check if the subsets of a candidate is present
        in the previous candidates.
        """
        candidate_subsets = [
            frozenset(x) for x in combinations(candidate, length - 1)]
        for candidate_subset in candidate_subsets:
            if candidate_subset not in prev_candidates:
                return False
        return True

    # Create candidates.
    next_candidates = []
    for candidate in [frozenset(x) for x in combinations(items, length)]:
        if length > 2 and not check_subsets(candidate):
            continue
        next_candidates.append(candidate)
    return next_candidates

def gen_support_records(
        transaction_manager,
        min_support,
        max_length=None,
        _generate_candidates_func=create_next_candidates):
    """
    Returns a generator of support records with given transactions.
    """
    candidates = transaction_manager.initial_candidates()
    length = 1
    while candidates:
        relations = set()
        for relation_candidate in candidates:
            support = transaction_manager.calc_support(relation_candidate)
            if support < min_support:
                continue
            candidate_set = frozenset(relation_candidate)
            relations.add(candidate_set)
            yield SupportRecord(candidate_set, support)
        length += 1
        if max_length and length > max_length:
            break
        candidates = _generate_candidates_func(relations, length)

def gen_ordered_statistics(transaction_manager, record):
    """
    Returns a generator of ordered statistics.
    """
    items = record.items
    combination_sets = [
        frozenset(x) for x in combinations(items, len(items) - 1)]
    for items_base in combination_sets:
        items_add = frozenset(items.difference(items_base))
        confidence = (
            record.support / transaction_manager.calc_support(items_base))
        lift = confidence / transaction_manager.calc_support(items_add)
        yield OrderedStatistic(
            frozenset(items_base), frozenset(items_add), confidence, lift)

def apriori(transactions, **kwargs):
    """
    Run Apriori algorithm.

    @param  transactions    A list of transactions.
    @param  min_support     The minimum support of the relation (float).
    @param  max_length      The maximum length of the relation (integer).
    """
    min_support = kwargs.get('min_support', 0.1)
    max_length = kwargs.get('max_length', None)
    min_confidence = kwargs.get('min_confidence', 0.0)
    _gen_support_records = kwargs.get(
        '_gen_support_records', gen_support_records)
    _gen_ordered_statistics = kwargs.get(
        '_gen_ordered_statistics', gen_ordered_statistics)

    # Calculate supports.
    transaction_manager = TransactionManager.create(transactions)
    support_records = _gen_support_records(
        transaction_manager, min_support, max_length)

    # Calculate ordered stats.
    for support_record in support_records:
        ordered_statistics = _gen_ordered_statistics(
            transaction_manager, support_record)
        filtered_ordered_statistics = [
            x for x in ordered_statistics if x.confidence >= min_confidence]
        if not filtered_ordered_statistics:
            continue
        yield RelationRecord(
            support_record.items, support_record.support,
            filtered_ordered_statistics)

def dump_as_json(record, output_file):
    """
    Dump an relation record as a json value.

    @param  record      A record.
    @param  output_file An output file.
    """
    def default_func(value):
        """
        Default conversion for JSON value.
        """
        if isinstance(value, frozenset):
            return sorted(value)
        raise TypeError(repr(value) + " is not JSON serializable")

    converted_record = record._replace(
        ordered_statistics=[x._asdict() for x in record.ordered_statistics])
    output_file.write(
        json.dumps(converted_record._asdict(), default=default_func))
    output_file.write('\n')

def dump_as_two_item_tsv(record, output_file):
    """
    Dump a relation record as TSV only for 2 item relations.

    @param  record      A record.
    @param  output_file An output file.
    """
    for ordered_stats in record.ordered_statistics:
        if len(ordered_stats.items_base) != 1:
            return
        if len(ordered_stats.items_add) != 1:
            return
        output_file.write('{0}\t{1}\t{2:.8f}\t{3:.8f}\t{4:.8f}\n'.format(
            list(ordered_stats.items_base)[0], list(ordered_stats.items_add)[0],
            record.support, ordered_stats.confidence, ordered_stats.lift))

def parse_args(argv):
    """
    Parse commandline arguments.
    """
    output_funcs = {
        'json': dump_as_json,
        'tsv': dump_as_two_item_tsv,
    }
    default_output_func_key = 'json'

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v', '--version', action='version',
        version='%(prog)s {0}'.format(__version__))
    parser.add_argument(
        'input', metavar='inpath', nargs='*',
        help='Input transaction file (default: stdin).',
        type=argparse.FileType('r'), default=[sys.stdin])
    parser.add_argument(
        '-o', '--output', metavar='outpath',
        help='Output file (default: stdout).',
        type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument(
        '-l', '--max-length', metavar='int',
        help='Max length of relations (default: infinite).',
        type=int, default=None)
    parser.add_argument(
        '-s', '--min-support', metavar='float',
        help='Minimum support ratio (must be > 0, default: 0.1).',
        type=float, default=0.1)
    parser.add_argument(
        '-c', '--min-confidence', metavar='float',
        help='Minimum confidence (default: 0.5).',
        type=float, default=0.5)
    parser.add_argument(
        '-d', '--delimiter', metavar='str',
        help='Delimiter for items of transactions (default: tab).',
        type=str, default='\t')
    parser.add_argument(
        '-f', '--out-format', help='Output format (default or tsv).',
        metavar='str', type=str, choices=output_funcs.keys(),
        default=default_output_func_key)
    args = parser.parse_args(argv)
    if args.min_support <= 0:
        raise ValueError('min support must be > 0')

    args.output_func = output_funcs[args.out_format]
    return args

def main():
    """
    Main.
    """
    args = parse_args(sys.argv[1:])

    transactions = [
        line.strip().split(args.delimiter) for line in chain(*args.input)]
    result = apriori(
        transactions,
        max_length=args.max_length,
        min_support=args.min_support,
        min_confidence=args.min_confidence)

    for record in result:
        args.output_func(record, args.output)

if __name__ == '__main__':
    main()
<commit_msg>Update the version to 0.9.0.<commit_after>#!/usr/bin/env python

"""
Implementation of Apriori algorithm.
"""

import sys
import argparse
import json
from collections import namedtuple
from itertools import combinations
from itertools import chain

__version__ = '0.9.0'
__author__ = 'ymoch'
__author_email__ = 'ymoch@githib.com'

# Ignore name errors because these names are namedtuples.
SupportRecord = namedtuple( # pylint: disable=C0103
    'SupportRecord', ('items', 'support'))
RelationRecord = namedtuple( # pylint: disable=C0103
    'RelationRecord', SupportRecord._fields + ('ordered_statistics',))
OrderedStatistic = namedtuple( # pylint: disable=C0103
    'OrderedStatistic', ('items_base', 'items_add', 'confidence', 'lift',))

class TransactionManager(object):
    """
    Transaction manager class.
    """

    def __init__(self, transactions):
        """
        Initialize.

        @param  transactions    A transaction list.
        """
        self.__num_transaction = 0
        self.__items = []
        self.__transaction_index_map = {}

        for transaction in transactions:
            self.add_transaction(transaction)

    def add_transaction(self, transaction):
        """
        Add a transaction.

        @param  transaction A transaction.
        """
        for item in transaction:
            if item not in self.__transaction_index_map:
                self.__items.append(item)
                self.__transaction_index_map[item] = set()
            self.__transaction_index_map[item].add(self.__num_transaction)
        self.__num_transaction += 1

    def calc_support(self, items):
        """
        Returns a support for items.
        """
        # Empty items is supported by all transactions.
        if not items:
            return 1.0

        # Create the transaction index intersection.
        sum_indexes = None
        for item in items:
            indexes = self.__transaction_index_map.get(item)
            if indexes is None:
                # No support for any set that contains a not existing item.
                return 0.0

            if sum_indexes is None:
                sum_indexes = indexes
            else:
                sum_indexes = sum_indexes.intersection(indexes)

        # Calculate the support.
        return float(len(sum_indexes)) / self.__num_transaction

    def initial_candidates(self):
        """
        Returns the initial candidates.
        """
        return [frozenset([item]) for item in self.__items]

    @property
    def num_transaction(self):
        """
        Returns the number of transactions.
        """
        return self.__num_transaction

    @property
    def items(self):
        """
        Returns the item list that the transaction is consisted of.
        """
        return self.__items

    @staticmethod
    def create(transactions):
        """
        Create the TransactionManager with a transaction instance.
        If the given instance is a TransactionManager, this returns itself.
        """
        if isinstance(transactions, TransactionManager):
            return transactions
        return TransactionManager(transactions)

def create_next_candidates(prev_candidates, length):
    """
    Returns the apriori candidates.
    """
    # Solve the items.
    items = set()
    for candidate in prev_candidates:
        for item in candidate:
            items.add(item)

    def check_subsets(candidate):
        """
        Check if the subsets of a candidate is present
        in the previous candidates.
        """
        candidate_subsets = [
            frozenset(x) for x in combinations(candidate, length - 1)]
        for candidate_subset in candidate_subsets:
            if candidate_subset not in prev_candidates:
                return False
        return True

    # Create candidates.
    next_candidates = []
    for candidate in [frozenset(x) for x in combinations(items, length)]:
        if length > 2 and not check_subsets(candidate):
            continue
        next_candidates.append(candidate)
    return next_candidates

def gen_support_records(
        transaction_manager,
        min_support,
        max_length=None,
        _generate_candidates_func=create_next_candidates):
    """
    Returns a generator of support records with given transactions.
    """
    candidates = transaction_manager.initial_candidates()
    length = 1
    while candidates:
        relations = set()
        for relation_candidate in candidates:
            support = transaction_manager.calc_support(relation_candidate)
            if support < min_support:
                continue
            candidate_set = frozenset(relation_candidate)
            relations.add(candidate_set)
            yield SupportRecord(candidate_set, support)
        length += 1
        if max_length and length > max_length:
            break
        candidates = _generate_candidates_func(relations, length)

def gen_ordered_statistics(transaction_manager, record):
    """
    Returns a generator of ordered statistics.
    """
    items = record.items
    combination_sets = [
        frozenset(x) for x in combinations(items, len(items) - 1)]
    for items_base in combination_sets:
        items_add = frozenset(items.difference(items_base))
        confidence = (
            record.support / transaction_manager.calc_support(items_base))
        lift = confidence / transaction_manager.calc_support(items_add)
        yield OrderedStatistic(
            frozenset(items_base), frozenset(items_add), confidence, lift)

def apriori(transactions, **kwargs):
    """
    Run Apriori algorithm.

    @param  transactions    A list of transactions.
    @param  min_support     The minimum support of the relation (float).
    @param  max_length      The maximum length of the relation (integer).
    """
    min_support = kwargs.get('min_support', 0.1)
    max_length = kwargs.get('max_length', None)
    min_confidence = kwargs.get('min_confidence', 0.0)
    _gen_support_records = kwargs.get(
        '_gen_support_records', gen_support_records)
    _gen_ordered_statistics = kwargs.get(
        '_gen_ordered_statistics', gen_ordered_statistics)

    # Calculate supports.
    transaction_manager = TransactionManager.create(transactions)
    support_records = _gen_support_records(
        transaction_manager, min_support, max_length)

    # Calculate ordered stats.
    for support_record in support_records:
        ordered_statistics = _gen_ordered_statistics(
            transaction_manager, support_record)
        filtered_ordered_statistics = [
            x for x in ordered_statistics if x.confidence >= min_confidence]
        if not filtered_ordered_statistics:
            continue
        yield RelationRecord(
            support_record.items, support_record.support,
            filtered_ordered_statistics)

def dump_as_json(record, output_file):
    """
    Dump an relation record as a json value.

    @param  record      A record.
    @param  output_file An output file.
    """
    def default_func(value):
        """
        Default conversion for JSON value.
        """
        if isinstance(value, frozenset):
            return sorted(value)
        raise TypeError(repr(value) + " is not JSON serializable")

    converted_record = record._replace(
        ordered_statistics=[x._asdict() for x in record.ordered_statistics])
    output_file.write(
        json.dumps(converted_record._asdict(), default=default_func))
    output_file.write('\n')

def dump_as_two_item_tsv(record, output_file):
    """
    Dump a relation record as TSV only for 2 item relations.

    @param  record      A record.
    @param  output_file An output file.
    """
    for ordered_stats in record.ordered_statistics:
        if len(ordered_stats.items_base) != 1:
            return
        if len(ordered_stats.items_add) != 1:
            return
        output_file.write('{0}\t{1}\t{2:.8f}\t{3:.8f}\t{4:.8f}\n'.format(
            list(ordered_stats.items_base)[0], list(ordered_stats.items_add)[0],
            record.support, ordered_stats.confidence, ordered_stats.lift))

def parse_args(argv):
    """
    Parse commandline arguments.
    """
    output_funcs = {
        'json': dump_as_json,
        'tsv': dump_as_two_item_tsv,
    }
    default_output_func_key = 'json'

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-v', '--version', action='version',
        version='%(prog)s {0}'.format(__version__))
    parser.add_argument(
        'input', metavar='inpath', nargs='*',
        help='Input transaction file (default: stdin).',
        type=argparse.FileType('r'), default=[sys.stdin])
    parser.add_argument(
        '-o', '--output', metavar='outpath',
        help='Output file (default: stdout).',
        type=argparse.FileType('w'), default=sys.stdout)
    parser.add_argument(
        '-l', '--max-length', metavar='int',
        help='Max length of relations (default: infinite).',
        type=int, default=None)
    parser.add_argument(
        '-s', '--min-support', metavar='float',
        help='Minimum support ratio (must be > 0, default: 0.1).',
        type=float, default=0.1)
    parser.add_argument(
        '-c', '--min-confidence', metavar='float',
        help='Minimum confidence (default: 0.5).',
        type=float, default=0.5)
    parser.add_argument(
        '-d', '--delimiter', metavar='str',
        help='Delimiter for items of transactions (default: tab).',
        type=str, default='\t')
    parser.add_argument(
        '-f', '--out-format', help='Output format (default or tsv).',
        metavar='str', type=str, choices=output_funcs.keys(),
        default=default_output_func_key)
    args = parser.parse_args(argv)
    if args.min_support <= 0:
        raise ValueError('min support must be > 0')

    args.output_func = output_funcs[args.out_format]
    return args

def main():
    """
    Main.
    """
    args = parse_args(sys.argv[1:])

    transactions = [
        line.strip().split(args.delimiter) for line in chain(*args.input)]
    result = apriori(
        transactions,
        max_length=args.max_length,
        min_support=args.min_support,
        min_confidence=args.min_confidence)

    for record in result:
        args.output_func(record, args.output)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2015 Ramil Nugmanov <stsouko@live.ru>
# This file is part of PREDICTOR.
#
# PREDICTOR is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
from math import sqrt, ceil
import numpy as np
import xmltodict as x2d

def getmodelset(conffile):
    xml = x2d.parse(open(conffile, 'r').read())['models']
    batch = xml['model']
    conf = xml.get('conf', {})
    models = {}

    if not isinstance(batch, list):
        batch = [batch]
    for x in batch:
        name = x['name']
        execlist = []
        scripts = x['scripts']['script']
        if not isinstance(scripts, list):
            scripts = [scripts]

        for y in scripts:
            params = y['params']['param']
            if not isinstance(params, list):
                params = [params]
            plist = []
            for z in params:
                plist.extend(z['name'].split() if 'type' in z and z['type'] == 'list' else [z['name']])

            execlist.append([y['exec_path']] + plist)

        models[name] = execlist

    return models, conf

class consensus_dragos():
    def __init__(self):
        self.__TRUST = 5
        self.__INlist = []
        self.__ALLlist = []
        super().__init__()

    def cumulate(self, P, AD):
        if AD:
            self.__INlist.append(P)
        self.__ALLlist.append(P)

    def report(self, units=None):
        if not self.__ALLlist:
            return False  # break if all models fails to predict

        reason = []
        result = []
        INarr = np.array(self.__INlist)
        ALLarr = np.array(self.__ALLlist)

        PavgALL = ALLarr.mean()
        sigmaALL = sqrt((ALLarr ** 2).mean() - PavgALL ** 2)

        if self.__INlist:
            PavgIN = INarr.mean()
            sigmaIN = sqrt((INarr ** 2).mean() - PavgIN ** 2)
            pavgdiff = PavgIN - PavgALL
            if pavgdiff > self.TOL:
                reason.append(self.__errors.get('diff', '%.2f') % pavgdiff)
                self.__TRUST -= 1
        else:
            self.__TRUST -= 1
            reason.append(self.__errors.get('zad', ''))

        proportion = len(self.__INlist) / len(self.__ALLlist)
        if proportion > self.Nlim:
            sigma = sigmaIN
            Pavg = PavgIN
        else:
            sigma = sigmaALL
            Pavg = PavgALL
            self.__TRUST -= 1
            if self.__INlist:
                reason.append(self.__errors.get('lad', '%d') % ceil(100 * proportion))

        proportion = sigma / self.TOL
        if proportion > 1:
            self.__TRUST -= int(proportion)
            reason.append(self.__errors.get('stp', '%d') % (proportion * 100 - 100))

        result.append(dict(type='text', attrib='predicted value ± sigma%s' % (' (%s)' % units if units else ''),
                           value='%.2f ± %.2f' % (Pavg, sigma)))
        result.append(dict(type='text', attrib='prediction trust', value=self.__trustdesc.get(self.__TRUST, 'None')))
        if reason:
            result.append(dict(type='text', attrib='reason', value='
'.join(reason)))

        return result

    __errors = dict(lad='Too few (less than %d %%) local models have applicability domains covering this structure',
                    diff='The other local models disagree (prediction value difference = %.2f) with the prediction of '
                         'the minority containing structure inside their applicability domain',
                    stp='Individual models failed to reach unanimity - prediction variance exceeds %d %% '
                        'of the property range width',
                    zad='None of the local models have applicability domains covering this structure')

    __trustdesc = {5: 'Optimal', 4: 'Good', 3: 'Medium', 2: 'Low'}

def bondbox(boxfile, descfile, dftype):
    box = {}
    AD = True
    with open(boxfile) as f:
        for line in f:
            fragment, *vrange = line.split()
            box[int(fragment)] = [int(x) for x in vrange]

    with open(descfile) as f:
        if dftype == 'svm':
            for fragment in f.read().split()[1:]:
                pos, count = fragment.split(':')
                min, max = box.get(int(pos), [0, 0])
                if not (min <= int(count) <= max):
                    AD = False
                    break
        elif dftype == 'csv':
            for pos, count in enumerate(f.read().split(';')[1:]):
                min, max = box.get(pos + 1, [0, 0])
                if not (min <= int(count) <= max):
                    AD = False
                    break
        else:
            AD = False
    return AD

<commit_msg>fix box<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2015 Ramil Nugmanov <stsouko@live.ru>
# This file is part of PREDICTOR.
#
# PREDICTOR is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
#  MA 02110-1301, USA.
#
from math import sqrt, ceil
import numpy as np
import xmltodict as x2d

def getmodelset(conffile):
    xml = x2d.parse(open(conffile, 'r').read())['models']
    batch = xml['model']
    conf = xml.get('conf', {})
    models = {}

    if not isinstance(batch, list):
        batch = [batch]
    for x in batch:
        name = x['name']
        execlist = []
        scripts = x['scripts']['script']
        if not isinstance(scripts, list):
            scripts = [scripts]

        for y in scripts:
            params = y['params']['param']
            if not isinstance(params, list):
                params = [params]
            plist = []
            for z in params:
                plist.extend(z['name'].split() if 'type' in z and z['type'] == 'list' else [z['name']])

            execlist.append([y['exec_path']] + plist)

        models[name] = execlist

    return models, conf

class consensus_dragos():
    def __init__(self):
        self.__TRUST = 5
        self.__INlist = []
        self.__ALLlist = []
        super().__init__()

    def cumulate(self, P, AD):
        if AD:
            self.__INlist.append(P)
        self.__ALLlist.append(P)

    def report(self, units=None):
        if not self.__ALLlist:
            return False  # break if all models fails to predict

        reason = []
        result = []
        INarr = np.array(self.__INlist)
        ALLarr = np.array(self.__ALLlist)

        PavgALL = ALLarr.mean()
        sigmaALL = sqrt((ALLarr ** 2).mean() - PavgALL ** 2)

        if self.__INlist:
            PavgIN = INarr.mean()
            sigmaIN = sqrt((INarr ** 2).mean() - PavgIN ** 2)
            pavgdiff = PavgIN - PavgALL
            if pavgdiff > self.TOL:
                reason.append(self.__errors.get('diff', '%.2f') % pavgdiff)
                self.__TRUST -= 1
        else:
            self.__TRUST -= 1
            reason.append(self.__errors.get('zad', ''))

        proportion = len(self.__INlist) / len(self.__ALLlist)
        if proportion > self.Nlim:
            sigma = sigmaIN
            Pavg = PavgIN
        else:
            sigma = sigmaALL
            Pavg = PavgALL
            self.__TRUST -= 1
            if self.__INlist:
                reason.append(self.__errors.get('lad', '%d') % ceil(100 * proportion))

        proportion = sigma / self.TOL
        if proportion > 1:
            self.__TRUST -= int(proportion)
            reason.append(self.__errors.get('stp', '%d') % (proportion * 100 - 100))

        result.append(dict(type='text', attrib='predicted value ± sigma%s' % (' (%s)' % units if units else ''),
                           value='%.2f ± %.2f' % (Pavg, sigma)))
        result.append(dict(type='text', attrib='prediction trust', value=self.__trustdesc.get(self.__TRUST, 'None')))
        if reason:
            result.append(dict(type='text', attrib='reason', value='
'.join(reason)))

        return result

    __errors = dict(lad='Too few (less than %d %%) local models have applicability domains covering this structure',
                    diff='The other local models disagree (prediction value difference = %.2f) with the prediction of '
                         'the minority containing structure inside their applicability domain',
                    stp='Individual models failed to reach unanimity - prediction variance exceeds %d %% '
                        'of the property range width',
                    zad='None of the local models have applicability domains covering this structure')

    __trustdesc = {5: 'Optimal', 4: 'Good', 3: 'Medium', 2: 'Low'}

def bondbox(boxfile, descfile, dftype):
    box = {}
    AD = True
    with open(boxfile) as f:
        for line in f:
            fragment, *vrange = line.split()
            box[int(fragment)] = [float(x) for x in vrange]

    with open(descfile) as f:
        if dftype == 'svm':
            for fragment in f.read().split()[1:]:
                pos, count = fragment.split(':')
                min, max = box.get(int(pos), [0, 0])
                if not (min <= float(count) <= max):
                    AD = False
                    break
        elif dftype == 'csv':
            for pos, count in enumerate(f.read().split(';')[1:]):
                min, max = box.get(pos + 1, [0, 0])
                if not (min <= float(count) <= max):
                    AD = False
                    break
        else:
            AD = False
    return AD

<|endoftext|>


<commit_before># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Representation of mathematical operators."""

from __future__ import absolute_import

__all__ = ()

from . import operator
from .operator import *
__all__ += operator.__all__

from . import default_ops
from .default_ops import *
__all__ += default_ops.__all__

from . import pspace_ops
from .pspace_ops import *
__all__ += pspace_ops.__all__

from . import oputils
from .oputils import *
__all__ += oputils.__all__<commit_msg>MAINT: pep8 fix in odl/operator/__init__<commit_after># Copyright 2014-2016 The ODL development group
#
# This file is part of ODL.
#
# ODL is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ODL is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ODL.  If not, see <http://www.gnu.org/licenses/>.

"""Representation of mathematical operators."""

from __future__ import absolute_import

__all__ = ()

from . import operator
from .operator import *
__all__ += operator.__all__

from . import default_ops
from .default_ops import *
__all__ += default_ops.__all__

from . import pspace_ops
from .pspace_ops import *
__all__ += pspace_ops.__all__

from . import oputils
from .oputils import *
__all__ += oputils.__all__
<|endoftext|>


<commit_before>import collections

class GenericType():
    def __init__(self, value = 0):
       self.value = value

    def __str__(self):
        return str(self.value)

    def pack(self):
        """ Pack a value into a binary buffer."""
        return pack(self.fmt, self.value)

    def unpack(self, buff, offset=0):
        """ Unpack a buff and stores at value property. """
        try:
            self.value = unpack_from(self.fmt, buff, offset)[0]
        except error as e:
            raise OFPException("Error while unpack data from buffer")

    def get_size(self):
        """ Return the size of type in bytes. """
        return calcsize(self.fmt)

class MetaStruct(type):
    @classmethod
    def __prepare__(self, name, bases):
        return collections.OrderedDict()

    def __new__(self, name, bases, classdict):
        classdict['__ordered__'] = [(key, type(value)) for key, value in
                            classdict.items() if key not in
                            ('__module__','__qualname__')]
        return type.__new__(self, name, bases, classdict)

class GenericStruct(metaclass=MetaStruct):
    def __init__(self, *args, **kwargs):
        for _attr, _class in self.__ordered__:
            if not callable(getattr(self, _attr)):
                # TODO: Validade data
                try:
                    setattr(self, _attr, kwargs[_attr])
                except KeyError:
                    pass

    def get_size(self):
        tot = 0
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            if _class is OFPHeader:
                tot += (getattr(self, _attr).get_size())
            elif not callable(attr):
                tot += (_class(attr).get_size())
        return tot

    def pack(self):
        hex = b''
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            if _class is OFPHeader:
                hex += getattr(self, _attr).pack()
                #print("{} {} {}".
                #      format(_attr, attr,getattr(self, _attr).pack()))
            elif not callable(attr):
                hex += _class(attr).pack()
                #print("{} {} {}"
                #      .format(_attr, attr,_class(attr).pack()))
        return hex

    def unpack(self, buff):
        begin = 0
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            if _class is OFPHeader:
                size = (getattr(self, _attr).get_size())
                getattr(self,_attr).unpack(buff, offset=begin)
            elif not callable(attr):
                size = (_class(attr).get_size())
                getattr(self,_attr).unpack(buff, offset=begin)
            begin += size

<commit_msg>Small refactor on common/base.py<commit_after>"""Contains basic and fundamental classes and constants"""

# System imports
import collections
import struct

# Third-party imports

# Local source tree imports
from common import exceptions

# CONSTANTS
OFP_ETH_ALEN = 6
OFP_MAX_PORT_NAME_LEN = 16
OFP_VERSION = 0x02

# CLASSES

class GenericType():
    def __init__(self, value = 0):
       self.value = value

    def __str__(self):
        return str(self.value)

    def pack(self):
        """ Pack a value into a binary buffer."""
        return struct.pack(self.fmt, self.value)

    def unpack(self, buff, offset=0):
        """ Unpack a buff and stores at value property. """
        try:
            self.value = struct.unpack_from(self.fmt, buff, offset)[0]
        except struct.error:
            raise exceptions.OFPException("Error while unpack data from buffer")

    def get_size(self):
        """ Return the size of type in bytes. """
        return struct.calcsize(self.fmt)

class MetaStruct(type):
    @classmethod
    def __prepare__(self, name, bases):
        return collections.OrderedDict()

    def __new__(self, name, bases, classdict):
        classdict['__ordered__'] = [(key, type(value)) for key, value in
                            classdict.items() if key not in
                            ('__module__','__qualname__')]
        return type.__new__(self, name, bases, classdict)

class GenericStruct(metaclass=MetaStruct):
    def __init__(self, *args, **kwargs):
        for _attr, _class in self.__ordered__:
            if not callable(getattr(self, _attr)):
                # TODO: Validade data
                try:
                    setattr(self, _attr, kwargs[_attr])
                except KeyError:
                    pass

    def get_size(self):
        tot = 0
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            #TODO: Ciclic reference
            if _class is OFPHeader:
                tot += (getattr(self, _attr).get_size())
            elif not callable(attr):
                tot += (_class(attr).get_size())
        return tot

    def pack(self):
        hex = b''
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            #TODO: Ciclic reference
            if _class is OFPHeader:
                hex += getattr(self, _attr).pack()
                #print("{} {} {}".
                #      format(_attr, attr,getattr(self, _attr).pack()))
            elif not callable(attr):
                hex += _class(attr).pack()
                #print("{} {} {}"
                #      .format(_attr, attr,_class(attr).pack()))
        return hex

    def unpack(self, buff):
        begin = 0
        for _attr, _class in self.__ordered__:
            attr = getattr(self, _attr)
            #TODO: Ciclic reference
            if _class is OFPHeader:
                size = (getattr(self, _attr).get_size())
                getattr(self,_attr).unpack(buff, offset=begin)
            elif not callable(attr):
                size = (_class(attr).get_size())
                getattr(self,_attr).unpack(buff, offset=begin)
            begin += size

<|endoftext|>