<commit_before># -*- coding: utf-8 -*-
#
#  This file is part of TinyIDS.
#
#  TinyIDS is a distributed Intrusion Detection System (IDS) for Unix systems.
#
#  Project development web site:
#
#      http://www.codetrax.org/projects/tinyids
#
#  Copyright (c) 2010 George Notaras, G-Loaded.eu, CodeTRAX.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import os
import logging
import glob
import socket
import getpass
import time

import TinyIDS.backends
from TinyIDS import config
from TinyIDS import crypto
from TinyIDS.util import sha1, load_backend

logger = logging.getLogger()

class NoBackendsToRun(Exception):
    pass

class TinyIDSClient:
    """A client implementation of the TinyIDS protocol."""

    cmd_end = '\r\n'
    max_response_len = 1024

    def __init__(self, command):

        # Client configuration (config.TinyIDSConfigParser instance)
        self.cfg = config.get_client_configuration()

        # Holds the current command
        self.command = command  # TEST | CHECK | UPDATE | DELETE | CHANGEPHRASE

        # Default hashing delay
        self.default_hashing_delay = self._get_hashing_delay()

        # Debug protocol
        self.debug_protocol = self.cfg.getboolean('main', 'debug_protocol')

        # Data hashing object
        self.hasher = sha1()
        # Immediately hash the machine's hostname (issue: #248)
        self.hash_data(socket.gethostname())

        # PKI Module
        _keys_dir = self.cfg.get('main', 'keys_dir')
        self.pki = crypto.RSAModule(_keys_dir)

        # This is socket.socket object while connected to server
        # Should be set to None as soon the connection is closed
        # or a socket error occurs.
        self.sock = None

        # Should hold the name of the server as long as there is a
        # valid connection to it.
        self.server_name = None

    def _close_socket(self):
        """Should be called after all socket errors."""
        if isinstance(self.sock, socket.socket):
            self.sock.close()
            if self.server_name:
                logger.info('- Closed connection with server: %s' % self.server_name)
        self.sock = None
        self.server_name = None

    def _get_hashing_delay(self):
        """Returns the hashing delay in seconds."""
        delay_msec = self.cfg.getfloat('main', 'hashing_delay')
        delay_sec = delay_msec / 1000
        logger.debug('Hashing delay set to %.3f seconds' % delay_sec)
        return delay_sec

    def _run_backends(self):
        """Runs all collector backends and passes the yielded information
        through the hashing algorithm."""

        backends_conf_dir = self.cfg.get('main', 'backends_conf_dir')

        # Get a list of all backend paths
        backend_paths = []
        core_backends_dir = TinyIDS.backends.__path__[0]
        backend_paths.extend(glob.glob(os.path.join(core_backends_dir, '*.py')))
        extra_backends_dir = self.cfg.get('main', 'extra_backends_dir')
        backend_paths.extend(glob.glob(os.path.join(extra_backends_dir, '*.py')))

        if not backend_paths:
            raise NoBackendsToRun

        #
        user_defined_backend_list = self.cfg.getlist('main', 'tests')
        user_defined_backend_list_finished = [] # holds names of backends that have finished
        if user_defined_backend_list:
            logger.debug('Using user-defined list of backends')

        # Load all needed backends and store them in a list
        for backend_path in backend_paths:
            backend_name = os.path.basename(backend_path)[:-3]
            backend_dir = os.path.dirname(backend_path)
            if backend_name == '__init__':
                continue
            elif user_defined_backend_list:
                #print "checking user list"
                if not backend_name in user_defined_backend_list:
                    logger.debug('Skipping backend: %s' % backend_name)
                    continue
            # Load backend
            m = load_backend(backend_dir, backend_name)
            if not hasattr(m, 'CollectorBackend'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            logger.info('Processing backend: %s' % backend_name)

            backend_config_file = os.path.join(backends_conf_dir, m.__name__ + '.conf')
            b = m.CollectorBackend(config_path=backend_config_file)
            if not hasattr(b, 'collect'):
                logger.error('Invalid TinyIDS backend: %s' % backend_path)
                continue

            # Collect information
            for data in b.collect():
                self.hash_data(data)
            logger.info('%s: Complete' % backend_name)

            if user_defined_backend_list:
                # If a user-defined list of backends is used, add the name of
                # the backend that has just run to the list:
                # user_defined_backend_list_finished
                user_defined_backend_list_finished.append(backend_name)

        if user_defined_backend_list:
            invalid_user_defined_tests = []
            for test in user_defined_backend_list:
                if test not in user_defined_backend_list_finished:
                    invalid_user_defined_tests.append(test)
            if invalid_user_defined_tests:
                logger.warning('Invalid user-defined test(s): %s' % ', '.join(invalid_user_defined_tests))
                if not user_defined_backend_list_finished:
                    raise NoBackendsToRun

    def _get_server_canonical_name(self, server_name):
        """Returns the name of the server after stripping the 'server__' prefix'"""
        return server_name.split('__')[1]

    def _get_enabled_server_list(self):
        """Returns a list of servers that have been enabled in the
        client configuration."""
        enabled_servers = []
        for section in self.cfg.sections():
            if not section.startswith('server__'):
                continue
            server_name = self._get_server_canonical_name(section)
            if not self.cfg.has_option(section, 'host'):
                logger.warning('Misconfigured server: %s' % server_name)
                continue
            if self.cfg.has_option(section, 'enabled'):
                is_enabled = self.cfg.getboolean(section, 'enabled')
                if not is_enabled:
                    continue
                enabled_servers.append(section)
        return enabled_servers

    def _send(self, host, port, data):
        """Sends the command to a TinyIDS server."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((host, port))
        logger.info('- Established connection to server: %s' % self.server_name)
        if self.debug_protocol:
            logger.debug('-> Sending command: %s' % data)
        if self.pki.public_key is not None:
            data = self.pki.encrypt(data)
            logger.info('- PKI: data encrypted')
        self.sock.send(data + self.cmd_end)
        logger.info('- Sent %s command to server: %s' % (self.command, self.server_name))

    def _get_server_response(self):
        """Receives a response from a TinyIDS server."""
        logger.info('- Awaiting server response...')
        response = self.sock.recv(self.max_response_len).strip()
        response = response.rstrip(self.cmd_end)
        logger.info('- Received response from server: %s' % self.server_name)
        if self.pki.public_key is not None:
            response = self.pki.verify(response)
            logger.info('- PKI: data verified')
        if self.debug_protocol:
            logger.debug('-> Received response: %s' % response.strip())
        return response.strip()

    def _check_command_status(self, response):
        if response.startswith('20'):
            logger.info('- RESULT: %s on %s: SUCCESS' % (self.command, self.server_name))
        else:
            logger.warning('- RESULT: %s on %s: FAILURE with error: %s' % (self.command, self.server_name, response))

    def _communicate(self, host, port, data):
        self._send(host, port, data)
        response = self._get_server_response()
        self._check_command_status(response)

    def _get_passphrase(self, msg):
        """Prompts the user for a passphrase."""
        data = ''
        while not data:
            data = getpass.getpass('%s: ' % msg)
        return data

    def _com_TEST(self, host, port):
        """
        Syntax: TEST <protocol_revision>
        """
        data = '%s %s' % (self.command, config.PROTOCOL_REVISION)
        self._communicate(host, port, data)

    def _com_CHECK(self, host, port):
        """
        Syntax: CHECK <hash>
        """
        data = '%s %s' % (self.command, self.get_checksum())
        self._communicate(host, port, data)

    def _com_UPDATE(self, host, port):
        """
        Syntax: UPDATE <hash> <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s %s' % (self.command, self.get_checksum(), passphrase)
        self._communicate(host, port, data)

    def _com_DELETE(self, host, port):
        """
        Syntax: DELETE <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s' % (self.command, passphrase)
        self._communicate(host, port, data)

    def _com_CHANGEPHRASE(self, host, port):
        """
        Syntax: CHANGEPHRASE <old_passphrase> <new_passphrase>
        """
        passphrase_old = self._get_passphrase('Old passphrase')
        while True:
            passphrase_new = self._get_passphrase('New passphrase')
            passphrase_new_confirm = self._get_passphrase('Confirm new passphrase')
            if passphrase_new == passphrase_new_confirm:
                break
            logger.error('passphrases do not match. try again...')
        data = '%s %s %s' % (self.command, passphrase_old, passphrase_new)
        self._communicate(host, port, data)

    # Public API

    def get_checksum(self):
        return self.hasher.hexdigest()

    def hash_data(self, data):
        """Passes data through the hashing algorithm."""
        self.hasher.update(data)
        time.sleep(self.default_hashing_delay)

    def run(self):
        """Main client method."""

        logger.info('Getting list of enabled servers')
        enabled_servers = self._get_enabled_server_list()
        if not enabled_servers:
            logger.warning('No servers configured. Shutting down...')
            self.client_close()
            return
        logger.info('Servers found. Proceeding with data hashing...')

        # Decide which method to execute
        func = getattr(self, '_com_%s' % self.command)

        # Tests are required to run only with the CHECK and UPDATE commands
        if self.command in ('CHECK', 'UPDATE'):
            logger.info('Hashing data. Please wait...')
            try:
                self._run_backends()
            except NoBackendsToRun:
                logger.warning('No valid backends. Shutting down...')
                self.client_close()
                return
            else:
                logger.info('Data hashing complete')

        # Execute command on server
        logger.info('Preparing to contact servers')
        for server in enabled_servers:
            # server is in 'server__<name>' format (a section name)
            # Here we set self.server_name to the canonical server name
            self.server_name = self._get_server_canonical_name(server)
            logger.info('* Contacting server: %s' % self.server_name)

            # Get server settings
            host = self.cfg.get(server, 'host')
            port = config.DEFAULT_PORT
            if self.cfg.has_option(server, 'port'):
                port = self.cfg.getint(server, 'port')
            if self.cfg.has_option(server, 'public_key'):
                public_key_fname = self.cfg.get(server, 'public_key')
                if public_key_fname:
                    try:
                        self.pki.load_external_public_key(public_key_fname) # sets self.pki.public_key
                    except crypto.InvalidPublicKey:
                        logger.warning('- RESULT: %s on %s: FAILURE with PKI error: invalid server public key: %s' % (self.command, self.server_name, public_key_fname))
                        self.server_name = None
                        continue

            # Run command on the server
            try:
                func(host, port)
            except socket.error, (errno, strerror):
                logger.error('Connection error: \'%s\'' % strerror)
                logger.warning('Skipping server: %s' % self.server_name)
                self.server_name = None
            except crypto.DataEncryptionError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not encrypt data for server' % (self.command, self.server_name))
            except crypto.DataVerificationError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not verify response from server' % (self.command, self.server_name))

            self._close_socket()
            self.pki.reset()    # sets self.pki.public_key to None

        logger.info('Finished with servers')

        self.client_close()

    def client_close(self):
        logger.info('Client shutting down...')

<commit_msg>Explained a bit better why we hash the hostname by default.<commit_after># -*- coding: utf-8 -*-
#
#  This file is part of TinyIDS.
#
#  TinyIDS is a distributed Intrusion Detection System (IDS) for Unix systems.
#
#  Project development web site:
#
#      http://www.codetrax.org/projects/tinyids
#
#  Copyright (c) 2010 George Notaras, G-Loaded.eu, CodeTRAX.org
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import os
import logging
import glob
import socket
import getpass
import time

import TinyIDS.backends
from TinyIDS import config
from TinyIDS import crypto
from TinyIDS.util import sha1, load_backend

logger = logging.getLogger()

class NoBackendsToRun(Exception):
    pass

class TinyIDSClient:
    """A client implementation of the TinyIDS protocol."""

    cmd_end = '\r\n'
    max_response_len = 1024

    def __init__(self, command):

        # Client configuration (config.TinyIDSConfigParser instance)
        self.cfg = config.get_client_configuration()

        # Holds the current command
        self.command = command  # TEST | CHECK | UPDATE | DELETE | CHANGEPHRASE

        # Default hashing delay
        self.default_hashing_delay = self._get_hashing_delay()

        # Debug protocol
        self.debug_protocol = self.cfg.getboolean('main', 'debug_protocol')

        # Data hashing object
        self.hasher = sha1()
        # Immediately hash the machine's hostname to avoid possible
        # identical hashes among identical machines (issue: #248)
        self.hash_data(socket.gethostname())

        # PKI Module
        _keys_dir = self.cfg.get('main', 'keys_dir')
        self.pki = crypto.RSAModule(_keys_dir)

        # This is socket.socket object while connected to server
        # Should be set to None as soon the connection is closed
        # or a socket error occurs.
        self.sock = None

        # Should hold the name of the server as long as there is a
        # valid connection to it.
        self.server_name = None

    def _close_socket(self):
        """Should be called after all socket errors."""
        if isinstance(self.sock, socket.socket):
            self.sock.close()
            if self.server_name:
                logger.info('- Closed connection with server: %s' % self.server_name)
        self.sock = None
        self.server_name = None

    def _get_hashing_delay(self):
        """Returns the hashing delay in seconds."""
        delay_msec = self.cfg.getfloat('main', 'hashing_delay')
        delay_sec = delay_msec / 1000
        logger.debug('Hashing delay set to %.3f seconds' % delay_sec)
        return delay_sec

    def _run_backends(self):
        """Runs all collector backends and passes the yielded information
        through the hashing algorithm."""

        backends_conf_dir = self.cfg.get('main', 'backends_conf_dir')

        # Get a list of all backend paths
        backend_paths = []
        core_backends_dir = TinyIDS.backends.__path__[0]
        backend_paths.extend(glob.glob(os.path.join(core_backends_dir, '*.py')))
        extra_backends_dir = self.cfg.get('main', 'extra_backends_dir')
        backend_paths.extend(glob.glob(os.path.join(extra_backends_dir, '*.py')))

        if not backend_paths:
            raise NoBackendsToRun

        #
        user_defined_backend_list = self.cfg.getlist('main', 'tests')
        user_defined_backend_list_finished = [] # holds names of backends that have finished
        if user_defined_backend_list:
            logger.debug('Using user-defined list of backends')

        # Load all needed backends and store them in a list
        for backend_path in backend_paths:
            backend_name = os.path.basename(backend_path)[:-3]
            backend_dir = os.path.dirname(backend_path)
            if backend_name == '__init__':
                continue
            elif user_defined_backend_list:
                #print "checking user list"
                if not backend_name in user_defined_backend_list:
                    logger.debug('Skipping backend: %s' % backend_name)
                    continue
            # Load backend
            m = load_backend(backend_dir, backend_name)
            if not hasattr(m, 'CollectorBackend'):
                logger.warning('Skipping invalid backend: %s' % backend_path)
                continue

            logger.info('Processing backend: %s' % backend_name)

            backend_config_file = os.path.join(backends_conf_dir, m.__name__ + '.conf')
            b = m.CollectorBackend(config_path=backend_config_file)
            if not hasattr(b, 'collect'):
                logger.error('Invalid TinyIDS backend: %s' % backend_path)
                continue

            # Collect information
            for data in b.collect():
                self.hash_data(data)
            logger.info('%s: Complete' % backend_name)

            if user_defined_backend_list:
                # If a user-defined list of backends is used, add the name of
                # the backend that has just run to the list:
                # user_defined_backend_list_finished
                user_defined_backend_list_finished.append(backend_name)

        if user_defined_backend_list:
            invalid_user_defined_tests = []
            for test in user_defined_backend_list:
                if test not in user_defined_backend_list_finished:
                    invalid_user_defined_tests.append(test)
            if invalid_user_defined_tests:
                logger.warning('Invalid user-defined test(s): %s' % ', '.join(invalid_user_defined_tests))
                if not user_defined_backend_list_finished:
                    raise NoBackendsToRun

    def _get_server_canonical_name(self, server_name):
        """Returns the name of the server after stripping the 'server__' prefix'"""
        return server_name.split('__')[1]

    def _get_enabled_server_list(self):
        """Returns a list of servers that have been enabled in the
        client configuration."""
        enabled_servers = []
        for section in self.cfg.sections():
            if not section.startswith('server__'):
                continue
            server_name = self._get_server_canonical_name(section)
            if not self.cfg.has_option(section, 'host'):
                logger.warning('Misconfigured server: %s' % server_name)
                continue
            if self.cfg.has_option(section, 'enabled'):
                is_enabled = self.cfg.getboolean(section, 'enabled')
                if not is_enabled:
                    continue
                enabled_servers.append(section)
        return enabled_servers

    def _send(self, host, port, data):
        """Sends the command to a TinyIDS server."""
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.sock.connect((host, port))
        logger.info('- Established connection to server: %s' % self.server_name)
        if self.debug_protocol:
            logger.debug('-> Sending command: %s' % data)
        if self.pki.public_key is not None:
            data = self.pki.encrypt(data)
            logger.info('- PKI: data encrypted')
        self.sock.send(data + self.cmd_end)
        logger.info('- Sent %s command to server: %s' % (self.command, self.server_name))

    def _get_server_response(self):
        """Receives a response from a TinyIDS server."""
        logger.info('- Awaiting server response...')
        response = self.sock.recv(self.max_response_len).strip()
        response = response.rstrip(self.cmd_end)
        logger.info('- Received response from server: %s' % self.server_name)
        if self.pki.public_key is not None:
            response = self.pki.verify(response)
            logger.info('- PKI: data verified')
        if self.debug_protocol:
            logger.debug('-> Received response: %s' % response.strip())
        return response.strip()

    def _check_command_status(self, response):
        if response.startswith('20'):
            logger.info('- RESULT: %s on %s: SUCCESS' % (self.command, self.server_name))
        else:
            logger.warning('- RESULT: %s on %s: FAILURE with error: %s' % (self.command, self.server_name, response))

    def _communicate(self, host, port, data):
        self._send(host, port, data)
        response = self._get_server_response()
        self._check_command_status(response)

    def _get_passphrase(self, msg):
        """Prompts the user for a passphrase."""
        data = ''
        while not data:
            data = getpass.getpass('%s: ' % msg)
        return data

    def _com_TEST(self, host, port):
        """
        Syntax: TEST <protocol_revision>
        """
        data = '%s %s' % (self.command, config.PROTOCOL_REVISION)
        self._communicate(host, port, data)

    def _com_CHECK(self, host, port):
        """
        Syntax: CHECK <hash>
        """
        data = '%s %s' % (self.command, self.get_checksum())
        self._communicate(host, port, data)

    def _com_UPDATE(self, host, port):
        """
        Syntax: UPDATE <hash> <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s %s' % (self.command, self.get_checksum(), passphrase)
        self._communicate(host, port, data)

    def _com_DELETE(self, host, port):
        """
        Syntax: DELETE <passphrase>
        """
        passphrase = self._get_passphrase('Passphrase')
        data = '%s %s' % (self.command, passphrase)
        self._communicate(host, port, data)

    def _com_CHANGEPHRASE(self, host, port):
        """
        Syntax: CHANGEPHRASE <old_passphrase> <new_passphrase>
        """
        passphrase_old = self._get_passphrase('Old passphrase')
        while True:
            passphrase_new = self._get_passphrase('New passphrase')
            passphrase_new_confirm = self._get_passphrase('Confirm new passphrase')
            if passphrase_new == passphrase_new_confirm:
                break
            logger.error('passphrases do not match. try again...')
        data = '%s %s %s' % (self.command, passphrase_old, passphrase_new)
        self._communicate(host, port, data)

    # Public API

    def get_checksum(self):
        return self.hasher.hexdigest()

    def hash_data(self, data):
        """Passes data through the hashing algorithm."""
        self.hasher.update(data)
        time.sleep(self.default_hashing_delay)

    def run(self):
        """Main client method."""

        logger.info('Getting list of enabled servers')
        enabled_servers = self._get_enabled_server_list()
        if not enabled_servers:
            logger.warning('No servers configured. Shutting down...')
            self.client_close()
            return
        logger.info('Servers found. Proceeding with data hashing...')

        # Decide which method to execute
        func = getattr(self, '_com_%s' % self.command)

        # Tests are required to run only with the CHECK and UPDATE commands
        if self.command in ('CHECK', 'UPDATE'):
            logger.info('Hashing data. Please wait...')
            try:
                self._run_backends()
            except NoBackendsToRun:
                logger.warning('No valid backends. Shutting down...')
                self.client_close()
                return
            else:
                logger.info('Data hashing complete')

        # Execute command on server
        logger.info('Preparing to contact servers')
        for server in enabled_servers:
            # server is in 'server__<name>' format (a section name)
            # Here we set self.server_name to the canonical server name
            self.server_name = self._get_server_canonical_name(server)
            logger.info('* Contacting server: %s' % self.server_name)

            # Get server settings
            host = self.cfg.get(server, 'host')
            port = config.DEFAULT_PORT
            if self.cfg.has_option(server, 'port'):
                port = self.cfg.getint(server, 'port')
            if self.cfg.has_option(server, 'public_key'):
                public_key_fname = self.cfg.get(server, 'public_key')
                if public_key_fname:
                    try:
                        self.pki.load_external_public_key(public_key_fname) # sets self.pki.public_key
                    except crypto.InvalidPublicKey:
                        logger.warning('- RESULT: %s on %s: FAILURE with PKI error: invalid server public key: %s' % (self.command, self.server_name, public_key_fname))
                        self.server_name = None
                        continue

            # Run command on the server
            try:
                func(host, port)
            except socket.error, (errno, strerror):
                logger.error('Connection error: \'%s\'' % strerror)
                logger.warning('Skipping server: %s' % self.server_name)
                self.server_name = None
            except crypto.DataEncryptionError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not encrypt data for server' % (self.command, self.server_name))
            except crypto.DataVerificationError:
                logger.warning('- RESULT: %s on %s: FAILURE with PKI error: could not verify response from server' % (self.command, self.server_name))

            self._close_socket()
            self.pki.reset()    # sets self.pki.public_key to None

        logger.info('Finished with servers')

        self.client_close()

    def client_close(self):
        logger.info('Client shutting down...')

<|endoftext|>


<commit_before>from datetime import datetime
import logging

from facebook_auth.graph_api import ObservableGraphAPI

try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

from django.conf import settings
from django.utils import timezone
from facepy import exceptions

from facebook_auth import models
from facebook_auth import utils

logger = logging.getLogger(__name__)

def _truncate(word, length, to_zero=False):
    if to_zero and len(word) > length:
        return word[0:0] # preserve type
    else:
        return word[:length]

FACEBOOK_TIMEOUT = getattr(settings, 'FACEBOOK_AUTH_BACKEND_FACEBOOK_TIMEOUT',
                           timezone.timedelta(seconds=20).total_seconds())

class UserFactory(object):
    graph_api_class = ObservableGraphAPI
    fallback_expiration_date = datetime(1990, 10, 10, 0, 0, 1).replace(
        tzinfo=timezone.utc)

    def __create_username(self, profile):
            return profile['id']  # TODO better username

    def _product_user(self, access_token, profile):
        user_id = int(profile['id'])
        username = self.__create_username(profile)
        user, created = models.FacebookUser.objects.get_or_create(
                user_id=user_id, username=username)
        if created:
            user.set_unusable_password()

        def copy_field(field, to_zero=False):
            if field in profile:
                length = user._meta.get_field_by_name(field)[0].max_length
                setattr(user, field, _truncate(profile[field], length,
                        to_zero=to_zero))

        copy_field('email', True)
        copy_field('first_name')
        copy_field('last_name')
        if access_token is not None:
            models.FacebookTokenManager().insert_token(
                access_token, str(user.user_id))
        user.save()
        self.create_profile_object(profile, user)
        return user

    def get_user(self, access_token):
        profile = utils.get_from_graph_api(
            self.graph_api_class(access_token, timeout=FACEBOOK_TIMEOUT), 'me')
        return self._product_user(access_token, profile)

    def _get_fallback_expiration_date(self):
        logger.warning('Deprecated fallback expiration_date used.')
        return self.fallback_expiration_date

    def get_user_by_id(self, uid):
        graph_api = self.graph_api_class(timeout=FACEBOOK_TIMEOUT)
        profile = utils.get_from_graph_api(graph_api, uid)
        return self._product_user(None, profile)

    def create_profile_object(self, profile, user):
        if 'facebook_profile' in settings.INSTALLED_APPS:
            from facebook_profile import models as profile_models
            from facebook_profile import parser as profile_parser
            parser = profile_parser.FacebookDataParser(profile, True, True)
            try:
                data = parser.run()
                profile = (profile_models.FacebookUserProfile
                           .objects.create_or_update(data))
                profile.user = user
                profile.save()
            except profile_parser.FacebookDataParserCriticalError:
                pass

USER_FACTORY = UserFactory()

class FacebookBackend(object):
    def authenticate(self, code=None, redirect_uri=None):
        graph = ObservableGraphAPI(timeout=FACEBOOK_TIMEOUT)
        args = {
            'client_id': settings.FACEBOOK_APP_ID,
            'client_secret': settings.FACEBOOK_APP_SECRET,
            'redirect_uri': redirect_uri,
            'code': code
        }
        try:
            data = graph.get('/oauth/access_token', **args)
        except exceptions.FacebookError as e:
            message = "Facebook login failed %s" % e.message
            code_used_message = 'This authorization code has been used.'
            if e.code == 100 and e.message == code_used_message:
                logger.info(message)
                return None
            else:
                logger.warning(message)
                raise
        except exceptions.FacepyError as e:
            logger.warning("Facebook login connection error")
            raise
        try:
            access_token = urlparse.parse_qs(data)['access_token'][-1]
        except KeyError as e:
            args['client_secret'] = '*******%s' % args['client_secret'][-4:]
            logger.error(e, extra={'facebook_response': data,
                                   'sent_args': args})
            return None
        user = USER_FACTORY.get_user(access_token)
        return user

    @staticmethod
    def _timestamp_to_datetime(timestamp):
        naive = datetime.fromtimestamp(int(timestamp))
        return naive.replace(tzinfo=timezone.utc)

    def get_user(self, user_id):
        try:
            return models.FacebookUser.objects.get(pk=user_id)
        except models.FacebookUser.DoesNotExist: #@UndefinedVariable
            return None

class FacebookJavascriptBackend(FacebookBackend):
    def authenticate(self, access_token):
        return USER_FACTORY.get_user(access_token)
<commit_msg>[close #8199] Version Facebook call to /me.<commit_after>from datetime import datetime
import logging

from facebook_auth.graph_api import ObservableGraphAPI

try:
    import urllib.parse as urlparse
except ImportError:
    import urlparse

from django.conf import settings
from django.utils import timezone
from facepy import exceptions

from facebook_auth import models
from facebook_auth import utils

logger = logging.getLogger(__name__)

def _truncate(word, length, to_zero=False):
    if to_zero and len(word) > length:
        return word[0:0] # preserve type
    else:
        return word[:length]

FACEBOOK_TIMEOUT = getattr(settings, 'FACEBOOK_AUTH_BACKEND_FACEBOOK_TIMEOUT',
                           timezone.timedelta(seconds=20).total_seconds())

class UserFactory(object):
    graph_api_class = ObservableGraphAPI
    fallback_expiration_date = datetime(1990, 10, 10, 0, 0, 1).replace(
        tzinfo=timezone.utc)

    def __create_username(self, profile):
            return profile['id']  # TODO better username

    def _product_user(self, access_token, profile):
        user_id = int(profile['id'])
        username = self.__create_username(profile)
        user, created = models.FacebookUser.objects.get_or_create(
                user_id=user_id, username=username)
        if created:
            user.set_unusable_password()

        def copy_field(field, to_zero=False):
            if field in profile:
                length = user._meta.get_field_by_name(field)[0].max_length
                setattr(user, field, _truncate(profile[field], length,
                        to_zero=to_zero))

        copy_field('email', True)
        copy_field('first_name')
        copy_field('last_name')
        if access_token is not None:
            models.FacebookTokenManager().insert_token(
                access_token, str(user.user_id))
        user.save()
        self.create_profile_object(profile, user)
        return user

    def get_user(self, access_token):
        profile = utils.get_from_graph_api(
            self.graph_api_class(access_token, timeout=FACEBOOK_TIMEOUT),
            'v2.1/me')
        return self._product_user(access_token, profile)

    def _get_fallback_expiration_date(self):
        logger.warning('Deprecated fallback expiration_date used.')
        return self.fallback_expiration_date

    def get_user_by_id(self, uid):
        graph_api = self.graph_api_class(timeout=FACEBOOK_TIMEOUT)
        profile = utils.get_from_graph_api(graph_api, uid)
        return self._product_user(None, profile)

    def create_profile_object(self, profile, user):
        if 'facebook_profile' in settings.INSTALLED_APPS:
            from facebook_profile import models as profile_models
            from facebook_profile import parser as profile_parser
            parser = profile_parser.FacebookDataParser(profile, True, True)
            try:
                data = parser.run()
                profile = (profile_models.FacebookUserProfile
                           .objects.create_or_update(data))
                profile.user = user
                profile.save()
            except profile_parser.FacebookDataParserCriticalError:
                pass

USER_FACTORY = UserFactory()

class FacebookBackend(object):
    def authenticate(self, code=None, redirect_uri=None):
        graph = ObservableGraphAPI(timeout=FACEBOOK_TIMEOUT)
        args = {
            'client_id': settings.FACEBOOK_APP_ID,
            'client_secret': settings.FACEBOOK_APP_SECRET,
            'redirect_uri': redirect_uri,
            'code': code
        }
        try:
            data = graph.get('/oauth/access_token', **args)
        except exceptions.FacebookError as e:
            message = "Facebook login failed %s" % e.message
            code_used_message = 'This authorization code has been used.'
            if e.code == 100 and e.message == code_used_message:
                logger.info(message)
                return None
            else:
                logger.warning(message)
                raise
        except exceptions.FacepyError as e:
            logger.warning("Facebook login connection error")
            raise
        try:
            access_token = urlparse.parse_qs(data)['access_token'][-1]
        except KeyError as e:
            args['client_secret'] = '*******%s' % args['client_secret'][-4:]
            logger.error(e, extra={'facebook_response': data,
                                   'sent_args': args})
            return None
        user = USER_FACTORY.get_user(access_token)
        return user

    @staticmethod
    def _timestamp_to_datetime(timestamp):
        naive = datetime.fromtimestamp(int(timestamp))
        return naive.replace(tzinfo=timezone.utc)

    def get_user(self, user_id):
        try:
            return models.FacebookUser.objects.get(pk=user_id)
        except models.FacebookUser.DoesNotExist: #@UndefinedVariable
            return None

class FacebookJavascriptBackend(FacebookBackend):
    def authenticate(self, access_token):
        return USER_FACTORY.get_user(access_token)
<|endoftext|>


<commit_before>import sys
sys.path.insert(1,"../../")
import h2o
from tests import pyunit_utils
# The purpose of this test is to detect a change in the _metric_json of MetricsBase objects. Many of the metric
# accessors require _metric_json to have a particular form.
from h2o.estimators.glm import H2OGeneralizedLinearEstimator
from h2o.estimators.gbm import H2OGradientBoostingEstimator

def metric_json_check():
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/prostate.csv"))

    # Regression metric json
    reg_mod = H2OGradientBoostingEstimator(distribution="gaussian")
    reg_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    reg_met = reg_mod.model_performance()
    reg_metric_json_keys_have = list(reg_met._metric_json.keys())
    reg_metric_json_keys_desired = [u'model_category',
                                    u'description',
                                    u'r2',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'mae',
                                    u'rmsle',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'scoring_time',
                                    u'predictions',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'mean_residual_deviance',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    reg_metric_diff = list(set(reg_metric_json_keys_have) - set(reg_metric_json_keys_desired))
    assert not reg_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) regression " \
                                "metric json. The difference is {2}".format(reg_metric_json_keys_have,
                                                                            reg_metric_json_keys_desired,
                                                                            reg_metric_diff)
    # Regression metric json (GLM)
    reg_mod = H2OGeneralizedLinearEstimator(family="gaussian")
    reg_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    reg_met = reg_mod.model_performance()
    reg_metric_json_keys_have = list(reg_met._metric_json.keys())
    reg_metric_json_keys_desired = [u'model_category',
                                    u'description',
                                    u'r2',
                                    u'residual_degrees_of_freedom',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'mae',
                                    u'rmsle',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'null_deviance',
                                    u'scoring_time',
                                    u'null_degrees_of_freedom',
                                    u'predictions',
                                    u'AIC',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'residual_deviance',
                                    u'mean_residual_deviance',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    reg_metric_diff = list(set(reg_metric_json_keys_have) - set(reg_metric_json_keys_desired))
    assert not reg_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) glm-regression " \
                                "metric json. The difference is {2}".format(reg_metric_json_keys_have,
                                                                            reg_metric_json_keys_desired,
                                                                            reg_metric_diff)

    # Binomial metric json
    bin_mod = H2OGradientBoostingEstimator(distribution="bernoulli")
    df["CAPSULE"] = df["CAPSULE"].asfactor()
    bin_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    bin_met = bin_mod.model_performance()
    bin_metric_json_keys_have = list(bin_met._metric_json.keys())
    bin_metric_json_keys_desired = [u'AUC',
                                    u'Gini',
                                    u'model_category',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'r2',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'gains_lift_table',
                                    u'logloss',
                                    u'scoring_time',
                                    u'thresholds_and_metric_scores',
                                    u'predictions',
                                    u'max_criteria_and_metric_scores',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'domain',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    bin_metric_diff = list(set(bin_metric_json_keys_have) - set(bin_metric_json_keys_desired))
    assert not bin_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) binomial " \
                                "metric json. The difference is {2}".format(bin_metric_json_keys_have,
                                                                            bin_metric_json_keys_desired,
                                                                            bin_metric_diff)

    # Binomial metric json (GLM)
    bin_mod = H2OGeneralizedLinearEstimator(family="binomial")
    bin_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    bin_metric_json_keys_have = list(bin_met._metric_json.keys())
    bin_metric_json_keys_desired = [u'frame',
                                    u'residual_deviance',
                                    u'max_criteria_and_metric_scores',
                                    u'MSE',
                                    u'RMSE',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'AIC',
                                    u'logloss',
                                    u'Gini',
                                    u'predictions',
                                    u'AUC',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'model_checksum',
                                    u'duration_in_ms',
                                    u'model_category',
                                    u'gains_lift_table',
                                    u'r2',
                                    u'residual_degrees_of_freedom',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'null_deviance',
                                    u'scoring_time',
                                    u'null_degrees_of_freedom',
                                    u'model',
                                    u'thresholds_and_metric_scores',
                                    u'domain',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    bin_metric_diff = list(set(bin_metric_json_keys_have) - set(bin_metric_json_keys_desired))
    assert not bin_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) glm-binomial " \
                                "metric json. The difference is {2}".format(bin_metric_json_keys_have,
                                                                            bin_metric_json_keys_desired,
                                                                            bin_metric_diff)

    # Multinomial metric json
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/airlines/AirlinesTrain.csv.zip"))
    myX = ["Origin", "Dest", "IsDepDelayed", "UniqueCarrier", "Distance", "fDayofMonth", "fDayOfWeek"]
    myY = "fYear"

    mul_mod = H2OGradientBoostingEstimator(distribution="multinomial")
    mul_mod.train(x=myX, y=myY, training_frame=df)
    mul_met = mul_mod.model_performance()
    mul_metric_json_keys_have = list(mul_met._metric_json.keys())
    mul_metric_json_keys_desired = [u'cm',
                                    u'model_category',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'r2',
                                    u'frame',
                                    u'nobs',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'logloss',
                                    u'scoring_time',
                                    u'predictions',
                                    u'hit_ratio_table',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    mul_metric_diff = list(set(mul_metric_json_keys_have) - set(mul_metric_json_keys_desired))
    assert not mul_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) multinomial " \
                                "metric json. The difference is {2}".format(mul_metric_json_keys_have,
                                                                            mul_metric_json_keys_desired,
                                                                            mul_metric_diff)

    # Clustering metric json
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/iris/iris.csv"))
    from h2o.estimators.kmeans import H2OKMeansEstimator
    clus_mod = H2OKMeansEstimator(k=3, standardize=False)
    clus_mod.train(x=list(range(4)), training_frame=df)
    clus_met = clus_mod.model_performance()
    clus_metric_json_keys_have = list(clus_met._metric_json.keys())
    clus_metric_json_keys_desired = [u'tot_withinss',
                                     u'model_category',
                                     u'description',
                                     u'frame',
                                     u'model_checksum',
                                     u'MSE',
                                     u'RMSE',
                                     u'__meta',
                                     u'_exclude_fields',
                                     u'scoring_time',
                                     u'betweenss',
                                     u'predictions',
                                     u'totss',
                                     u'model',
                                     u'duration_in_ms',
                                     u'frame_checksum',
                                     u'nobs',
                                     u'centroid_stats'
                                     u'custom_metric_name',
                                     u'custom_metric_value']
    clus_metric_diff = list(set(clus_metric_json_keys_have) - set(clus_metric_json_keys_desired))
    assert not clus_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) clustering " \
                                "metric json. The difference is {2}".format(clus_metric_json_keys_have,
                                                                            clus_metric_json_keys_desired,
                                                                            clus_metric_diff)

if __name__ == "__main__":
    pyunit_utils.standalone_test(metric_json_check)
else:
    metric_json_check()
<commit_msg>Fix typo in json test<commit_after>import sys
sys.path.insert(1,"../../")
import h2o
from tests import pyunit_utils
# The purpose of this test is to detect a change in the _metric_json of MetricsBase objects. Many of the metric
# accessors require _metric_json to have a particular form.
from h2o.estimators.glm import H2OGeneralizedLinearEstimator
from h2o.estimators.gbm import H2OGradientBoostingEstimator

def metric_json_check():
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/logreg/prostate.csv"))

    # Regression metric json
    reg_mod = H2OGradientBoostingEstimator(distribution="gaussian")
    reg_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    reg_met = reg_mod.model_performance()
    reg_metric_json_keys_have = list(reg_met._metric_json.keys())
    reg_metric_json_keys_desired = [u'model_category',
                                    u'description',
                                    u'r2',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'mae',
                                    u'rmsle',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'scoring_time',
                                    u'predictions',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'mean_residual_deviance',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    reg_metric_diff = list(set(reg_metric_json_keys_have) - set(reg_metric_json_keys_desired))
    assert not reg_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) regression " \
                                "metric json. The difference is {2}".format(reg_metric_json_keys_have,
                                                                            reg_metric_json_keys_desired,
                                                                            reg_metric_diff)
    # Regression metric json (GLM)
    reg_mod = H2OGeneralizedLinearEstimator(family="gaussian")
    reg_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    reg_met = reg_mod.model_performance()
    reg_metric_json_keys_have = list(reg_met._metric_json.keys())
    reg_metric_json_keys_desired = [u'model_category',
                                    u'description',
                                    u'r2',
                                    u'residual_degrees_of_freedom',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'mae',
                                    u'rmsle',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'null_deviance',
                                    u'scoring_time',
                                    u'null_degrees_of_freedom',
                                    u'predictions',
                                    u'AIC',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'residual_deviance',
                                    u'mean_residual_deviance',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    reg_metric_diff = list(set(reg_metric_json_keys_have) - set(reg_metric_json_keys_desired))
    assert not reg_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) glm-regression " \
                                "metric json. The difference is {2}".format(reg_metric_json_keys_have,
                                                                            reg_metric_json_keys_desired,
                                                                            reg_metric_diff)

    # Binomial metric json
    bin_mod = H2OGradientBoostingEstimator(distribution="bernoulli")
    df["CAPSULE"] = df["CAPSULE"].asfactor()
    bin_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    bin_met = bin_mod.model_performance()
    bin_metric_json_keys_have = list(bin_met._metric_json.keys())
    bin_metric_json_keys_desired = [u'AUC',
                                    u'Gini',
                                    u'model_category',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'r2',
                                    u'frame',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'gains_lift_table',
                                    u'logloss',
                                    u'scoring_time',
                                    u'thresholds_and_metric_scores',
                                    u'predictions',
                                    u'max_criteria_and_metric_scores',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'domain',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    bin_metric_diff = list(set(bin_metric_json_keys_have) - set(bin_metric_json_keys_desired))
    assert not bin_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) binomial " \
                                "metric json. The difference is {2}".format(bin_metric_json_keys_have,
                                                                            bin_metric_json_keys_desired,
                                                                            bin_metric_diff)

    # Binomial metric json (GLM)
    bin_mod = H2OGeneralizedLinearEstimator(family="binomial")
    bin_mod.train(x=list(range(3,df.ncol)), y="CAPSULE", training_frame=df)
    bin_metric_json_keys_have = list(bin_met._metric_json.keys())
    bin_metric_json_keys_desired = [u'frame',
                                    u'residual_deviance',
                                    u'max_criteria_and_metric_scores',
                                    u'MSE',
                                    u'RMSE',
                                    u'frame_checksum',
                                    u'nobs',
                                    u'AIC',
                                    u'logloss',
                                    u'Gini',
                                    u'predictions',
                                    u'AUC',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'model_checksum',
                                    u'duration_in_ms',
                                    u'model_category',
                                    u'gains_lift_table',
                                    u'r2',
                                    u'residual_degrees_of_freedom',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'null_deviance',
                                    u'scoring_time',
                                    u'null_degrees_of_freedom',
                                    u'model',
                                    u'thresholds_and_metric_scores',
                                    u'domain',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    bin_metric_diff = list(set(bin_metric_json_keys_have) - set(bin_metric_json_keys_desired))
    assert not bin_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) glm-binomial " \
                                "metric json. The difference is {2}".format(bin_metric_json_keys_have,
                                                                            bin_metric_json_keys_desired,
                                                                            bin_metric_diff)

    # Multinomial metric json
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/airlines/AirlinesTrain.csv.zip"))
    myX = ["Origin", "Dest", "IsDepDelayed", "UniqueCarrier", "Distance", "fDayofMonth", "fDayOfWeek"]
    myY = "fYear"

    mul_mod = H2OGradientBoostingEstimator(distribution="multinomial")
    mul_mod.train(x=myX, y=myY, training_frame=df)
    mul_met = mul_mod.model_performance()
    mul_metric_json_keys_have = list(mul_met._metric_json.keys())
    mul_metric_json_keys_desired = [u'cm',
                                    u'model_category',
                                    u'description',
                                    u'mean_per_class_error',
                                    u'r2',
                                    u'frame',
                                    u'nobs',
                                    u'model_checksum',
                                    u'MSE',
                                    u'RMSE',
                                    u'__meta',
                                    u'_exclude_fields',
                                    u'logloss',
                                    u'scoring_time',
                                    u'predictions',
                                    u'hit_ratio_table',
                                    u'model',
                                    u'duration_in_ms',
                                    u'frame_checksum',
                                    u'custom_metric_name',
                                    u'custom_metric_value']
    mul_metric_diff = list(set(mul_metric_json_keys_have) - set(mul_metric_json_keys_desired))
    assert not mul_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) multinomial " \
                                "metric json. The difference is {2}".format(mul_metric_json_keys_have,
                                                                            mul_metric_json_keys_desired,
                                                                            mul_metric_diff)

    # Clustering metric json
    df = h2o.import_file(path=pyunit_utils.locate("smalldata/iris/iris.csv"))
    from h2o.estimators.kmeans import H2OKMeansEstimator
    clus_mod = H2OKMeansEstimator(k=3, standardize=False)
    clus_mod.train(x=list(range(4)), training_frame=df)
    clus_met = clus_mod.model_performance()
    clus_metric_json_keys_have = list(clus_met._metric_json.keys())
    clus_metric_json_keys_desired = [u'tot_withinss',
                                     u'model_category',
                                     u'description',
                                     u'frame',
                                     u'model_checksum',
                                     u'MSE',
                                     u'RMSE',
                                     u'__meta',
                                     u'_exclude_fields',
                                     u'scoring_time',
                                     u'betweenss',
                                     u'predictions',
                                     u'totss',
                                     u'model',
                                     u'duration_in_ms',
                                     u'frame_checksum',
                                     u'nobs',
                                     u'centroid_stats',
                                     u'custom_metric_name',
                                     u'custom_metric_value']
    clus_metric_diff = list(set(clus_metric_json_keys_have) - set(clus_metric_json_keys_desired))
    assert not clus_metric_diff, "There's a difference between the current ({0}) and the desired ({1}) clustering " \
                                "metric json. The difference is {2}".format(clus_metric_json_keys_have,
                                                                            clus_metric_json_keys_desired,
                                                                            clus_metric_diff)

if __name__ == "__main__":
    pyunit_utils.standalone_test(metric_json_check)
else:
    metric_json_check()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.db import models, transaction
from django.db.models.base import ModelBase
from django.utils.translation import ugettext, ugettext_lazy as _
from django.core import exceptions

from dateutil import rrule
from eventtools.models.xtimespan import XTimespanModel

from eventtools.conf import settings
from eventtools.utils.pprint_timespan import (
    pprint_datetime_span, pprint_date_span)

from datetime import date, time, datetime, timedelta

class GeneratorModel(XTimespanModel):
    """
    Stores information about repeating Occurrences, and generates them,
    unless they already exist, or match an Exception.

    The public API is quite simple:

    save() generates Occurrences.

    clean() makes sure the Generator has valid values (and is called by admin
    before the instance is saved)

    robot_description() attempts to provide an English description of this
    generator. It's not great at the moment and might be replaced or deprecated
    in favour of a hand-written description in the Event.

    EventModel() returns the Model of the Event that this Generator links to.
    """

    #define a FK called 'event' in the subclass
    rule = models.ForeignKey("eventtools.Rule", verbose_name=_('rule'))
    repeat_until = models.DateField(
        null=True, blank = True,
        verbose_name=_('repeat until'),
        help_text=_(u"Occurrences will repeat up to and including this date. If ommitted, the next year's worth of "
            "occurrences will be created."
        )
    )

    class Meta:
        abstract = True
        ordering = ('start',)
        verbose_name = _("repeating occurrence")
        verbose_name_plural = _("repeating occurrences")

    def __unicode__(self):
        return u"%s, %s" % (self.event, self.robot_description())

    @classmethod
    def EventModel(cls):
        return cls._meta.get_field('event').rel.to

    def clean(self, ExceptionClass=exceptions.ValidationError):
        super(GeneratorModel, self).clean()
        if not self.rule_id:
            raise ExceptionClass('A Rule must be given')

        if self.start and self.repeat_until and self.repeat_until < self.start.date():
            raise ExceptionClass(
                'Repeat until date must not be earlier than start date')

        self.is_clean = True

    @transaction.commit_on_success()
    def save(self, *args, **kwargs):
        """
        Generally (and for a combination of field changes), we take a
        two-pass approach:

         1) First update existing occurrences to match update-compatible fields.
         2) Then synchronise the candidate occurrences with the existing
            occurrences.
            * For candidate occurrences that exist, do nothing.
            * For candidate occurrences that do not exist, add them.
            * For existing occurrences that are not candidates, unhook them from
                the generator.

        Finally, we also update other generators, because they might have had
        clashing occurrences which no longer clash.
        """

        cascade = kwargs.pop('cascade', True)

        if not getattr(self, 'is_clean', False):
            # if we're saving directly, the ModelForm clean isn't called, so
            # we do it here.
            self.clean(ExceptionClass=AttributeError)

        # Occurrences updates/generates
        if self.pk:
            self._update_existing_occurrences() # need to do this before save, so we can detect changes
        r = super(GeneratorModel, self).save(*args, **kwargs)
        self._sync_occurrences() #need to do this after save, so we have a pk to hang new occurrences from.

        # finally, we should also update other generators, because they might
        # have had clashing occurrences
        if cascade:
            for generator in self.event.generators.exclude(pk=self.pk):
                generator.save(cascade=False)

        return r

    def _generate_dates(self):
        rule = self.rule.get_rrule(dtstart=self.start)
        date_iter = iter(rule)
        drop_dead_date = datetime.combine(self.repeat_until or date.today() \
            + settings.DEFAULT_GENERATOR_LIMIT, time.max)

        while True:
            d = date_iter.next()
            if d > drop_dead_date:
                break
            yield d

    @transaction.commit_on_success()
    def _update_existing_occurrences(self):
        """
        When you change a generator and save it, it updates existing occurrences
        according to the following rules:

        Generally, if we can't automatically delete occurrences, we unhook them
        from the generator, and make them one-off. This is to prevent losing
        information like tickets sold or shout-outs (we leave implementors to
        decide the workflow in these cases). We want to minimise the number of
        events that are deleted or unhooked, however. So:

         * If start time or duration is changed, then no occurrences are
           added or removed - we timeshift all occurrences. We assume that
           visitors/ticket holders are alerted to the time change elsewhere.

         * If other fields are changed - repetition rule, repeat_until, start
           date - then there is a chance that Occurrences will be added or
           removed.

         * Occurrences that are added are fine, they are added in the normal
           way.

         * Occurrences that are removed are deleted or unhooked, for reasons
           described above.
        """

        """
        Pass 1)
        if start date or time is changed:
            update the start times of my occurrences
        if end date or time is changed:
            update the end times of my occurrences

        Pass 2 is in _sync_occurrences, below.
        """

        # TODO: it would be ideal to minimise the consequences of shifting one
        # occurrence to replace another - ie to leave most occurrences untouched
        # and to create only new ones and unhook ungenerated ones.
        # I tried this by using start date (which is unique per generator) as
        # a nominal 'key', but it gets fiddly when you want to vary the end
        # date to before the old start date. For now we'll just update the dates
        # and times.

        saved_self = type(self).objects.get(pk=self.pk)

        start_shift = self.start - saved_self.start
        duration_changed = self._duration != saved_self._duration

        if start_shift or duration_changed:
            # Update occurrences in opposite direction to the adjustment of the
            # 'start' field, to avoid updating an occurrence to clash with an
            # existing one's (event_id, start) DB uniqueness constraint (#606)
            if start_shift.total_seconds() >= 0:
                start_order_by = '-start'  # Moving to future, start from latest
            else:
                start_order_by = 'start'  # Moving to past, start from earliest

            for o in self.occurrences.order_by(start_order_by):
                o.start += start_shift
                o._duration = self._duration
                o.save()

    @transaction.commit_on_success()
    def _sync_occurrences(self):

        """
        Pass 2)

        Generate a list of candidate occurrences.
        * For candidate occurrences that exist, do nothing.
        * For candidate occurrences that do not exist, add them.
        * For existing occurrences that are not candidates, delete them, or unhook them from the
          generator if they are protected by a Foreign Key.

        In detail:
        Get a list, A, of already-generated occurrences.

        Generate candidate Occurrences.
        For each candidate Occurrence:
            if it exists for the event:
                if I created it, unhook, and remove from the list A.
                else do nothing
            if it is an exclusion, do nothing
            otherwise create it.

        The items remaining in list A are 'orphan' occurrences, that were
        previously generated, but would no longer be. These are unhooked from
        the generator.
        """

        all_occurrences = self.event.occurrences_in_listing().all() #regardless of generator
        existing_but_not_regenerated = set(self.occurrences.all()) #generated by me only

        for start in self._generate_dates():
            # if the proposed occurrence exists, then don't make a new one.
            # However, if it belongs to me:
            #       and if it is marked as an exclusion:
            #           do nothing (it will later get deleted/unhooked)
            #       else:
            #           remove it from the set of existing_but_not_regenerated
            #           occurrences so it stays hooked up

            try:
                o = all_occurrences.filter(start=start)[0]
                if o.generated_by == self:
                    if not o.is_exclusion():
                        existing_but_not_regenerated.discard(o)
                continue
            except IndexError:
                # no occurrence exists yet.
                pass

            # if the proposed occurrence is an exclusion, don't save it.
            if self.event.exclusions.filter(
                event=self.event, start=start
            ).count():
                continue

            #OK, we're good to create the occurrence.
            o = self.occurrences.create(event=self.event, start=start, _duration=self._duration)
#            print "created %s" % o
            #implied generated_by = self

        # Finally, delete any unaccounted_for occurrences. If we can't delete, due to protection set by FKs to it, then
        # unhook it instead.
        for o in existing_but_not_regenerated:
#            print "deleting %s" % o
            o.delete()

    def delete(self, *args, **kwargs):
        """
        If I am deleted, then cascade to my Occurrences, UNLESS there is is something FKed to them that is protecting them,
        in which case the FK is set to NULL.
        """
        for o in self.occurrences.all():
            o.delete()

        super(GeneratorModel,self).delete(*args, **kwargs)

    def robot_description(self):
        r = "%s, repeating %s" % (
            pprint_datetime_span(self.start, self.end()),
            unicode(self.rule).lower(),
        )

        if self.repeat_until:
            r += " until %s" % pprint_date_span(self.repeat_until, self.repeat_until)

        return r
<commit_msg>Fix generator's limited occurrences check to work with both timezone-aware or naive occurrence datetimes<commit_after># -*- coding: utf-8 -*-

from django.db import models, transaction
from django.db.models.base import ModelBase
from django.utils.translation import ugettext, ugettext_lazy as _
from django.utils.timezone import get_current_timezone
from django.core import exceptions

from dateutil import rrule
from eventtools.models.xtimespan import XTimespanModel

from eventtools.conf import settings
from eventtools.utils.pprint_timespan import (
    pprint_datetime_span, pprint_date_span)

from datetime import date, time, datetime, timedelta

class GeneratorModel(XTimespanModel):
    """
    Stores information about repeating Occurrences, and generates them,
    unless they already exist, or match an Exception.

    The public API is quite simple:

    save() generates Occurrences.

    clean() makes sure the Generator has valid values (and is called by admin
    before the instance is saved)

    robot_description() attempts to provide an English description of this
    generator. It's not great at the moment and might be replaced or deprecated
    in favour of a hand-written description in the Event.

    EventModel() returns the Model of the Event that this Generator links to.
    """

    #define a FK called 'event' in the subclass
    rule = models.ForeignKey("eventtools.Rule", verbose_name=_('rule'))
    repeat_until = models.DateField(
        null=True, blank = True,
        verbose_name=_('repeat until'),
        help_text=_(u"Occurrences will repeat up to and including this date. If ommitted, the next year's worth of "
            "occurrences will be created."
        )
    )

    class Meta:
        abstract = True
        ordering = ('start',)
        verbose_name = _("repeating occurrence")
        verbose_name_plural = _("repeating occurrences")

    def __unicode__(self):
        return u"%s, %s" % (self.event, self.robot_description())

    @classmethod
    def EventModel(cls):
        return cls._meta.get_field('event').rel.to

    def clean(self, ExceptionClass=exceptions.ValidationError):
        super(GeneratorModel, self).clean()
        if not self.rule_id:
            raise ExceptionClass('A Rule must be given')

        if self.start and self.repeat_until and self.repeat_until < self.start.date():
            raise ExceptionClass(
                'Repeat until date must not be earlier than start date')

        self.is_clean = True

    @transaction.commit_on_success()
    def save(self, *args, **kwargs):
        """
        Generally (and for a combination of field changes), we take a
        two-pass approach:

         1) First update existing occurrences to match update-compatible fields.
         2) Then synchronise the candidate occurrences with the existing
            occurrences.
            * For candidate occurrences that exist, do nothing.
            * For candidate occurrences that do not exist, add them.
            * For existing occurrences that are not candidates, unhook them from
                the generator.

        Finally, we also update other generators, because they might have had
        clashing occurrences which no longer clash.
        """

        cascade = kwargs.pop('cascade', True)

        if not getattr(self, 'is_clean', False):
            # if we're saving directly, the ModelForm clean isn't called, so
            # we do it here.
            self.clean(ExceptionClass=AttributeError)

        # Occurrences updates/generates
        if self.pk:
            self._update_existing_occurrences() # need to do this before save, so we can detect changes
        r = super(GeneratorModel, self).save(*args, **kwargs)
        self._sync_occurrences() #need to do this after save, so we have a pk to hang new occurrences from.

        # finally, we should also update other generators, because they might
        # have had clashing occurrences
        if cascade:
            for generator in self.event.generators.exclude(pk=self.pk):
                generator.save(cascade=False)

        return r

    def _generate_dates(self):
        drop_dead_date = datetime.combine(self.repeat_until or date.today() \
            + settings.DEFAULT_GENERATOR_LIMIT, time.max)

        # We may need a timezone-aware datetime if our rule generates
        # non-naive datetime occurrences
        drop_dead_date_with_tzinfo = drop_dead_date.replace(
            tzinfo=get_current_timezone())

        # Yield rule's occurrence datetimes up until "drop dead" date(time)
        rule = self.rule.get_rrule(dtstart=self.start)
        date_iter = iter(rule)
        while True:
            d = date_iter.next()
            if d.tzinfo:
                dddate = drop_dead_date_with_tzinfo
            else:
                dddate = drop_dead_date
            if d > dddate:
                break
            yield d

    @transaction.commit_on_success()
    def _update_existing_occurrences(self):
        """
        When you change a generator and save it, it updates existing occurrences
        according to the following rules:

        Generally, if we can't automatically delete occurrences, we unhook them
        from the generator, and make them one-off. This is to prevent losing
        information like tickets sold or shout-outs (we leave implementors to
        decide the workflow in these cases). We want to minimise the number of
        events that are deleted or unhooked, however. So:

         * If start time or duration is changed, then no occurrences are
           added or removed - we timeshift all occurrences. We assume that
           visitors/ticket holders are alerted to the time change elsewhere.

         * If other fields are changed - repetition rule, repeat_until, start
           date - then there is a chance that Occurrences will be added or
           removed.

         * Occurrences that are added are fine, they are added in the normal
           way.

         * Occurrences that are removed are deleted or unhooked, for reasons
           described above.
        """

        """
        Pass 1)
        if start date or time is changed:
            update the start times of my occurrences
        if end date or time is changed:
            update the end times of my occurrences

        Pass 2 is in _sync_occurrences, below.
        """

        # TODO: it would be ideal to minimise the consequences of shifting one
        # occurrence to replace another - ie to leave most occurrences untouched
        # and to create only new ones and unhook ungenerated ones.
        # I tried this by using start date (which is unique per generator) as
        # a nominal 'key', but it gets fiddly when you want to vary the end
        # date to before the old start date. For now we'll just update the dates
        # and times.

        saved_self = type(self).objects.get(pk=self.pk)

        start_shift = self.start - saved_self.start
        duration_changed = self._duration != saved_self._duration

        if start_shift or duration_changed:
            # Update occurrences in opposite direction to the adjustment of the
            # 'start' field, to avoid updating an occurrence to clash with an
            # existing one's (event_id, start) DB uniqueness constraint (#606)
            if start_shift.total_seconds() >= 0:
                start_order_by = '-start'  # Moving to future, start from latest
            else:
                start_order_by = 'start'  # Moving to past, start from earliest

            for o in self.occurrences.order_by(start_order_by):
                o.start += start_shift
                o._duration = self._duration
                o.save()

    @transaction.commit_on_success()
    def _sync_occurrences(self):

        """
        Pass 2)

        Generate a list of candidate occurrences.
        * For candidate occurrences that exist, do nothing.
        * For candidate occurrences that do not exist, add them.
        * For existing occurrences that are not candidates, delete them, or unhook them from the
          generator if they are protected by a Foreign Key.

        In detail:
        Get a list, A, of already-generated occurrences.

        Generate candidate Occurrences.
        For each candidate Occurrence:
            if it exists for the event:
                if I created it, unhook, and remove from the list A.
                else do nothing
            if it is an exclusion, do nothing
            otherwise create it.

        The items remaining in list A are 'orphan' occurrences, that were
        previously generated, but would no longer be. These are unhooked from
        the generator.
        """

        all_occurrences = self.event.occurrences_in_listing().all() #regardless of generator
        existing_but_not_regenerated = set(self.occurrences.all()) #generated by me only

        for start in self._generate_dates():
            # if the proposed occurrence exists, then don't make a new one.
            # However, if it belongs to me:
            #       and if it is marked as an exclusion:
            #           do nothing (it will later get deleted/unhooked)
            #       else:
            #           remove it from the set of existing_but_not_regenerated
            #           occurrences so it stays hooked up

            try:
                o = all_occurrences.filter(start=start)[0]
                if o.generated_by == self:
                    if not o.is_exclusion():
                        existing_but_not_regenerated.discard(o)
                continue
            except IndexError:
                # no occurrence exists yet.
                pass

            # if the proposed occurrence is an exclusion, don't save it.
            if self.event.exclusions.filter(
                event=self.event, start=start
            ).count():
                continue

            #OK, we're good to create the occurrence.
            o = self.occurrences.create(event=self.event, start=start, _duration=self._duration)
#            print "created %s" % o
            #implied generated_by = self

        # Finally, delete any unaccounted_for occurrences. If we can't delete, due to protection set by FKs to it, then
        # unhook it instead.
        for o in existing_but_not_regenerated:
#            print "deleting %s" % o
            o.delete()

    def delete(self, *args, **kwargs):
        """
        If I am deleted, then cascade to my Occurrences, UNLESS there is is something FKed to them that is protecting them,
        in which case the FK is set to NULL.
        """
        for o in self.occurrences.all():
            o.delete()

        super(GeneratorModel,self).delete(*args, **kwargs)

    def robot_description(self):
        r = "%s, repeating %s" % (
            pprint_datetime_span(self.start, self.end()),
            unicode(self.rule).lower(),
        )

        if self.repeat_until:
            r += " until %s" % pprint_date_span(self.repeat_until, self.repeat_until)

        return r
<|endoftext|>


<commit_before>from celery.decorators import task
from celery.task.schedules import crontab
from celery.decorators import periodic_task

from projects.models import Project, Conf
from projects.utils import  find_file, run

import os
import re
import glob
import fnmatch

ghetto_hack = re.compile(r'(?P<key>.*)\s*=\s*u?[\'\"](?P<value>.*)[\'\"]')

@task
def update_docs(slug, type='git'):
    """
    A Celery task that updates the documentation for a project.
    """
    project = Project.objects.get(slug=slug)
    path = project.user_doc_path
    if not os.path.exists(path):
        os.makedirs(path)
    os.chdir(path)
    if os.path.exists(os.path.join(path, project.slug)):
        os.chdir(project.slug)
        if type is 'git':
            command = 'git fetch && git reset --hard origin/master'
            print command
            run(command)
    else:
        if type is 'git':
            repo = project.github_repo
            repo.replace('.git', '')
            command = 'git clone %s.git %s' % (repo, project.slug)
            print command
            run(command)
    build_docs(project)

def build_docs(project):
    """
    A helper function for the celery task to do the actual doc building.
    """
    conf_dir = project.find('conf.py')[0].replace('/conf.py', '')
    os.chdir(conf_dir)
    lines = open('conf.py').readlines()
    data = {}
    for line in lines:
        for we_care in ['copyright', 'project', 'version', 'release', 'html_theme']:
            match = ghetto_hack.search(line)
            if match:
                data[match.group(1).strip()] = match.group(2).strip()
    conf = Conf.objects.get_or_create(project=project)[0]
    conf.copyright = data['copyright']
    conf.theme = data.get('html_theme', 'default')
    conf.path = os.getcwd()
    conf.save()

    project.write_conf()

    make_dir = project.find('Makefile')[0].replace('/Makefile', '')
    os.chdir(make_dir)
    os.system('make html')

#@periodic_task(run_every=crontab(hour="*", minute="*/30", day_of_week="*"))
@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*"))
def update_docs_pull():
    for project in Project.objects.all():
        print "Building %s" % project
        try:
            build_docs(project)
        except Exception, e:
            print e

<commit_msg>Slightly more graceful fallback when no Makefile exists.<commit_after>from celery.decorators import task
from celery.task.schedules import crontab
from celery.decorators import periodic_task

from projects.models import Project, Conf
from projects.utils import  find_file, run

import os
import re
import glob
import fnmatch

ghetto_hack = re.compile(r'(?P<key>.*)\s*=\s*u?[\'\"](?P<value>.*)[\'\"]')

@task
def update_docs(slug, type='git'):
    """
    A Celery task that updates the documentation for a project.
    """
    project = Project.objects.get(slug=slug)
    path = project.user_doc_path
    if not os.path.exists(path):
        os.makedirs(path)
    os.chdir(path)
    if os.path.exists(os.path.join(path, project.slug)):
        os.chdir(project.slug)
        if type is 'git':
            command = 'git fetch && git reset --hard origin/master'
            print command
            run(command)
    else:
        if type is 'git':
            repo = project.github_repo
            repo.replace('.git', '')
            command = 'git clone %s.git %s' % (repo, project.slug)
            print command
            run(command)
    build_docs(project)

def build_docs(project):
    """
    A helper function for the celery task to do the actual doc building.
    """
    conf_dir = project.find('conf.py')[0].replace('/conf.py', '')
    os.chdir(conf_dir)
    lines = open('conf.py').readlines()
    data = {}
    for line in lines:
        for we_care in ['copyright', 'project', 'version', 'release', 'html_theme']:
            match = ghetto_hack.search(line)
            if match:
                data[match.group(1).strip()] = match.group(2).strip()
    conf = Conf.objects.get_or_create(project=project)[0]
    conf.copyright = data['copyright']
    conf.theme = data.get('html_theme', 'default')
    conf.path = os.getcwd()
    conf.save()

    project.write_conf()

    try:
        make_dir = project.find('Makefile')[0].replace('/Makefile', '')
        os.chdir(make_dir)
        os.system('make html')
    except IndexError:
        os.chdir(conf_dir)
        os.system('sphinx-build -b html . _build')

#@periodic_task(run_every=crontab(hour="*", minute="*/30", day_of_week="*"))
@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*"))
def update_docs_pull():
    for project in Project.objects.all():
        print "Building %s" % project
        try:
            build_docs(project)
        except Exception, e:
            print e
<|endoftext|>