<commit_before># Copyright 2017 CBSD Project Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

### Program Constants
WINNF_PROTOCOL_VERSION =                    "v1.1"
SHORTER_GRANT_EXPIRY_TIME =                 360
DEFAULT_RULE_APPLIED =                      "FCC_PART_96"
DEFAULT_CHANNEL_TYPE =                      "GAA"
SECONDS_TO_ADD_FOR_GRANT_EXPIRE_TIME =      604800          # seconds ahead for grantExpireTime - 1 week
SECONDS_TO_ADD_FOR_TX_EXPIRE_TIME =         200             # seconds ahead for transmitExpireTime
SECONDS_FOR_DELAY_RESPONSE =                SECONDS_TO_ADD_FOR_TX_EXPIRE_TIME   # Worst-case value
CPI_SIGNATURE_VALID_TYPES =                 ['RS256', 'ES256']
CPI_INSTALLPARAM_CATA_OPTIONAL =            ['antennaAzimuth', 'antennaDowntilt', 'antennaBeamwidth']   # params REG-Cond for Cat B, optional for Cat A

HEARTBEAT_INTERVAL =                        60
HEARTBEAT_INTERVAL_GRACE_PERIOD =           0.005    # seconds

CBRS_SPECTRUM_HIGH =                        3700000000
CBRS_SPECTRUM_LOW =                         3550000000
MAXEIRP_LOW =                               -137
MAXERIP_HIGH =                              37

SPECTRUM_PAL_HIGH =                         3650000000
SPECTRUM_PAL_LOW =                          3550000000
SPECTRUM_GAA_HIGH =                         3700000000
SPECTRUM_GAA_LOW =                          3550000000

WINNF_APPROVED_CIPHER_LIST = [
    'AES128-GCM-SHA256',                # TLS_RSA_WITH_AES_128_GCM_SHA256
    'AES256-GCM-SHA384',                # TLS_RSA_WITH_AES_256_GCM_SHA384
    'ECDHE-ECDSA-AES128-GCM-SHA256',    # TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
    'ECDHE-ECDSA-AES256-GCM-SHA384',    # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
    'ECDHE-RSA-AES128-GCM-SHA256',      # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
    '@STRENGTH'
    ]

WINNF_APPROVED_CIPHERS = ':'.join(WINNF_APPROVED_CIPHER_LIST) # string of ciphers separated by colons

MAX_NUM_HBT_FOR_MEASREPORT =    5   # number of HBT responses for measReport to appear, before failing test

### Other Constants
ERROR_VALIDATION_MESSAGE =                  "ERROR - An Error occurred while comparing between actual and expected request"
RESULTS_OF_TEST_MESSAGE =                   "The final result of the test : "
SET_CSV_FILE_MESSAGE =                      "Please enter a csv file path that include the test you request to run (pay attention insert the csv file without the csv suffix) \n, type quit and press ctrl+c to close"
START_TEST_MESSAGE =                        "Start of the test"
CLOSE_USER_SESSION =                        "Session user will now close"
SENT_FLASK_REQUEST =                        "sent flask request to engine"
SHUTDOWN_FUNCTION_NAME =                    'shutdown'
TEST_HAD_BEEN_FINISHED_FLASK =              " all the steps of the test had been finished see final results at the console "
NAME_OF_SERVER_WERKZUG =                    "werkzeug.server.shutdown"
SERVER_SHUT_DOWN_MESSAGE =                  "Server shutting down due to"
E_NODE_B_SENT_TO_ENGINE_REQUEST =           "cbsd sent request to engine from server"
INIT_TEST_DEFINITION =                      "init the test definition by the test steps sent from the user"
SUFFIX_OF_JSON_FILE =                       ".json"
COMPARING_JSONS_MESSAGE =                   "compare between requested json to expected json"
REQUEST_NODE_NAME =                         "Request"
RESPONSE_NODE_NAME =                        "response"
QUESTION_NODE_NAME =                        "questions"
HEART_BEAT_SUFFIX_HTTP =                    "heartbeat"
SPECTRUM_INQUIERY_SUFFIX_HTTP =             "spectrumInquiry"
HEART_BEAT_ARRIVED_MESSAGE =                "heartBeat request arrived to engine"
GRANT_SUFFIX_HTTP =                         "grant"
RELINQUISHMENT_SUFFIX_HTTP =                "relinquishment"
NSTEP_SESSION_WITH_TECHNITIAN =             "arrived to nstep starting question answer session with the technician"
HEARTBEAT_FROM_ENGINE_TO_CBSD_MESSAGE =      "return heart beat part of json from engine to the cbsd"
ADDITIONAL_COMMENTS_MESSAGE =               "the additional comments for the current test are : "
ANSWERS_NODE =                              "answers"
QUESTION_NODE =                             "question"
SUFFIX_NOT_EXISTS_IN_EXPECTED_JSON_FILE =   "ERROR - the suffix of the type of calling from http request is not exists in the expected json file from the csv"
HEART_BEAT_REACHED_TO_LIMIT_MESSAGE =       "ERROR - heartbeat had reached to the limit of heartbeat set from the config file"
HEART_BEAT_TIMEOUT_MESSAGE  =               "ERROR - the duration between the heartbeats are more then expected from json file"
EXPECT_ANSWER_NODE =                        "expectedAnswer"
CHOOSE_ONE_OF_THE_ANSWERS_MESSAGE =         " please choose one of the answers :"
LAST_STEP_TYPE =                            "nstep"
HEART_BEAT_REPEATS_SUFFIX =                 "Repeats"
ADD_TEST_TO_SPECIFIC_FOLDER_MESSAGE =       "would you like to add the test to specific folder ? (select yes or no)"
CLI_SESSION                        =        "CLISession"
GRANT_BEFORE_HEARTBEAT_ERROR       =        "ERROR - heartbeat request can be only after grant request"
SELECT_TO_ADD_TEST_MESSAGE         =        "you have choose to add the test "
SELECT_TO_ADD_FOLDER_MESSAGE    =           " to the specific folder :"
SELECTED_TEST_FROM_USER_MESSAGE =           "the selected test from the user : "
PASS_MESSAGE =                              "PASS"
FAIL_MESSAGE =                              "FAIL"
JSON_REQUEST_NOT_INCLUDE_KEY =              "ERROR - the http request does not include key : "
JSON_THIS_CBRS_STEPS_HAD_BEEN_FINISHED =    {
                                                "message" : "the specific cbrs had finished all steps successfully but the domain proxy still have cbrs that not finished all steps",
                                                "response": {
                                                    "responseCode": 0
                                                }
                                            }
REGISTRATION_SUFFIX_HTTP =                  "registration"
DEREGISTRATION_SUFFIX_HTTP =                "deregistration"
EMPTY_CSV_FILE_NAME_MESSAGE =               "cannot enter empty line for csv file try again"
ENTER_YES_OR_NO_MESSAGE =                   "valid input to the question is yes or no"
TYPE_NAME_OF_FOLDER      =                  "enter log of test to the specific folder"
VALIDATION_PASSED_MESSAGE =                 "validation passed successfully, the engine sent "
GOODBYE_MESSAGE =                           "The last test had been finished and no other csv file had been entered,  goodbye"
QUIT_PROGRAM_MESSAGE =                      "you decided to quit the program have a good day"
START_POWER_MEAS_TEST =                     "The Mock-SAS has been started please enabling CBSD for the power measurement test, the Mock-SAS will keep running during the test"
BAD_CARRIER_BW_SELECT =                     "The combination of carrier number and bandwidth is not valid, please try again"<commit_msg>Change heartbeatInterval grace period to 5 seconds<commit_after># Copyright 2017 CBSD Project Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

### Program Constants
WINNF_PROTOCOL_VERSION =                    "v1.1"
SHORTER_GRANT_EXPIRY_TIME =                 360
DEFAULT_RULE_APPLIED =                      "FCC_PART_96"
DEFAULT_CHANNEL_TYPE =                      "GAA"
SECONDS_TO_ADD_FOR_GRANT_EXPIRE_TIME =      604800          # seconds ahead for grantExpireTime - 1 week
SECONDS_TO_ADD_FOR_TX_EXPIRE_TIME =         200             # seconds ahead for transmitExpireTime
SECONDS_FOR_DELAY_RESPONSE =                SECONDS_TO_ADD_FOR_TX_EXPIRE_TIME   # Worst-case value
CPI_SIGNATURE_VALID_TYPES =                 ['RS256', 'ES256']
CPI_INSTALLPARAM_CATA_OPTIONAL =            ['antennaAzimuth', 'antennaDowntilt', 'antennaBeamwidth']   # params REG-Cond for Cat B, optional for Cat A

HEARTBEAT_INTERVAL =                        60
HEARTBEAT_INTERVAL_GRACE_PERIOD =           5.0    # seconds

CBRS_SPECTRUM_HIGH =                        3700000000
CBRS_SPECTRUM_LOW =                         3550000000
MAXEIRP_LOW =                               -137
MAXERIP_HIGH =                              37

SPECTRUM_PAL_HIGH =                         3650000000
SPECTRUM_PAL_LOW =                          3550000000
SPECTRUM_GAA_HIGH =                         3700000000
SPECTRUM_GAA_LOW =                          3550000000

WINNF_APPROVED_CIPHER_LIST = [
    'AES128-GCM-SHA256',                # TLS_RSA_WITH_AES_128_GCM_SHA256
    'AES256-GCM-SHA384',                # TLS_RSA_WITH_AES_256_GCM_SHA384
    'ECDHE-ECDSA-AES128-GCM-SHA256',    # TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
    'ECDHE-ECDSA-AES256-GCM-SHA384',    # TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
    'ECDHE-RSA-AES128-GCM-SHA256',      # TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
    '@STRENGTH'
    ]

WINNF_APPROVED_CIPHERS = ':'.join(WINNF_APPROVED_CIPHER_LIST) # string of ciphers separated by colons

MAX_NUM_HBT_FOR_MEASREPORT =    5   # number of HBT responses for measReport to appear, before failing test

### Other Constants
ERROR_VALIDATION_MESSAGE =                  "ERROR - An Error occurred while comparing between actual and expected request"
RESULTS_OF_TEST_MESSAGE =                   "The final result of the test : "
SET_CSV_FILE_MESSAGE =                      "Please enter a csv file path that include the test you request to run (pay attention insert the csv file without the csv suffix) \n, type quit and press ctrl+c to close"
START_TEST_MESSAGE =                        "Start of the test"
CLOSE_USER_SESSION =                        "Session user will now close"
SENT_FLASK_REQUEST =                        "sent flask request to engine"
SHUTDOWN_FUNCTION_NAME =                    'shutdown'
TEST_HAD_BEEN_FINISHED_FLASK =              " all the steps of the test had been finished see final results at the console "
NAME_OF_SERVER_WERKZUG =                    "werkzeug.server.shutdown"
SERVER_SHUT_DOWN_MESSAGE =                  "Server shutting down due to"
E_NODE_B_SENT_TO_ENGINE_REQUEST =           "cbsd sent request to engine from server"
INIT_TEST_DEFINITION =                      "init the test definition by the test steps sent from the user"
SUFFIX_OF_JSON_FILE =                       ".json"
COMPARING_JSONS_MESSAGE =                   "compare between requested json to expected json"
REQUEST_NODE_NAME =                         "Request"
RESPONSE_NODE_NAME =                        "response"
QUESTION_NODE_NAME =                        "questions"
HEART_BEAT_SUFFIX_HTTP =                    "heartbeat"
SPECTRUM_INQUIERY_SUFFIX_HTTP =             "spectrumInquiry"
HEART_BEAT_ARRIVED_MESSAGE =                "heartBeat request arrived to engine"
GRANT_SUFFIX_HTTP =                         "grant"
RELINQUISHMENT_SUFFIX_HTTP =                "relinquishment"
NSTEP_SESSION_WITH_TECHNITIAN =             "arrived to nstep starting question answer session with the technician"
HEARTBEAT_FROM_ENGINE_TO_CBSD_MESSAGE =      "return heart beat part of json from engine to the cbsd"
ADDITIONAL_COMMENTS_MESSAGE =               "the additional comments for the current test are : "
ANSWERS_NODE =                              "answers"
QUESTION_NODE =                             "question"
SUFFIX_NOT_EXISTS_IN_EXPECTED_JSON_FILE =   "ERROR - the suffix of the type of calling from http request is not exists in the expected json file from the csv"
HEART_BEAT_REACHED_TO_LIMIT_MESSAGE =       "ERROR - heartbeat had reached to the limit of heartbeat set from the config file"
HEART_BEAT_TIMEOUT_MESSAGE  =               "ERROR - the duration between the heartbeats are more then expected from json file"
EXPECT_ANSWER_NODE =                        "expectedAnswer"
CHOOSE_ONE_OF_THE_ANSWERS_MESSAGE =         " please choose one of the answers :"
LAST_STEP_TYPE =                            "nstep"
HEART_BEAT_REPEATS_SUFFIX =                 "Repeats"
ADD_TEST_TO_SPECIFIC_FOLDER_MESSAGE =       "would you like to add the test to specific folder ? (select yes or no)"
CLI_SESSION                        =        "CLISession"
GRANT_BEFORE_HEARTBEAT_ERROR       =        "ERROR - heartbeat request can be only after grant request"
SELECT_TO_ADD_TEST_MESSAGE         =        "you have choose to add the test "
SELECT_TO_ADD_FOLDER_MESSAGE    =           " to the specific folder :"
SELECTED_TEST_FROM_USER_MESSAGE =           "the selected test from the user : "
PASS_MESSAGE =                              "PASS"
FAIL_MESSAGE =                              "FAIL"
JSON_REQUEST_NOT_INCLUDE_KEY =              "ERROR - the http request does not include key : "
JSON_THIS_CBRS_STEPS_HAD_BEEN_FINISHED =    {
                                                "message" : "the specific cbrs had finished all steps successfully but the domain proxy still have cbrs that not finished all steps",
                                                "response": {
                                                    "responseCode": 0
                                                }
                                            }
REGISTRATION_SUFFIX_HTTP =                  "registration"
DEREGISTRATION_SUFFIX_HTTP =                "deregistration"
EMPTY_CSV_FILE_NAME_MESSAGE =               "cannot enter empty line for csv file try again"
ENTER_YES_OR_NO_MESSAGE =                   "valid input to the question is yes or no"
TYPE_NAME_OF_FOLDER      =                  "enter log of test to the specific folder"
VALIDATION_PASSED_MESSAGE =                 "validation passed successfully, the engine sent "
GOODBYE_MESSAGE =                           "The last test had been finished and no other csv file had been entered,  goodbye"
QUIT_PROGRAM_MESSAGE =                      "you decided to quit the program have a good day"
START_POWER_MEAS_TEST =                     "The Mock-SAS has been started please enabling CBSD for the power measurement test, the Mock-SAS will keep running during the test"
BAD_CARRIER_BW_SELECT =                     "The combination of carrier number and bandwidth is not valid, please try again"<|endoftext|>


<commit_before># Author: Stefan Wunsch CERN  09/2019

################################################################################
# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers.                      #
# All rights reserved.                                                         #
#                                                                              #
# For the licensing terms see $ROOTSYS/LICENSE.                                #
# For the list of contributors see $ROOTSYS/README/CREDITS.                    #
################################################################################

from ROOT import pythonization
from libROOTPythonizations import AsRVec
import numpy as np

try:
    from libROOTPythonizations import AsRTensor
    has_rtensor = True
except:
    has_rtensor = False

def Compute(self, x):
    # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray
    if isinstance(x, np.ndarray):
        if len(x.shape) == 1:
            x_ = AsRVec(x)
            y = self._OriginalCompute(x_)
            return np.asarray(y)
        elif len(x.shape) == 2:
            x_ = AsRTensor(x)
            y = self._OriginalCompute(x_)
            return np.asarray(y)
        else:
            raise Exception("Call to Compute can process only numpy arrays of rank 1 or 2.")

    # As fall-through we go to the original compute function and use the error-handling from cppyy
    return self._OriginalCompute(x)

@pythonization()
def pythonize_rbdt(klass, name):
    # Parameters:
    # klass: class to be pythonized
    # name: name of the class

    if name.startswith("TMVA::Experimental::RBDT") and has_rtensor:
        klass._OriginalCompute = klass.Compute
        klass.Compute = Compute

    return True
<commit_msg>[Exp PyROOT] Protect import of numpy in RBDT pythonization<commit_after># Author: Stefan Wunsch CERN  09/2019

################################################################################
# Copyright (C) 1995-2019, Rene Brun and Fons Rademakers.                      #
# All rights reserved.                                                         #
#                                                                              #
# For the licensing terms see $ROOTSYS/LICENSE.                                #
# For the list of contributors see $ROOTSYS/README/CREDITS.                    #
################################################################################

from ROOT import pythonization
from libROOTPythonizations import AsRVec

try:
    from libROOTPythonizations import AsRTensor
    has_rtensor = True
except:
    has_rtensor = False

def Compute(self, x):
    # Import numpy lazily
    try:
        import numpy as np
    except:
        raise ImportError("Failed to import numpy during call of RBDT::Compute.")

    # numpy.array is a factory and the actual type of a numpy array is numpy.ndarray
    if isinstance(x, np.ndarray):
        if len(x.shape) == 1:
            x_ = AsRVec(x)
            y = self._OriginalCompute(x_)
            return np.asarray(y)
        elif len(x.shape) == 2:
            x_ = AsRTensor(x)
            y = self._OriginalCompute(x_)
            return np.asarray(y)
        else:
            raise Exception("Call to Compute can process only numpy arrays of rank 1 or 2.")

    # As fall-through we go to the original compute function and use the error-handling from cppyy
    return self._OriginalCompute(x)

@pythonization()
def pythonize_rbdt(klass, name):
    # Parameters:
    # klass: class to be pythonized
    # name: name of the class

    if name.startswith("TMVA::Experimental::RBDT") and has_rtensor:
        klass._OriginalCompute = klass.Compute
        klass.Compute = Compute

    return True
<|endoftext|>


<commit_before>"""This module tests GET requests to the Cloud Sumamry Record endpoint."""

import logging
import MySQLdb

from api.views.CloudRecordSummaryView import CloudRecordSummaryView
from django.test import Client, TestCase
from mock import Mock
from rest_framework.test import APIRequestFactory

QPATH_TEST = '/tmp/django-test/'

class CloudRecordSummaryTest(TestCase):
    """Tests GET requests to the Cloud Sumamry Record endpoint."""

    def setUp(self):
        """Prevent logging from appearing in test output."""
        logging.disable(logging.CRITICAL)

    def test_cloud_record_summary_get_403(self):
        """Test a unauthorized GET request."""
        # Mock the functionality of the IAM
        CloudRecordSummaryView._token_to_id = Mock(return_value="FakeService")

        with self.settings(ALLOWED_FOR_GET='TestService',
                           RETURN_HEADERS=["WallDuration",
                                           "Day",
                                           "Month",
                                           "Year"]):
            test_client = Client()
            response = test_client.get(
                                    '/api/v1/cloud/record/summary?'
                                    'group=TestGroup&'
                                    'from=20000101&to=20191231',
                                    HTTP_AUTHORIZATION="Bearer TestToken")

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 403)

    def test_cloud_record_summary_get_401(self):
        """Test a unauthenticated GET request."""
        test_client = Client()
        # Test without the HTTP_AUTHORIZATION header
        response = test_client.get(
                                '/api/v1/cloud/record/summary?'
                                'group=TestGroup&'
                                'from=20000101&to=20191231')

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 401)

        # Test with a malformed HTTP_AUTHORIZATION header
        response = test_client.get(
                                '/api/v1/cloud/record/summary?'
                                'group=TestGroup&'
                                'from=20000101&to=20191231',
                                HTTP_AUTHORIZATION='TestToken')

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 401)

    def test_cloud_record_summary_get_200(self):
        """Test a successful GET request."""
        # Connect to database
        database = self._connect_to_database()
        # Clean up any lingering example data.
        self._clear_database(database)
        # Add example data
        self._populate_database(database)

        # Mock the functionality of the IAM
        CloudRecordSummaryView._token_to_id = Mock(return_value="TestService")

        with self.settings(ALLOWED_FOR_GET='TestService',
                           RETURN_HEADERS=["WallDuration",
                                           "Day",
                                           "Month",
                                           "Year"]):
            test_client = Client()
            response = test_client.get(
                                    '/api/v1/cloud/record/summary?'
                                    'group=TestGroup&'
                                    'from=20000101&to=20191231',
                                    HTTP_AUTHORIZATION="Bearer TestToken")

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 200)

        expected_response = ('{'
                             '"count":2,'
                             '"next":null,'
                             '"previous":null,'
                             '"results":[{'
                             '"WallDuration":86399,'
                             '"Year":2016,'
                             '"Day":30,'
                             '"Month":7'
                             '},{'
                             '"WallDuration":43200,'
                             '"Year":2016,'
                             '"Day":31,'
                             '"Month":7}]}')

        # Check the response received is as expected.
        self.assertEqual(response.content, expected_response)
        # Clean up after test.
        self._clear_database(database)
        database.close()

    def test_parse_query_parameters(self):
        """Test the parsing of query parameters."""
        # test a get with group, summary, start and end
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get(('/api/v1/cloud/record/summary?'
                               'group=Group1&'
                               'service=Service1&'
                               'from=FromDate&to=ToDate'), {})

        parsed_responses = test_cloud_view._parse_query_parameters(request)
        self.assertEqual(parsed_responses,
                         ("Group1", "Service1", "FromDate", "ToDate"))

        # test a get with just an end date
        request = factory.get('/api/v1/cloud/record/summary?to=ToDate', {})
        parsed_responses = test_cloud_view._parse_query_parameters(request)
        self.assertEqual(parsed_responses,
                         (None, None, None, "ToDate"))

    def test_paginate_result(self):
        """Test an empty result is paginated correctly."""
        # test when no page is given.
        test_cloud_view = CloudRecordSummaryView()
        content = test_cloud_view._paginate_result(None, [])
        expected_content = {'count': 0,
                            'previous': None,
                            u'results': [],
                            'next': None}

        self.assertEqual(content, expected_content)

        # test when page number is incorrect/invalid
        factory = APIRequestFactory()

        # test when page number is not a number
        request = factory.get('/api/v1/cloud/record/summary?page=a')
        content = test_cloud_view._paginate_result(request, [])
        self.assertEqual(content, expected_content)

        # test when page number is out of bounds
        request = factory.get('/api/v1/cloud/record/summary?page=9999')
        content = test_cloud_view._paginate_result(request, [])
        self.assertEqual(content, expected_content)

    def test_request_to_token(self):
        """Test a token can be extracted from request."""
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get('/api/v1/cloud/record/summary?from=FromDate',
                              HTTP_AUTHORIZATION='Bearer ThisIsAToken')

        token = test_cloud_view._request_to_token(request)
        self.assertEqual(token, 'ThisIsAToken')

    def test_request_to_token_fail(self):
        """Test the response of a tokenless request."""
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get('/api/v1/cloud/record/summary?from=FromDate')

        self.assertRaises(KeyError, test_cloud_view._request_to_token, request)

    def test_is_client_authorized(self):
        """Test a example client is authorised."""
        test_cloud_view = CloudRecordSummaryView()
        with self.settings(ALLOWED_FOR_GET='IAmAllowed'):
            self.assertTrue(
                test_cloud_view._is_client_authorized(
                    'IAmAllowed'))

    def test_is_client_authorized_fail(self):
        """Test the failure of un-authorised clients."""
        test_cloud_view = CloudRecordSummaryView()
        with self.settings(ALLOWED_FOR_GET='IAmAllowed'):
            self.assertFalse(
                test_cloud_view._is_client_authorized(
                    'IAmNotAllowed'))

    def test_filter_cursor(self):
        """Test the filtering of a query object based on settings."""
        test_cloud_view = CloudRecordSummaryView()

        # this approximates a cursor object
        # in future, a cursor dict may be better
        # to use in _filter_cursor
        cursor_headers = [['SiteName'], ['Day'], ['Month'], ['Year']]
        cursor_data = [[['SiteName', 'Test'],
                        ['Day', 01],
                        ['Month', 02],
                        ['Year', 2000]]]

        cursor = Mock()
        cursor.description = cursor_headers
        cursor.fetchall = Mock(return_value=cursor_data)

        with self.settings(RETURN_HEADERS=['SiteName', 'Day']):
            result = test_cloud_view._filter_cursor(cursor)

        expected_result = [{'SiteName': ['SiteName', 'Test'],
                            'Day': ['Day', 1]}]
        self.assertEqual(result, expected_result)

    def tearDown(self):
        """Delete any messages under QPATH and re-enable logging.INFO."""
        logging.disable(logging.NOTSET)

    def _populate_database(self, database):
        """Populate the database with example summaries."""
        cursor = database.cursor()

        # Insert example usage data
        cursor.execute('INSERT INTO CloudRecords '
                       '(VMUUID, SiteID, GlobalUserNameID, VOID, '
                       'VOGroupID, VORoleID, Status, StartTime, '
                       'SuspendDuration, WallDuration, PublisherDNID, '
                       'CloudType, ImageId) '
                       'VALUES '
                       '("TEST-VM", 1, 1, 1, 1, 1, "Running", '
                       '"2016-07-30 00:00:00", 0, 86399, 1, "TEST", "1");')

        # Insert example usage data
        cursor.execute('INSERT INTO CloudRecords '
                       '(VMUUID, SiteID, GlobalUserNameID, VOID, '
                       'VOGroupID, VORoleID, Status, StartTime, '
                       'SuspendDuration, WallDuration, PublisherDNID, '
                       'CloudType, ImageId) '
                       'VALUES '
                       '("TEST-VM", 1, 1, 1, 1, 1, "Running", '
                       '"2016-07-30 00:00:00", 0, 129599, 1, "TEST", "1");')

        # These INSERT statements are needed
        # because we query VCloudSummaries
        cursor.execute('INSERT INTO Sites VALUES (1, "TestSite");')
        cursor.execute('INSERT INTO VOs VALUES (1, "TestVO");')
        cursor.execute('INSERT INTO VOGroups VALUES (1, "TestGroup");')
        cursor.execute('INSERT INTO VORoles VALUES (1, "TestRole");')
        cursor.execute('INSERT INTO DNs VALUES (1, "TestDN");')

        # Summarise example usage data
        cursor.execute('CALL SummariseVMs();')
        database.commit()

    def _clear_database(self, database):
        """Clear the database of example data."""
        cursor = database.cursor()

        cursor.execute('DELETE FROM CloudRecords '
                       'WHERE VMUUID="TEST-VM";')

        cursor.execute('DELETE FROM CloudSummaries '
                       'WHERE CloudType="TEST";')

        cursor.execute('DELETE FROM Sites '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VOs '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VOGroups '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VORoles '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM DNs '
                       'WHERE id=1;')

        database.commit()

    def _connect_to_database(self,
                             host='localhost',
                             user='root',
                             password='',
                             name='apel_rest'):
        """Connect to and return a cursor to the given database."""
        database = MySQLdb.connect(host, user, password, name)
        return database
<commit_msg>Add a 400 unit test<commit_after>"""This module tests GET requests to the Cloud Sumamry Record endpoint."""

import logging
import MySQLdb

from api.views.CloudRecordSummaryView import CloudRecordSummaryView
from django.test import Client, TestCase
from mock import Mock
from rest_framework.test import APIRequestFactory

QPATH_TEST = '/tmp/django-test/'

class CloudRecordSummaryTest(TestCase):
    """Tests GET requests to the Cloud Sumamry Record endpoint."""

    def setUp(self):
        """Prevent logging from appearing in test output."""
        logging.disable(logging.CRITICAL)

    def test_cloud_record_summary_get_400(self):
        """Test a GET request without the from field."""
        # Mock the functionality of the IAM
        CloudRecordSummaryView._token_to_id = Mock(return_value="TestService")

        with self.settings(ALLOWED_FOR_GET='TestService',
                           RETURN_HEADERS=["WallDuration",
                                           "Day",
                                           "Month",
                                           "Year"]):
            test_client = Client()
            response = test_client.get(
                                    '/api/v1/cloud/record/summary?'
                                    'group=TestGroup',
                                    HTTP_AUTHORIZATION="Bearer TestToken")

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 400)

    def test_cloud_record_summary_get_403(self):
        """Test a unauthorized GET request."""
        # Mock the functionality of the IAM
        CloudRecordSummaryView._token_to_id = Mock(return_value="FakeService")

        with self.settings(ALLOWED_FOR_GET='TestService',
                           RETURN_HEADERS=["WallDuration",
                                           "Day",
                                           "Month",
                                           "Year"]):
            test_client = Client()
            response = test_client.get(
                                    '/api/v1/cloud/record/summary?'
                                    'group=TestGroup&'
                                    'from=20000101&to=20191231',
                                    HTTP_AUTHORIZATION="Bearer TestToken")

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 403)

    def test_cloud_record_summary_get_401(self):
        """Test a unauthenticated GET request."""
        test_client = Client()
        # Test without the HTTP_AUTHORIZATION header
        response = test_client.get(
                                '/api/v1/cloud/record/summary?'
                                'group=TestGroup&'
                                'from=20000101&to=20191231')

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 401)

        # Test with a malformed HTTP_AUTHORIZATION header
        response = test_client.get(
                                '/api/v1/cloud/record/summary?'
                                'group=TestGroup&'
                                'from=20000101&to=20191231',
                                HTTP_AUTHORIZATION='TestToken')

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 401)

    def test_cloud_record_summary_get_200(self):
        """Test a successful GET request."""
        # Connect to database
        database = self._connect_to_database()
        # Clean up any lingering example data.
        self._clear_database(database)
        # Add example data
        self._populate_database(database)

        # Mock the functionality of the IAM
        CloudRecordSummaryView._token_to_id = Mock(return_value="TestService")

        with self.settings(ALLOWED_FOR_GET='TestService',
                           RETURN_HEADERS=["WallDuration",
                                           "Day",
                                           "Month",
                                           "Year"]):
            test_client = Client()
            response = test_client.get(
                                    '/api/v1/cloud/record/summary?'
                                    'group=TestGroup&'
                                    'from=20000101&to=20191231',
                                    HTTP_AUTHORIZATION="Bearer TestToken")

        # Check the expected response code has been received.
        self.assertEqual(response.status_code, 200)

        expected_response = ('{'
                             '"count":2,'
                             '"next":null,'
                             '"previous":null,'
                             '"results":[{'
                             '"WallDuration":86399,'
                             '"Year":2016,'
                             '"Day":30,'
                             '"Month":7'
                             '},{'
                             '"WallDuration":43200,'
                             '"Year":2016,'
                             '"Day":31,'
                             '"Month":7}]}')

        # Check the response received is as expected.
        self.assertEqual(response.content, expected_response)
        # Clean up after test.
        self._clear_database(database)
        database.close()

    def test_parse_query_parameters(self):
        """Test the parsing of query parameters."""
        # test a get with group, summary, start and end
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get(('/api/v1/cloud/record/summary?'
                               'group=Group1&'
                               'service=Service1&'
                               'from=FromDate&to=ToDate'), {})

        parsed_responses = test_cloud_view._parse_query_parameters(request)
        self.assertEqual(parsed_responses,
                         ("Group1", "Service1", "FromDate", "ToDate"))

        # test a get with just an end date
        request = factory.get('/api/v1/cloud/record/summary?to=ToDate', {})
        parsed_responses = test_cloud_view._parse_query_parameters(request)
        self.assertEqual(parsed_responses,
                         (None, None, None, "ToDate"))

    def test_paginate_result(self):
        """Test an empty result is paginated correctly."""
        # test when no page is given.
        test_cloud_view = CloudRecordSummaryView()
        content = test_cloud_view._paginate_result(None, [])
        expected_content = {'count': 0,
                            'previous': None,
                            u'results': [],
                            'next': None}

        self.assertEqual(content, expected_content)

        # test when page number is incorrect/invalid
        factory = APIRequestFactory()

        # test when page number is not a number
        request = factory.get('/api/v1/cloud/record/summary?page=a')
        content = test_cloud_view._paginate_result(request, [])
        self.assertEqual(content, expected_content)

        # test when page number is out of bounds
        request = factory.get('/api/v1/cloud/record/summary?page=9999')
        content = test_cloud_view._paginate_result(request, [])
        self.assertEqual(content, expected_content)

    def test_request_to_token(self):
        """Test a token can be extracted from request."""
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get('/api/v1/cloud/record/summary?from=FromDate',
                              HTTP_AUTHORIZATION='Bearer ThisIsAToken')

        token = test_cloud_view._request_to_token(request)
        self.assertEqual(token, 'ThisIsAToken')

    def test_request_to_token_fail(self):
        """Test the response of a tokenless request."""
        test_cloud_view = CloudRecordSummaryView()
        factory = APIRequestFactory()
        request = factory.get('/api/v1/cloud/record/summary?from=FromDate')

        self.assertRaises(KeyError, test_cloud_view._request_to_token, request)

    def test_is_client_authorized(self):
        """Test a example client is authorised."""
        test_cloud_view = CloudRecordSummaryView()
        with self.settings(ALLOWED_FOR_GET='IAmAllowed'):
            self.assertTrue(
                test_cloud_view._is_client_authorized(
                    'IAmAllowed'))

    def test_is_client_authorized_fail(self):
        """Test the failure of un-authorised clients."""
        test_cloud_view = CloudRecordSummaryView()
        with self.settings(ALLOWED_FOR_GET='IAmAllowed'):
            self.assertFalse(
                test_cloud_view._is_client_authorized(
                    'IAmNotAllowed'))

    def test_filter_cursor(self):
        """Test the filtering of a query object based on settings."""
        test_cloud_view = CloudRecordSummaryView()

        # this approximates a cursor object
        # in future, a cursor dict may be better
        # to use in _filter_cursor
        cursor_headers = [['SiteName'], ['Day'], ['Month'], ['Year']]
        cursor_data = [[['SiteName', 'Test'],
                        ['Day', 01],
                        ['Month', 02],
                        ['Year', 2000]]]

        cursor = Mock()
        cursor.description = cursor_headers
        cursor.fetchall = Mock(return_value=cursor_data)

        with self.settings(RETURN_HEADERS=['SiteName', 'Day']):
            result = test_cloud_view._filter_cursor(cursor)

        expected_result = [{'SiteName': ['SiteName', 'Test'],
                            'Day': ['Day', 1]}]
        self.assertEqual(result, expected_result)

    def tearDown(self):
        """Delete any messages under QPATH and re-enable logging.INFO."""
        logging.disable(logging.NOTSET)

    def _populate_database(self, database):
        """Populate the database with example summaries."""
        cursor = database.cursor()

        # Insert example usage data
        cursor.execute('INSERT INTO CloudRecords '
                       '(VMUUID, SiteID, GlobalUserNameID, VOID, '
                       'VOGroupID, VORoleID, Status, StartTime, '
                       'SuspendDuration, WallDuration, PublisherDNID, '
                       'CloudType, ImageId) '
                       'VALUES '
                       '("TEST-VM", 1, 1, 1, 1, 1, "Running", '
                       '"2016-07-30 00:00:00", 0, 86399, 1, "TEST", "1");')

        # Insert example usage data
        cursor.execute('INSERT INTO CloudRecords '
                       '(VMUUID, SiteID, GlobalUserNameID, VOID, '
                       'VOGroupID, VORoleID, Status, StartTime, '
                       'SuspendDuration, WallDuration, PublisherDNID, '
                       'CloudType, ImageId) '
                       'VALUES '
                       '("TEST-VM", 1, 1, 1, 1, 1, "Running", '
                       '"2016-07-30 00:00:00", 0, 129599, 1, "TEST", "1");')

        # These INSERT statements are needed
        # because we query VCloudSummaries
        cursor.execute('INSERT INTO Sites VALUES (1, "TestSite");')
        cursor.execute('INSERT INTO VOs VALUES (1, "TestVO");')
        cursor.execute('INSERT INTO VOGroups VALUES (1, "TestGroup");')
        cursor.execute('INSERT INTO VORoles VALUES (1, "TestRole");')
        cursor.execute('INSERT INTO DNs VALUES (1, "TestDN");')

        # Summarise example usage data
        cursor.execute('CALL SummariseVMs();')
        database.commit()

    def _clear_database(self, database):
        """Clear the database of example data."""
        cursor = database.cursor()

        cursor.execute('DELETE FROM CloudRecords '
                       'WHERE VMUUID="TEST-VM";')

        cursor.execute('DELETE FROM CloudSummaries '
                       'WHERE CloudType="TEST";')

        cursor.execute('DELETE FROM Sites '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VOs '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VOGroups '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM VORoles '
                       'WHERE id=1;')

        cursor.execute('DELETE FROM DNs '
                       'WHERE id=1;')

        database.commit()

    def _connect_to_database(self,
                             host='localhost',
                             user='root',
                             password='',
                             name='apel_rest'):
        """Connect to and return a cursor to the given database."""
        database = MySQLdb.connect(host, user, password, name)
        return database
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
from six.moves import range

class LossAccPlot(object):
    """Class to plot training and validation loss and accuracy.
    """
    def __init__(self,
                 save_to_filepath=None,
                 show_regressions=True,
                 show_averages=True,
                 show_loss_plot=True,
                 show_acc_plot=True,
                 show_plot_window=True):
        """Constructs the plotter.
        Args:
            save_to_filepath: The filepath to a file at which the plot
                is ought to be saved, e.g. "/tmp/last_plot.png". Set this value
                to None if you don't want to save the plot.
            show_plot_window: Whether to show the plot in a window (True)
                or hide it (False). Hiding it makes only sense if you
                set save_to_filepath.
            linestyles: List of two string values containing the stylings
                of the chart lines. The first value is for the training
                line, the second for the validation line. Loss and accuracy
                charts will both use that styling.
            linestyles_first_epoch: Different stylings for the chart lines
                for the very first epoch (no two points yet to draw a line).
            show_regression: Whether or not to show a regression, indicating
                where each line might end up in the future.
            poly_forward_perc: Percentage value (e.g. 0.1 = 10%) indicating
                for how far in the future each regression line will be
                calculated. The percentage is relative to the current epoch.
                E.g. if epoch is 100 and this value is 0.2, then the regression
                will be calculated for 20 values in the future.
            poly_backward_perc: Similar to poly_forward_perc. Percentage of
                the data basis to use in order to calculate the regression.
                E.g. if epoch is 100 and this value is 0.2, then the last
                20 values will be used to predict the future values.
            poly_n_forward_min: Minimum value for how far in the future
                the regression values will be predicted for each line.
                E.g. 10 means that there will always be at least 10 predicted
                values, even for e.g. epoch 5.
            poly_n_backward_min: Similar to poly_n_forward_min. Minimum
                epochs to use backwards for predicting future values.
            poly_degree: Degree of the polynomial to use when predicting
                future values. Should usually be 1.
        """
        assert show_loss_plot or show_acc_plot
        assert save_to_filepath is not None or show_plot_window

        self.linestyles = linestyles
        self.linestyles_first_epoch = linestyles_first_epoch
        self.show_regressions = show_regressions
        self.show_averages = show_averages
        self.show_loss_plot = show_loss_plot
        self.show_acc_plot = show_acc_plot
        self.show_plot_window = show_plot_window
        self.save_to_filepath = save_to_filepath

        self.poly_forward_perc=0.1,
        self.poly_backward_perc=0.2,
        self.poly_n_forward_min=5,
        self.poly_n_backward_min=10,
        self.poly_degree=1

        self.linestyles = {
            "loss_train": "r-",
            "loss_train_regression": "r:",
            "loss_val": "b-",
            "loss_val_regression": "b:",
            "acc_train": "r-",
            "acc_train_regression": "r:",
            "acc_val": "b-",
            "acc_val_regression": "b:"
        }
        # different linestyles for the first epoch, because there will be only
        # one value available => no line can be drawn
        self.linestyles_first_epoch = {
            "loss_train": "rs-",
            "loss_train_regression": "r:",
            "loss_val": "b^-",
            "loss_val_regression": "b^:",
            "acc_train": "rs-",
            "acc_train_regression": "r:",
            "acc_val": "b^-",
            "acc_val_regression": "b^:"
        }

        # ----
        # Initialize plots
        # ----
        if show_loss_plot and show_acc_plot:
            fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(24, 8))
            self.fig = fig
            self.ax_loss = ax1
            self.ax_acc = ax2
        else:
            fig, (ax,) = plt.subplots(ncols=1, figsize=(12, 8))
            self.fig = fig
            self.ax_loss = ax if show_loss_plot else None
            self.ax_acc = ax if show_acc_plot else None

        # set_position is neccessary here in order to place the legend properly
        for ax in [self.ax_loss, self.ax_acc]:
            if ax is not None:
                box = ax.get_position()
                ax.set_position([box.x0, box.y0 + box.height * 0.1,
                                 box.width, box.height * 0.9])

        self.values_loss_train_x = []
        self.values_loss_val_x = []
        self.values_acc_train_x = []
        self.values_acc_val_x = []
        self.values_loss_train_y = []
        self.values_loss_val_y = []
        self.values_acc_train_y = []
        self.values_acc_val_y = []

    def add_values(self, x_index, loss_train=None, loss_val=None, acc_train=None,
                   acc_val=None, redraw=True):
        if loss_train is not None:
            self.values_loss_train_x.append(x_index)
            self.values_loss_train_y.append(loss_train)
        if loss_val is not None:
            self.values_loss_val_x.append(x_index)
            self.values_loss_val_y.append(loss_val)
        if acc_train is not None:
            self.values_acc_train_x.append(x_index)
            self.values_acc_train_y.append(acc_train)
        if acc_val is not None:
            self.values_acc_val_x.append(x_index)
            self.values_acc_val_y.append(acc_val)

        if redraw:
            self._redraw_plot()

    def update(self, epoch, train_loss, train_acc, val_loss, val_acc):
        """Updates the plot with the latest data.
        Args:
            train_loss: All of the training loss values of each
                epoch (list of floats).
            train_acc: All of the training accuracy values of each
                epoch (list of floats).
            val_loss: All of the validation loss values of each
                epoch (list of floats).
            val_acc: All of the validation accuracy values of each
                epoch (list of floats).
        """
        self._redraw_plot(epoch, train_loss, train_acc, val_loss, val_acc)

        # show plot window or redraw an existing one
        if self.show_plot_window:
            plt.figure(self.fig.number)
            plt.show(block=False)
            plt.draw()

        # save
        if self.save_to_filepath:
            self._save_plot(self.save_to_filepath)

    def _save_plot(self, filepath):
        """Saves the current plot to a file.
        Args:
            filepath: The path to the file, e.g. "/tmp/last_plot.png".
        """
        self.fig.savefig(filepath)

    def _redraw_plot(self, epoch, train_loss, train_acc, val_loss, val_acc):
        """Redraws the plot with new values.
        Args:
            epoch: The index of the current epoch, starting at 0.
            train_loss: All of the training loss values of each
                epoch (list of floats).
            train_acc: All of the training accuracy values of each
                epoch (list of floats).
            val_loss: All of the validation loss values of each
                epoch (list of floats).
            val_acc: All of the validation accuracy values of each
                epoch (list of floats).
        """

        ax1 = self.ax1
        ax2 = self.ax2

        # List of each epoch (x-axis)
        epochs = list(range(0, epoch+1))

        # Clear loss and accuracy charts
        ax1.clear()
        ax2.clear()

        # Set titles of charts (at the top)
        ax1.set_title('loss')
        ax2.set_title('accuracy')

        # Set the styles of the lines used in the charts
        # Different line style for epochs after the  first one, because
        # the very first epoch has only one data point and therefore no line
        # and would be invisible without the changed style.
        linestyles = self.linestyles if epoch > 0 else self.linestyles_first_epoch

        # Plot the lines
        if train_loss:
            ax1.plot(epochs, train_loss, linestyles[0], label='train loss')
        if val_loss:
            ax1.plot(epochs, val_loss, linestyles[1], label='val loss')
        if train_acc:
            ax2.plot(epochs, train_acc, linestyles[0], label='train acc')
        if val_acc:
            ax2.plot(epochs, val_acc, linestyles[1], label='val acc')

        if self.show_regressions:
            # Compute the regression lines for the n_forward future epochs.
            # n_forward is calculated relative to the current epoch
            # (e.g. at epoch 100 compute 10 next, at 200 the 20 next ones...).
            n_forward = int(max((epoch+1)*self.poly_forward_perc,
                                self.poly_forward_min))

            # Compute regression lines based on n_backwards epochs
            # in the past, e.g. based on the last 10 values.
            # n_backwards is calculated relative to the current epoch
            # (e.g. at epoch 100 compute based on the last 10 values,
            # at 200 based on the last 20 values...).
            n_backwards = int(max((epoch+1)*self.poly_backward_perc,
                                  self.poly_backward_min))

            # List of epochs for which to estimate/predict the likely value.
            # (epoch..epoch+n_forward instead of epoch+1..epoch+n_forward
            # so that the regression line is better connected to the line its
            # based on (no obvious gap).)
            future_epochs = [i for i in range(epoch, epoch + n_forward)]

            self.plot_regression_line(ax1, train_loss, epochs, future_epochs,
                                      n_backwards, linestyles[2],
                                      'train loss regression')
            self.plot_regression_line(ax1, val_loss, epochs, future_epochs,
                                      n_backwards, linestyles[3],
                                      'val loss regression')
            self.plot_regression_line(ax2, train_acc, epochs, future_epochs,
                                      n_backwards, linestyles[2],
                                      'train acc regression')
            self.plot_regression_line(ax2, val_acc, epochs, future_epochs,
                                      n_backwards, linestyles[3],
                                      'val acc regression')

        # Add legend (below chart)
        ax1.legend(['train loss', 'val loss',
                    'train loss regression', 'val loss regression'],
                   bbox_to_anchor=(0.7, -0.08), ncol=2)
        ax2.legend(['train acc', 'val acc',
                    'train acc regression', 'val acc regression'],
                   bbox_to_anchor=(0.7, -0.08), ncol=2)

        # Labels for x and y axis
        ax1.set_ylabel('loss')
        ax1.set_xlabel('epoch')
        ax2.set_ylabel('accuracy')
        ax2.set_xlabel('epoch')

        # Show a grid in both charts
        ax1.grid(True)
        ax2.grid(True)

    def plot_regression_line(self, plot_ax, data, epochs, future_epochs,
                             n_backwards, linestyle, label):
        """Calculates and plots a regression line.
        Args:
            plot_ax: The ax on which to plot the line.
            data: The data used to perform the regression
                (e.g. training loss values).
            epochs: List of all epochs (0, 1, 2, ...).
            future_epochs: List of the future epochs for which values are
                ought to be predicted.
            n_backwards: How far back to go in time (in epochs) in order
                to compute the regression. (E.g. 10 = calculate it on the
                last 10 values max.)
            linestyle: Linestyle of the regression line.
            label: Label of the regression line.
        """
        # dont try to draw anything if the data list is empty or it's the
        # first epoch
        if len(data) > 1:
            poly = np.poly1d(np.polyfit(epochs[-n_backwards:],
                                        data[-n_backwards:], self.poly_degree))
            future_values = [poly(i) for i in future_epochs]
            plot_ax.plot(future_epochs, future_values, linestyle, label=label)
<commit_msg>Changed redraw function<commit_after>from __future__ import absolute_import
from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
from six.moves import range

class LossAccPlot(object):
    """Class to plot training and validation loss and accuracy.
    """
    def __init__(self,
                 save_to_filepath=None,
                 show_regressions=True,
                 show_averages=True,
                 show_loss_plot=True,
                 show_acc_plot=True,
                 show_plot_window=True):
        """Constructs the plotter.
        Args:
            save_to_filepath: The filepath to a file at which the plot
                is ought to be saved, e.g. "/tmp/last_plot.png". Set this value
                to None if you don't want to save the plot.
            show_plot_window: Whether to show the plot in a window (True)
                or hide it (False). Hiding it makes only sense if you
                set save_to_filepath.
            linestyles: List of two string values containing the stylings
                of the chart lines. The first value is for the training
                line, the second for the validation line. Loss and accuracy
                charts will both use that styling.
            linestyles_first_epoch: Different stylings for the chart lines
                for the very first epoch (no two points yet to draw a line).
            show_regression: Whether or not to show a regression, indicating
                where each line might end up in the future.
            poly_forward_perc: Percentage value (e.g. 0.1 = 10%) indicating
                for how far in the future each regression line will be
                calculated. The percentage is relative to the current epoch.
                E.g. if epoch is 100 and this value is 0.2, then the regression
                will be calculated for 20 values in the future.
            poly_backward_perc: Similar to poly_forward_perc. Percentage of
                the data basis to use in order to calculate the regression.
                E.g. if epoch is 100 and this value is 0.2, then the last
                20 values will be used to predict the future values.
            poly_n_forward_min: Minimum value for how far in the future
                the regression values will be predicted for each line.
                E.g. 10 means that there will always be at least 10 predicted
                values, even for e.g. epoch 5.
            poly_n_backward_min: Similar to poly_n_forward_min. Minimum
                epochs to use backwards for predicting future values.
            poly_degree: Degree of the polynomial to use when predicting
                future values. Should usually be 1.
        """
        assert show_loss_plot or show_acc_plot
        assert save_to_filepath is not None or show_plot_window

        self.linestyles = linestyles
        self.linestyles_first_epoch = linestyles_first_epoch
        self.show_regressions = show_regressions
        self.show_averages = show_averages
        self.show_loss_plot = show_loss_plot
        self.show_acc_plot = show_acc_plot
        self.show_plot_window = show_plot_window
        self.save_to_filepath = save_to_filepath

        self.poly_forward_perc=0.1,
        self.poly_backward_perc=0.2,
        self.poly_n_forward_min=5,
        self.poly_n_backward_min=10,
        self.poly_degree=1

        self.linestyles = {
            "loss_train": "r-",
            "loss_train_regression": "r:",
            "loss_val": "b-",
            "loss_val_regression": "b:",
            "acc_train": "r-",
            "acc_train_regression": "r:",
            "acc_val": "b-",
            "acc_val_regression": "b:"
        }
        # different linestyles for the first epoch, because there will be only
        # one value available => no line can be drawn
        self.linestyles_one_value = {
            "loss_train": "rs-",
            "loss_train_regression": "r:",
            "loss_val": "b^-",
            "loss_val_regression": "b^:",
            "acc_train": "rs-",
            "acc_train_regression": "r:",
            "acc_val": "b^-",
            "acc_val_regression": "b^:"
        }

        # ----
        # Initialize plots
        # ----
        if show_loss_plot and show_acc_plot:
            fig, (ax1, ax2) = plt.subplots(ncols=2, figsize=(24, 8))
            self.fig = fig
            self.ax_loss = ax1
            self.ax_acc = ax2
        else:
            fig, (ax,) = plt.subplots(ncols=1, figsize=(12, 8))
            self.fig = fig
            self.ax_loss = ax if show_loss_plot else None
            self.ax_acc = ax if show_acc_plot else None

        # set_position is neccessary here in order to place the legend properly
        for ax in [self.ax_loss, self.ax_acc]:
            if ax is not None:
                box = ax.get_position()
                ax.set_position([box.x0, box.y0 + box.height * 0.1,
                                 box.width, box.height * 0.9])

        self.values_loss_train_x = []
        self.values_loss_val_x = []
        self.values_acc_train_x = []
        self.values_acc_val_x = []
        self.values_loss_train_y = []
        self.values_loss_val_y = []
        self.values_acc_train_y = []
        self.values_acc_val_y = []

    def add_values(self, x_index, loss_train=None, loss_val=None, acc_train=None,
                   acc_val=None, redraw=True):
        if loss_train is not None:
            self.values_loss_train_x.append(x_index)
            self.values_loss_train_y.append(loss_train)
        if loss_val is not None:
            self.values_loss_val_x.append(x_index)
            self.values_loss_val_y.append(loss_val)
        if acc_train is not None:
            self.values_acc_train_x.append(x_index)
            self.values_acc_train_y.append(acc_train)
        if acc_val is not None:
            self.values_acc_val_x.append(x_index)
            self.values_acc_val_y.append(acc_val)

        if redraw:
            self._redraw()
            if self.save_to_filepath:
                self._save_plot(self.save_to_filepath)

    def _save_plot(self, filepath):
        """Saves the current plot to a file.
        Args:
            filepath: The path to the file, e.g. "/tmp/last_plot.png".
        """
        self.fig.savefig(filepath)

    def _redraw(self):
        """Redraws the plot with new values.
        Args:
            epoch: The index of the current epoch, starting at 0.
            train_loss: All of the training loss values of each
                epoch (list of floats).
            train_acc: All of the training accuracy values of each
                epoch (list of floats).
            val_loss: All of the validation loss values of each
                epoch (list of floats).
            val_acc: All of the validation accuracy values of each
                epoch (list of floats).
        """

        if self.first_redraw and self.show_plot_window:
            plt.figure(self.fig.number)
            plt.show(block=False)
            #plt.draw()

        ax1 = self.ax_loss
        ax2 = self.ax_acc

        # List of each epoch (x-axis)
        epochs = list(range(0, epoch+1))

        # Clear loss and accuracy charts
        if ax1:
            ax1.clear()
        if ax2:
            ax2.clear()

        # Set titles of charts (at the top)
        if ax1:
            ax1.set_title("loss")
        if ax2:
            ax2.set_title("accuracy")

        # Set the styles of the lines used in the charts
        # Different line style for epochs after the  first one, because
        # the very first epoch has only one data point and therefore no line
        # and would be invisible without the changed style.
        ls_loss_train = self.linestyles["loss_train"]
        ls_loss_val = self.linestyles["loss_val"]
        ls_acc_train = self.linestyles["acc_train"]
        ls_acc_val = self.linestyles["acc_val"]
        if len(self.values_loss_train_x) == 1:
            ls_loss_train = self.linestyles_one_value["loss_train"]
        if len(self.values_loss_val_x) == 1:
            ls_loss_val = self.linestyles_one_value["loss_val"]
        if len(self.values_acc_train_x) == 1:
            ls_acc_train = self.linestyles_one_value["acc_train"]
        if len(self.values_acc_val_x) == 1:
            ls_acc_val = self.linestyles_one_value["acc_val"]

        # Plot the lines
        if ax1:
            ax1.plot(self.values_loss_train_x, self.values_loss_train_y,
                     ls_loss_train, label="loss train")
            ax1.plot(self.values_loss_val_x, self.values_loss_val_y,
                     ls_loss_val, label="loss val.")
        if ax2:
            ax2.plot(self.values_acc_train_x, self.values_acc_train_y,
                     ls_acc_train, label="acc. train")
            ax2.plot(self.values_acc_val_x, self.values_acc_val_y,
                     ls_acc_val, label="acc. val.")

        if self.show_regressions:
            # Compute the regression lines for the n_forward future epochs.
            # n_forward is calculated relative to the current epoch
            # (e.g. at epoch 100 compute 10 next, at 200 the 20 next ones...).
            n_forward = int(max((epoch+1)*self.poly_forward_perc,
                                self.poly_forward_min))

            # Compute regression lines based on n_backwards epochs
            # in the past, e.g. based on the last 10 values.
            # n_backwards is calculated relative to the current epoch
            # (e.g. at epoch 100 compute based on the last 10 values,
            # at 200 based on the last 20 values...).
            n_backwards = int(max((epoch+1)*self.poly_backward_perc,
                                  self.poly_backward_min))

            # List of epochs for which to estimate/predict the likely value.
            # (epoch..epoch+n_forward instead of epoch+1..epoch+n_forward
            # so that the regression line is better connected to the line its
            # based on (no obvious gap).)
            future_epochs = [i for i in range(epoch, epoch + n_forward)]

            self.plot_regression_line(ax1, train_loss, epochs, future_epochs,
                                      n_backwards, linestyles[2],
                                      'train loss regression')
            self.plot_regression_line(ax1, val_loss, epochs, future_epochs,
                                      n_backwards, linestyles[3],
                                      'val loss regression')
            self.plot_regression_line(ax2, train_acc, epochs, future_epochs,
                                      n_backwards, linestyles[2],
                                      'train acc regression')
            self.plot_regression_line(ax2, val_acc, epochs, future_epochs,
                                      n_backwards, linestyles[3],
                                      'val acc regression')

        # Add legend (below chart)
        if ax1:
            ax1.legend(["loss train", "loss val.",
                        "loss train regression", "loss val. regression"],
                       bbox_to_anchor=(0.7, -0.08), ncol=2)
        if ax2:
            ax2.legend(["acc. train", "acc. val.",
                        "acc. train regression", "acc. val. regression"],
                       bbox_to_anchor=(0.7, -0.08), ncol=2)

        # Labels for x and y axis
        if ax1:
            ax1.set_ylabel("loss")
            ax1.set_xlabel("epoch")
        if ax2:
            ax2.set_ylabel("accuracy")
            ax2.set_xlabel("epoch")

        # Show a grid in both charts
        if ax1:
            ax1.grid(True)
        if ax2:
            ax2.grid(True)

    def plot_regression_line(self, plot_ax, data, epochs, future_epochs,
                             n_backwards, linestyle, label):
        """Calculates and plots a regression line.
        Args:
            plot_ax: The ax on which to plot the line.
            data: The data used to perform the regression
                (e.g. training loss values).
            epochs: List of all epochs (0, 1, 2, ...).
            future_epochs: List of the future epochs for which values are
                ought to be predicted.
            n_backwards: How far back to go in time (in epochs) in order
                to compute the regression. (E.g. 10 = calculate it on the
                last 10 values max.)
            linestyle: Linestyle of the regression line.
            label: Label of the regression line.
        """
        # dont try to draw anything if the data list is empty or it's the
        # first epoch
        if len(data) > 1:
            poly = np.poly1d(np.polyfit(epochs[-n_backwards:],
                                        data[-n_backwards:], self.poly_degree))
            future_values = [poly(i) for i in future_epochs]
            plot_ax.plot(future_epochs, future_values, linestyle, label=label)
<|endoftext|>


<commit_before># encoding: utf-8
"""
A base class for objects that are configurable.

Inheritance diagram:

.. inheritance-diagram:: IPython.config.configurable
   :parts: 3

Authors:

* Brian Granger
* Fernando Perez
* Min RK
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import datetime
from copy import deepcopy

from loader import Config
from IPython.utils.traitlets import HasTraits, Instance
from IPython.utils.text import indent, wrap_paragraphs

#-----------------------------------------------------------------------------
# Helper classes for Configurables
#-----------------------------------------------------------------------------

class ConfigurableError(Exception):
    pass

class MultipleInstanceError(ConfigurableError):
    pass

#-----------------------------------------------------------------------------
# Configurable implementation
#-----------------------------------------------------------------------------

class Configurable(HasTraits):

    config = Instance(Config, (), {})
    parent = Instance('IPython.config.configurable.Configurable')
    created = None

    def __init__(self, **kwargs):
        """Create a configurable given a config config.

        Parameters
        ----------
        config : Config
            If this is empty, default values are used. If config is a
            :class:`Config` instance, it will be used to configure the
            instance.
        parent : Configurable instance
            The parent

        Notes
        -----
        Subclasses of Configurable must call the :meth:`__init__` method of
        :class:`Configurable` *before* doing anything else and using
        :func:`super`::

            class MyConfigurable(Configurable):
                def __init__(self, config=None):
                    super(MyConfigurable, self).__init__(config=config)
                    # Then any other code you need to finish initialization.

        This ensures that instances will be configured properly.
        """
        parent = kwargs.pop('parent', None)
        if parent:
            # config is implied from parent
            if 'config' not in kwargs:
                kwargs['config'] = parent.config
            self.parent = parent

        config = kwargs.pop('config', None)
        if config is not None:
            # We used to deepcopy, but for now we are trying to just save
            # by reference.  This *could* have side effects as all components
            # will share config. In fact, I did find such a side effect in
            # _config_changed below. If a config attribute value was a mutable type
            # all instances of a component were getting the same copy, effectively
            # making that a class attribute.
            # self.config = deepcopy(config)
            self.config = config
        # This should go second so individual keyword arguments override
        # the values in config.
        super(Configurable, self).__init__(**kwargs)
        self.created = datetime.datetime.now()

    #-------------------------------------------------------------------------
    # Static trait notifiations
    #-------------------------------------------------------------------------

    @classmethod
    def section_names(cls):
        """return section names as a list"""
        return  [c.__name__ for c in reversed(cls.__mro__) if
            issubclass(c, Configurable) and issubclass(cls, c)
        ]

    def _load_config(self, cfg, section_names=None, traits=None):
        """load traits from a Config object"""

        if traits is None:
            traits = self.traits(config=True)
        if section_names is None:
            section_names = self.section_names()

        for sname in section_names:
            # Don't do a blind getattr as that would cause the config to
            # dynamically create the section with name self.__class__.__name__.
            if cfg._has_section(sname):
                my_config = cfg[sname]
                for k, v in traits.iteritems():
                    try:
                        # Here we grab the value from the config
                        # If k has the naming convention of a config
                        # section, it will be auto created.
                        config_value = my_config[k]
                    except KeyError:
                        pass
                    else:
                        # We have to do a deepcopy here if we don't deepcopy the entire
                        # config object. If we don't, a mutable config_value will be
                        # shared by all instances, effectively making it a class attribute.
                        setattr(self, k, deepcopy(config_value))

    def _config_changed(self, name, old, new):
        """Update all the class traits having ``config=True`` as metadata.

        For any class trait with a ``config`` metadata attribute that is
        ``True``, we update the trait with the value of the corresponding
        config entry.
        """
        # Get all traits with a config metadata entry that is True
        traits = self.traits(config=True)

        # We auto-load config section for this class as well as any parent
        # classes that are Configurable subclasses.  This starts with Configurable
        # and works down the mro loading the config for each section.
        section_names = self.section_names()
        self._load_config(new, traits=traits, section_names=section_names)

        # load parent config as well, if we have one
        parent_section_names = [] if self.parent is None else self.parent.section_names()
        for parent in parent_section_names:
            print parent, new._has_section(parent), new[parent]
            if not new._has_section(parent):
                continue
            self._load_config(new[parent], traits=traits, section_names=section_names)

    def update_config(self, config):
        """Fire the traits events when the config is updated."""
        # Save a copy of the current config.
        newconfig = deepcopy(self.config)
        # Merge the new config into the current one.
        newconfig.merge(config)
        # Save the combined config as self.config, which triggers the traits
        # events.
        self.config = newconfig

    @classmethod
    def class_get_help(cls, inst=None):
        """Get the help string for this class in ReST format.

        If `inst` is given, it's current trait values will be used in place of
        class defaults.
        """
        assert inst is None or isinstance(inst, cls)
        final_help = []
        final_help.append(u'%s options' % cls.__name__)
        final_help.append(len(final_help[0])*u'-')
        for k, v in sorted(cls.class_traits(config=True).iteritems()):
            help = cls.class_get_trait_help(v, inst)
            final_help.append(help)
        return '\n'.join(final_help)

    @classmethod
    def class_get_trait_help(cls, trait, inst=None):
        """Get the help string for a single trait.

        If `inst` is given, it's current trait values will be used in place of
        the class default.
        """
        assert inst is None or isinstance(inst, cls)
        lines = []
        header = "--%s.%s=<%s>" % (cls.__name__, trait.name, trait.__class__.__name__)
        lines.append(header)
        if inst is not None:
            lines.append(indent('Current: %r' % getattr(inst, trait.name), 4))
        else:
            try:
                dvr = repr(trait.get_default_value())
            except Exception:
                dvr = None # ignore defaults we can't construct
            if dvr is not None:
                if len(dvr) > 64:
                    dvr = dvr[:61]+'...'
                lines.append(indent('Default: %s' % dvr, 4))
        if 'Enum' in trait.__class__.__name__:
            # include Enum choices
            lines.append(indent('Choices: %r' % (trait.values,)))

        help = trait.get_metadata('help')
        if help is not None:
            help = '\n'.join(wrap_paragraphs(help, 76))
            lines.append(indent(help, 4))
        return '\n'.join(lines)

    @classmethod
    def class_print_help(cls, inst=None):
        """Get the help string for a single trait and print it."""
        print cls.class_get_help(inst)

    @classmethod
    def class_config_section(cls):
        """Get the config class config section"""
        def c(s):
            """return a commented, wrapped block."""
            s = '\n\n'.join(wrap_paragraphs(s, 78))

            return '# ' + s.replace('\n', '\n# ')

        # section header
        breaker = '#' + '-'*78
        s = "# %s configuration" % cls.__name__
        lines = [breaker, s, breaker, '']
        # get the description trait
        desc = cls.class_traits().get('description')
        if desc:
            desc = desc.default_value
        else:
            # no description trait, use __doc__
            desc = getattr(cls, '__doc__', '')
        if desc:
            lines.append(c(desc))
            lines.append('')

        parents = []
        for parent in cls.mro():
            # only include parents that are not base classes
            # and are not the class itself
            # and have some configurable traits to inherit
            if parent is not cls and issubclass(parent, Configurable) and \
                    parent.class_traits(config=True):
                parents.append(parent)

        if parents:
            pstr = ', '.join([ p.__name__ for p in parents ])
            lines.append(c('%s will inherit config from: %s'%(cls.__name__, pstr)))
            lines.append('')

        for name, trait in cls.class_traits(config=True).iteritems():
            help = trait.get_metadata('help') or ''
            lines.append(c(help))
            lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))
            lines.append('')
        return '\n'.join(lines)

class SingletonConfigurable(Configurable):
    """A configurable that only allows one instance.

    This class is for classes that should only have one instance of itself
    or *any* subclass. To create and retrieve such a class use the
    :meth:`SingletonConfigurable.instance` method.
    """

    _instance = None

    @classmethod
    def _walk_mro(cls):
        """Walk the cls.mro() for parent classes that are also singletons

        For use in instance()
        """

        for subclass in cls.mro():
            if issubclass(cls, subclass) and \
                    issubclass(subclass, SingletonConfigurable) and \
                    subclass != SingletonConfigurable:
                yield subclass

    @classmethod
    def clear_instance(cls):
        """unset _instance for this class and singleton parents.
        """
        if not cls.initialized():
            return
        for subclass in cls._walk_mro():
            if isinstance(subclass._instance, cls):
                # only clear instances that are instances
                # of the calling class
                subclass._instance = None

    @classmethod
    def instance(cls, *args, **kwargs):
        """Returns a global instance of this class.

        This method create a new instance if none have previously been created
        and returns a previously created instance is one already exists.

        The arguments and keyword arguments passed to this method are passed
        on to the :meth:`__init__` method of the class upon instantiation.

        Examples
        --------

        Create a singleton class using instance, and retrieve it::

            >>> from IPython.config.configurable import SingletonConfigurable
            >>> class Foo(SingletonConfigurable): pass
            >>> foo = Foo.instance()
            >>> foo == Foo.instance()
            True

        Create a subclass that is retrived using the base class instance::

            >>> class Bar(SingletonConfigurable): pass
            >>> class Bam(Bar): pass
            >>> bam = Bam.instance()
            >>> bam == Bar.instance()
            True
        """
        # Create and save the instance
        if cls._instance is None:
            inst = cls(*args, **kwargs)
            # Now make sure that the instance will also be returned by
            # parent classes' _instance attribute.
            for subclass in cls._walk_mro():
                subclass._instance = inst

        if isinstance(cls._instance, cls):
            return cls._instance
        else:
            raise MultipleInstanceError(
                'Multiple incompatible subclass instances of '
                '%s are being created.' % cls.__name__
            )

    @classmethod
    def initialized(cls):
        """Has an instance been created?"""
        return hasattr(cls, "_instance") and cls._instance is not None

class LoggingConfigurable(Configurable):
    """A parent class for Configurables that log.

    Subclasses have a log trait, and the default behavior
    is to get the logger from the currently running Application
    via Application.instance().log.
    """

    log = Instance('logging.Logger')
    def _log_default(self):
        from IPython.config.application import Application
        return Application.instance().log

<commit_msg>override null config with parent's config<commit_after># encoding: utf-8
"""
A base class for objects that are configurable.

Inheritance diagram:

.. inheritance-diagram:: IPython.config.configurable
   :parts: 3

Authors:

* Brian Granger
* Fernando Perez
* Min RK
"""

#-----------------------------------------------------------------------------
#  Copyright (C) 2008-2011  The IPython Development Team
#
#  Distributed under the terms of the BSD License.  The full license is in
#  the file COPYING, distributed as part of this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

import datetime
from copy import deepcopy

from loader import Config
from IPython.utils.traitlets import HasTraits, Instance
from IPython.utils.text import indent, wrap_paragraphs

#-----------------------------------------------------------------------------
# Helper classes for Configurables
#-----------------------------------------------------------------------------

class ConfigurableError(Exception):
    pass

class MultipleInstanceError(ConfigurableError):
    pass

#-----------------------------------------------------------------------------
# Configurable implementation
#-----------------------------------------------------------------------------

class Configurable(HasTraits):

    config = Instance(Config, (), {})
    parent = Instance('IPython.config.configurable.Configurable')
    created = None

    def __init__(self, **kwargs):
        """Create a configurable given a config config.

        Parameters
        ----------
        config : Config
            If this is empty, default values are used. If config is a
            :class:`Config` instance, it will be used to configure the
            instance.
        parent : Configurable instance
            The parent

        Notes
        -----
        Subclasses of Configurable must call the :meth:`__init__` method of
        :class:`Configurable` *before* doing anything else and using
        :func:`super`::

            class MyConfigurable(Configurable):
                def __init__(self, config=None):
                    super(MyConfigurable, self).__init__(config=config)
                    # Then any other code you need to finish initialization.

        This ensures that instances will be configured properly.
        """
        parent = kwargs.pop('parent', None)
        if parent:
            # config is implied from parent
            if kwargs.get('config', None) is None:
                kwargs['config'] = parent.config
            self.parent = parent

        config = kwargs.pop('config', None)
        if config is not None:
            # We used to deepcopy, but for now we are trying to just save
            # by reference.  This *could* have side effects as all components
            # will share config. In fact, I did find such a side effect in
            # _config_changed below. If a config attribute value was a mutable type
            # all instances of a component were getting the same copy, effectively
            # making that a class attribute.
            # self.config = deepcopy(config)
            self.config = config
        # This should go second so individual keyword arguments override
        # the values in config.
        super(Configurable, self).__init__(**kwargs)
        self.created = datetime.datetime.now()

    #-------------------------------------------------------------------------
    # Static trait notifiations
    #-------------------------------------------------------------------------

    @classmethod
    def section_names(cls):
        """return section names as a list"""
        return  [c.__name__ for c in reversed(cls.__mro__) if
            issubclass(c, Configurable) and issubclass(cls, c)
        ]

    def _load_config(self, cfg, section_names=None, traits=None):
        """load traits from a Config object"""

        if traits is None:
            traits = self.traits(config=True)
        if section_names is None:
            section_names = self.section_names()

        for sname in section_names:
            # Don't do a blind getattr as that would cause the config to
            # dynamically create the section with name self.__class__.__name__.
            if cfg._has_section(sname):
                my_config = cfg[sname]
                for k, v in traits.iteritems():
                    try:
                        # Here we grab the value from the config
                        # If k has the naming convention of a config
                        # section, it will be auto created.
                        config_value = my_config[k]
                    except KeyError:
                        pass
                    else:
                        # We have to do a deepcopy here if we don't deepcopy the entire
                        # config object. If we don't, a mutable config_value will be
                        # shared by all instances, effectively making it a class attribute.
                        setattr(self, k, deepcopy(config_value))

    def _config_changed(self, name, old, new):
        """Update all the class traits having ``config=True`` as metadata.

        For any class trait with a ``config`` metadata attribute that is
        ``True``, we update the trait with the value of the corresponding
        config entry.
        """
        # Get all traits with a config metadata entry that is True
        traits = self.traits(config=True)

        # We auto-load config section for this class as well as any parent
        # classes that are Configurable subclasses.  This starts with Configurable
        # and works down the mro loading the config for each section.
        section_names = self.section_names()
        self._load_config(new, traits=traits, section_names=section_names)

        # load parent config as well, if we have one
        parent_section_names = [] if self.parent is None else self.parent.section_names()
        for parent in parent_section_names:
            if not new._has_section(parent):
                continue
            self._load_config(new[parent], traits=traits, section_names=section_names)

    def update_config(self, config):
        """Fire the traits events when the config is updated."""
        # Save a copy of the current config.
        newconfig = deepcopy(self.config)
        # Merge the new config into the current one.
        newconfig.merge(config)
        # Save the combined config as self.config, which triggers the traits
        # events.
        self.config = newconfig

    @classmethod
    def class_get_help(cls, inst=None):
        """Get the help string for this class in ReST format.

        If `inst` is given, it's current trait values will be used in place of
        class defaults.
        """
        assert inst is None or isinstance(inst, cls)
        final_help = []
        final_help.append(u'%s options' % cls.__name__)
        final_help.append(len(final_help[0])*u'-')
        for k, v in sorted(cls.class_traits(config=True).iteritems()):
            help = cls.class_get_trait_help(v, inst)
            final_help.append(help)
        return '\n'.join(final_help)

    @classmethod
    def class_get_trait_help(cls, trait, inst=None):
        """Get the help string for a single trait.

        If `inst` is given, it's current trait values will be used in place of
        the class default.
        """
        assert inst is None or isinstance(inst, cls)
        lines = []
        header = "--%s.%s=<%s>" % (cls.__name__, trait.name, trait.__class__.__name__)
        lines.append(header)
        if inst is not None:
            lines.append(indent('Current: %r' % getattr(inst, trait.name), 4))
        else:
            try:
                dvr = repr(trait.get_default_value())
            except Exception:
                dvr = None # ignore defaults we can't construct
            if dvr is not None:
                if len(dvr) > 64:
                    dvr = dvr[:61]+'...'
                lines.append(indent('Default: %s' % dvr, 4))
        if 'Enum' in trait.__class__.__name__:
            # include Enum choices
            lines.append(indent('Choices: %r' % (trait.values,)))

        help = trait.get_metadata('help')
        if help is not None:
            help = '\n'.join(wrap_paragraphs(help, 76))
            lines.append(indent(help, 4))
        return '\n'.join(lines)

    @classmethod
    def class_print_help(cls, inst=None):
        """Get the help string for a single trait and print it."""
        print cls.class_get_help(inst)

    @classmethod
    def class_config_section(cls):
        """Get the config class config section"""
        def c(s):
            """return a commented, wrapped block."""
            s = '\n\n'.join(wrap_paragraphs(s, 78))

            return '# ' + s.replace('\n', '\n# ')

        # section header
        breaker = '#' + '-'*78
        s = "# %s configuration" % cls.__name__
        lines = [breaker, s, breaker, '']
        # get the description trait
        desc = cls.class_traits().get('description')
        if desc:
            desc = desc.default_value
        else:
            # no description trait, use __doc__
            desc = getattr(cls, '__doc__', '')
        if desc:
            lines.append(c(desc))
            lines.append('')

        parents = []
        for parent in cls.mro():
            # only include parents that are not base classes
            # and are not the class itself
            # and have some configurable traits to inherit
            if parent is not cls and issubclass(parent, Configurable) and \
                    parent.class_traits(config=True):
                parents.append(parent)

        if parents:
            pstr = ', '.join([ p.__name__ for p in parents ])
            lines.append(c('%s will inherit config from: %s'%(cls.__name__, pstr)))
            lines.append('')

        for name, trait in cls.class_traits(config=True).iteritems():
            help = trait.get_metadata('help') or ''
            lines.append(c(help))
            lines.append('# c.%s.%s = %r'%(cls.__name__, name, trait.get_default_value()))
            lines.append('')
        return '\n'.join(lines)

class SingletonConfigurable(Configurable):
    """A configurable that only allows one instance.

    This class is for classes that should only have one instance of itself
    or *any* subclass. To create and retrieve such a class use the
    :meth:`SingletonConfigurable.instance` method.
    """

    _instance = None

    @classmethod
    def _walk_mro(cls):
        """Walk the cls.mro() for parent classes that are also singletons

        For use in instance()
        """

        for subclass in cls.mro():
            if issubclass(cls, subclass) and \
                    issubclass(subclass, SingletonConfigurable) and \
                    subclass != SingletonConfigurable:
                yield subclass

    @classmethod
    def clear_instance(cls):
        """unset _instance for this class and singleton parents.
        """
        if not cls.initialized():
            return
        for subclass in cls._walk_mro():
            if isinstance(subclass._instance, cls):
                # only clear instances that are instances
                # of the calling class
                subclass._instance = None

    @classmethod
    def instance(cls, *args, **kwargs):
        """Returns a global instance of this class.

        This method create a new instance if none have previously been created
        and returns a previously created instance is one already exists.

        The arguments and keyword arguments passed to this method are passed
        on to the :meth:`__init__` method of the class upon instantiation.

        Examples
        --------

        Create a singleton class using instance, and retrieve it::

            >>> from IPython.config.configurable import SingletonConfigurable
            >>> class Foo(SingletonConfigurable): pass
            >>> foo = Foo.instance()
            >>> foo == Foo.instance()
            True

        Create a subclass that is retrived using the base class instance::

            >>> class Bar(SingletonConfigurable): pass
            >>> class Bam(Bar): pass
            >>> bam = Bam.instance()
            >>> bam == Bar.instance()
            True
        """
        # Create and save the instance
        if cls._instance is None:
            inst = cls(*args, **kwargs)
            # Now make sure that the instance will also be returned by
            # parent classes' _instance attribute.
            for subclass in cls._walk_mro():
                subclass._instance = inst

        if isinstance(cls._instance, cls):
            return cls._instance
        else:
            raise MultipleInstanceError(
                'Multiple incompatible subclass instances of '
                '%s are being created.' % cls.__name__
            )

    @classmethod
    def initialized(cls):
        """Has an instance been created?"""
        return hasattr(cls, "_instance") and cls._instance is not None

class LoggingConfigurable(Configurable):
    """A parent class for Configurables that log.

    Subclasses have a log trait, and the default behavior
    is to get the logger from the currently running Application
    via Application.instance().log.
    """

    log = Instance('logging.Logger')
    def _log_default(self):
        from IPython.config.application import Application
        return Application.instance().log

<|endoftext|>