<commit_before># -*- coding: utf-8 -*-

import os
import re

from flask import Flask, g, request_started, request_finished
from flask_restplus import abort
from flask_uploads import configure_uploads

from APITaxi_models import db, security, HailLog
from APITaxi_utils.login_manager import init_app as init_login_manager
from APITaxi_utils.version import check_version, add_version_header

from . import api, tasks
from .api.extensions import documents
from .commands.warm_up_redis import warm_up_redis_func
from .extensions import redis_store, redis_store_saved, user_datastore
from APITaxi2 import create_app as create_new_app

__author__ = 'Vincent Lara'
__contact__ = "vincent.lara@data.gouv.fr"
__homepage__ = "https://github.com/"
__version__ = '0.1.0'
__doc__ = 'Flask application to serve APITaxi'

def load_configuration(app):
    """Load application configuration:

    - from default_settings.py
    - from the settings file set in the environment variable
      APITAXI_CONFIG_FILE
    - from variables set in environment
    """
    app.config.from_object('APITaxi.default_settings')

    if 'APITAXI_CONFIG_FILE' in os.environ:
        app.config.from_envvar('APITAXI_CONFIG_FILE')

    if 'ENV' not in app.config:
        raise ValueError('Configuration variable ENV is required')

    valid_env = ('PROD', 'STAGING', 'DEV')
    if app.config['ENV'] not in valid_env:
        raise ValueError('ENV {} invalid, must be any of {}'.format(
            app.config['ENV'], valid_env
        ))

    for param in app.config:
        if param not in os.environ:
            continue
        app.config[param] = os.environ[param]

    app.url_map.strict_slashes = False

def print_url_map(url_map):
    for rule in sorted(url_map.iter_rules(), key=lambda r: r.rule):
        methods = [m for m in rule.methods if m not in('OPTIONS', 'HEAD')]
        print(('\t%-45s -> %s' % (rule.rule, ', '.join(methods))))

def unauthorized():
    """By default, @login_required returns HTTP/301 HTML response to redirect
    the browser to the login form. Since we are in the context of an API,
    clients are expecting a JSON response.
    """
    abort(401, error='You are not logged in. Please provide a valid X-Api-Key header.')

def create_legacy_app():
    app = Flask(__name__)

    load_configuration(app)

    db.init_app(app)
    redis_store.init_app(app)
    redis_store_saved.init_app(app)
    api.init_app(app)

    request_started.connect(check_version, app)
    request_finished.connect(add_version_header, app)

    configure_uploads(app, (documents,))

    init_login_manager(app, user_datastore, None)
    app.login_manager.unauthorized_handler(unauthorized)

    tasks.init_app(app)

    user_datastore.init_app(db, security.User, security.Role)

    @app.before_first_request
    def warm_up_redis():
        warm_up_redis_func(app, db, security.User, redis_store)

    def delete_redis_keys(response):
        if not hasattr(g, 'keys_to_delete'):
            return response
        redis_store.delete(*g.keys_to_delete)
        return response

    app.after_request_funcs.setdefault(None, []).append(
        HailLog.after_request(redis_store_saved)
    )
    app.after_request_funcs.setdefault(None, []).append(
        delete_redis_keys
    )

    # Only display url_map if debug and from the worker thread.
    if app.debug and os.environ.get('WERKZEUG_RUN_MAIN'):
        print_url_map(app.url_map)

    return app

class RegexpDispatcherMiddleware:
    """ Forwards requests to applications depending on method and path.
    """
    def __init__(self, app, overrides=None):
        self.app = app
        self.overrides = overrides or {}

    def __call__(self, environ, start_response):
        for override in self.overrides.values():
            if (
                re.match(override['regexp'], environ['PATH_INFO'])
                and ('methods' not in override
                     or environ['REQUEST_METHOD'] in override['methods'])
            ):
                return override['app'](environ, start_response)
        return self.app(environ, start_response)

def create_app(proxy_v2=True):
    """Forward all requests to legacy application, except for routes that have
    been updated on the new API."""
    legacy_app = create_legacy_app()

    if proxy_v2:
        new_app = create_new_app()
        legacy_app.wsgi_app = RegexpDispatcherMiddleware(legacy_app.wsgi_app, {
            'Customers': {
                'regexp': r'^/customers/.*$',
                'app': new_app,
            },
            'Users': {
                'regexp': r'^/users(/.*)?$',
                'app': new_app,
            },
            'Drivers': {
                'regexp': r'^/drivers(/.*)?$',
                'app': new_app,
            },
            'ADS': {
                'regexp': r'^/ads(/.*)?$',
                'app': new_app,
            },
            'GET and PUT /taxis/:id': {
                'regexp': r'^/taxis/.+$',
                'app': new_app,
                'methods': ['GET', 'PUT']
            },
            'GET and POST /taxis': {
                'regexp': r'^/taxis/?$',
                'app': new_app,
                'methods': ['GET', 'POST']
            },
            'POST /vehicles': {
                'regexp': r'^/vehicles/?$',
                'app': new_app,
                'methods': ['POST']
            },
            'GET /hails(/:id)?': {
                'regexp': r'^/hails(/.*)?$',
                'app': new_app,
                'methods': ['GET', 'PUT']
            },
        })

    return legacy_app
<commit_msg>Improve routing to APITaxi2<commit_after># -*- coding: utf-8 -*-

import os
import re

from flask import Flask, g, request_started, request_finished
from flask_restplus import abort
from flask_uploads import configure_uploads

from APITaxi_models import db, security, HailLog
from APITaxi_utils.login_manager import init_app as init_login_manager
from APITaxi_utils.version import check_version, add_version_header

from . import api, tasks
from .api.extensions import documents
from .commands.warm_up_redis import warm_up_redis_func
from .extensions import redis_store, redis_store_saved, user_datastore
from APITaxi2 import create_app as create_new_app

__author__ = 'Vincent Lara'
__contact__ = "vincent.lara@data.gouv.fr"
__homepage__ = "https://github.com/"
__version__ = '0.1.0'
__doc__ = 'Flask application to serve APITaxi'

def load_configuration(app):
    """Load application configuration:

    - from default_settings.py
    - from the settings file set in the environment variable
      APITAXI_CONFIG_FILE
    - from variables set in environment
    """
    app.config.from_object('APITaxi.default_settings')

    if 'APITAXI_CONFIG_FILE' in os.environ:
        app.config.from_envvar('APITAXI_CONFIG_FILE')

    if 'ENV' not in app.config:
        raise ValueError('Configuration variable ENV is required')

    valid_env = ('PROD', 'STAGING', 'DEV')
    if app.config['ENV'] not in valid_env:
        raise ValueError('ENV {} invalid, must be any of {}'.format(
            app.config['ENV'], valid_env
        ))

    for param in app.config:
        if param not in os.environ:
            continue
        app.config[param] = os.environ[param]

    app.url_map.strict_slashes = False

def print_url_map(url_map):
    for rule in sorted(url_map.iter_rules(), key=lambda r: r.rule):
        methods = [m for m in rule.methods if m not in('OPTIONS', 'HEAD')]
        print(('\t%-45s -> %s' % (rule.rule, ', '.join(methods))))

def unauthorized():
    """By default, @login_required returns HTTP/301 HTML response to redirect
    the browser to the login form. Since we are in the context of an API,
    clients are expecting a JSON response.
    """
    abort(401, error='You are not logged in. Please provide a valid X-Api-Key header.')

def create_legacy_app():
    app = Flask(__name__)

    load_configuration(app)

    db.init_app(app)
    redis_store.init_app(app)
    redis_store_saved.init_app(app)
    api.init_app(app)

    request_started.connect(check_version, app)
    request_finished.connect(add_version_header, app)

    configure_uploads(app, (documents,))

    init_login_manager(app, user_datastore, None)
    app.login_manager.unauthorized_handler(unauthorized)

    tasks.init_app(app)

    user_datastore.init_app(db, security.User, security.Role)

    @app.before_first_request
    def warm_up_redis():
        warm_up_redis_func(app, db, security.User, redis_store)

    def delete_redis_keys(response):
        if not hasattr(g, 'keys_to_delete'):
            return response
        redis_store.delete(*g.keys_to_delete)
        return response

    app.after_request_funcs.setdefault(None, []).append(
        HailLog.after_request(redis_store_saved)
    )
    app.after_request_funcs.setdefault(None, []).append(
        delete_redis_keys
    )

    # Only display url_map if debug and from the worker thread.
    if app.debug and os.environ.get('WERKZEUG_RUN_MAIN'):
        print_url_map(app.url_map)

    return app

class RegexpDispatcherMiddleware:
    """ Forwards requests to applications depending on method and path.
    """
    def __init__(self, app, overrides=None):
        self.app = app
        self.overrides = overrides or {}

    def __call__(self, environ, start_response):
        for override in self.overrides.values():
            if (
                re.match(override['regexp'], environ['PATH_INFO'])
                and ('methods' not in override
                     or environ['REQUEST_METHOD'] in override['methods'])
            ):
                return override['app'](environ, start_response)
        return self.app(environ, start_response)

def create_app(proxy_v2=True):
    """Forward all requests to legacy application, except for routes that have
    been updated on the new API."""
    legacy_app = create_legacy_app()

    if proxy_v2:
        new_app = create_new_app()
        legacy_app.wsgi_app = RegexpDispatcherMiddleware(legacy_app.wsgi_app, {
            'Customers': {
                'regexp': r'^/customers(/.*)?$',
                'app': new_app,
            },
            'Users': {
                'regexp': r'^/users(/.*)?$',
                'app': new_app,
            },
            'Drivers': {
                'regexp': r'^/drivers(/.*)?$',
                'app': new_app,
            },
            'ADS': {
                'regexp': r'^/ads(/.*)?$',
                'app': new_app,
            },
            'Taxis': {
                'regexp': r'^/taxis(/.*)?$',
                'app': new_app,
            },
            'Vehicles': {
                'regexp': r'^/vehicles(/.*)?$',
                'app': new_app,
            },
            'GET and PUT /hails(/:id)?': {
                'regexp': r'^/hails(/.*)?$',
                'app': new_app,
                'methods': ['OPTIONS', 'GET', 'PUT']
            },
        })

    return legacy_app
<|endoftext|>


<commit_before>import os
import logging
import cliff.command
import configparser
import json
import process_config

class SysConfig(cliff.command.Command):
    "This command will setup the pancancer system configuration."
    log = logging.getLogger(__name__)

    def get_parser(self,prog_name):
        parser = super(SysConfig,self).get_parser(prog_name)
        parser.add_argument('--config',dest='config_path',help='The path to your pancancer config file, if it already exists.',required=False)
        parser.add_argument("--force",dest='force_config',required=False, action='store_true')
        # Need to have a "force" option
        return parser

    def _get_config_value(self,user_value,prev_user_value,message, alt_condition=None):
        "This function will prompt the use with a question and will keep prompting them until a valid value is given."
        while (alt_condition == None and user_value.strip() == '') or (alt_condition != None and alt_condition(user_value)):
            prev_value_prompt = ''
            if prev_user_value != '':
                prev_value_prompt = '[Press \"ENTER\" to use the previous value: '+str(prev_user_value)+', or type a new value if you want]'
            user_value = input(message + ' ' + prev_value_prompt+'? ')
            user_value = str(user_value).strip() or str(prev_user_value).strip()
            if user_value=='':
                self.log.info('This value cannot be blank.')

        return user_value

    def take_action(self, parsed_args):
        self.log.info('Setting up pancancer config files.')
        config_path=vars(parsed_args)['config_path']
        force_config=vars(parsed_args)['force_config']
        self.log.debug('force_config:' + str(force_config))
        if config_path != None:
            # if the user specifies a path to an existing file, they can use that instead of the interactive process.
            self.log.debug('path to config file: ' + config_path)
            process_config.main(config_path)
        else:
            # if the user did not specify a config, ask them questions and then WRITE/update a config!
            pancancer_config_path=os.path.expanduser('~/.pancancer/simple_pancancer_config.json')
            config_data={}
            if os.path.isfile(pancancer_config_path):
                #Read the pancancer_config.json file if it exists
                with open(pancancer_config_path,'r') as pancancer_config_file:
                    config_data = json.load(pancancer_config_file)

            pancancer_config={}

            aws_config_path = os.path.expanduser('~/.aws/config')
            aws_key=''
            aws_secret_key=''
            prev_aws_key=''
            prev_aws_secret_key=''
            if os.path.isfile(aws_config_path):
                # read the file, get the values
                aws_config = configparser.ConfigParser()
                aws_config.read(aws_config_path)
                prev_aws_key = aws_config['default']['aws_access_key_id']
                prev_aws_secret_key = aws_config['default']['aws_secret_access_key']

            # AWS Key and Secret Key should only be asked for if for some reason they are NOT available (they *should* be, but stuff happens...)
            if force_config  or prev_aws_key.strip() == '' or prev_aws_secret_key.strip() == '':
                aws_key = self._get_config_value(aws_key,prev_aws_key,'What is your AWS Key')
                aws_secret_key = self._get_config_value(aws_secret_key,prev_aws_secret_key,'What is your AWS Secret Key')
                # If the AWS keys weren't available, ~/.aws/config is either not present or missing values, so re-write it.
                aws_config = configparser.ConfigParser()
                aws_config['default'] = {'aws_access_key_id':aws_key, 'aws_secret_access_key':aws_secret_key}
                with open(aws_config_path,'w') as aws_configfile:
                    aws_config.write(aws_configfile,space_around_delimiters=False)
            else:
                aws_key = prev_aws_key
                aws_secret_key = prev_aws_secret_key

            # also: get fleet name, PEM key path, and key name from env. If everything went smoothly, these values should be in
            # /opt/from_host/config/pancancer.config
            # Actually, fleet name should be coming from process_config.py, not here, since it's passed into the container as an environment variable.
            # If it's NOT there, we should askt he user for these values.
            bootstrap_config_path='/opt/from_host/config/pancancer.config'
            pem_key_path=''
            prev_pem_key_path=''
            key_name=''
            prev_key_name=''
            prev_security_group = ''
            security_group = ''
            prev_fleet_size = ''
            fleet_size=''
            if os.path.isfile(bootstrap_config_path):
                self.log.debug('bootstrap config file exists at: '+bootstrap_config_path)
                # load simple config file (it should have been generated by install_bootrap) and get values.
                H = dict(line.strip().split('=') for line in open(bootstrap_config_path))
                if 'PEM_PATH' in H:
                    prev_pem_key_path = H['PEM_PATH']
                if 'KEY_NAME' in H:
                    prev_key_name = H['KEY_NAME']

                # Fleet size could be defined in the bootstrap config file which should come from the host machine, or it could be defined in the pancancer config file.
                # If there's nothing in pancancer config, we'll check in the bootstrap config, and if nothing there, default to 1
                if 'max_fleet_size' in config_data:
                    prev_fleet_size = config_data['max_fleet_size']
                else:
                    if 'FLEET_SIZE' in H:
                        prev_fleet_size = H['FLEET_SIZE']
                if str(prev_fleet_size).strip() == '':
                    prev_fleet_size = 1
                if force_config:
                    fleet_size = self._get_config_value(fleet_size, prev_fleet_size, 'How many VMs do you want in your fleet', alt_condition = lambda x: x.strip()=='' or x.isdigit() == False or int(x)<=0 )
                else:
                    fleet_size = prev_fleet_size

                if 'security_group' in config_data:
                    prev_security_group = config_data['security_group']
                else:
                    if 'SECURITY_GROUP' in H:
                        prev_security_group = H['SECURITY_GROUP']
                if prev_security_group.strip() == '':
                    prev_security_group = 'default'
                if force_config:
                    security_group = self._get_config_value(security_group, prev_security_group,'What AWS Security Group should the VMs belong to')
                else:
                    security_group = prev_security_group

                if prev_pem_key_path.strip() != '':
                    pem_key_path = prev_pem_key_path
                if prev_key_name.strip() != '':
                    key_name = prev_key_name

            else:
                self.log.debug('bootstrap config file cannot be found at: '+bootstrap_config_path+' so a new one will be written.')

            # Only ask the use about these if one or the other does not exist (indicates there might be a config value/file problem somewhere)
            if force_config or prev_pem_key_path.strip()=='' or prev_key_name.strip()=='' :
                pem_key_path = self._get_config_value(pem_key_path, prev_pem_key_path, 'What is the path to the AWS pem key file that your new VMs to use')
                key_name = self._get_config_value(key_name, prev_key_name, 'What is the Name of this key in AWS')
                #Now write the bootstrap config file, but only if we had to ask the user for help filling in the details.
                with open(bootstrap_config_path,'w') as bootstrap_config:
                    bootstrap_config.write('PEM_PATH='+pem_key_path+'\n')
                    bootstrap_config.write('KEY_NAME='+key_name+'\n')
                    bootstrap_config.write('FLEET_NAME='+os.environ['FLEET_NAME']+'\n')

            # Now, write the simple JSON config that will be used for the rest of pancancer system.
            pancancer_config = { 'max_fleet_size':fleet_size, 'path_to_key': pem_key_path,
                                'name_of_key':key_name, 'aws_secret_key':aws_secret_key,
                                'security_group': security_group, 'aws_key':aws_key }

            if not os.path.exists(os.path.expanduser('~/.pancancer')):
                os.makedirs(os.path.expanduser('~/.pancancer'))

            with open(pancancer_config_path,'w') as simple_pancancer_config:
                simple_pancancer_config.write(str(json.dumps(pancancer_config,sort_keys=True, indent=4) ))
                self.log.info('Your new pancancer system config file has been written to '+pancancer_config_path)
                self.log.info('The next time you want to run \"pancancer sysconfig\", you can use this file like this: ')
                self.log.info('pancancer sysconfig --config '+pancancer_config_path)

            # Now that a config is written, USE IT!!
            process_config.main(pancancer_config_path)
<commit_msg>copy aws config to .gnos<commit_after>import os
import logging
import cliff.command
import configparser
import json
import process_config
import shutil

class SysConfig(cliff.command.Command):
    "This command will setup the pancancer system configuration."
    log = logging.getLogger(__name__)

    def get_parser(self,prog_name):
        parser = super(SysConfig,self).get_parser(prog_name)
        parser.add_argument('--config',dest='config_path',help='The path to your pancancer config file, if it already exists.',required=False)
        parser.add_argument("--force",dest='force_config',required=False, action='store_true')
        # Need to have a "force" option
        return parser

    def _get_config_value(self,user_value,prev_user_value,message, alt_condition=None):
        "This function will prompt the use with a question and will keep prompting them until a valid value is given."
        while (alt_condition == None and user_value.strip() == '') or (alt_condition != None and alt_condition(user_value)):
            prev_value_prompt = ''
            if prev_user_value != '':
                prev_value_prompt = '[Press \"ENTER\" to use the previous value: '+str(prev_user_value)+', or type a new value if you want]'
            user_value = input(message + ' ' + prev_value_prompt+'? ')
            user_value = str(user_value).strip() or str(prev_user_value).strip()
            if user_value=='':
                self.log.info('This value cannot be blank.')

        return user_value

    def take_action(self, parsed_args):
        self.log.info('Setting up pancancer config files.')
        config_path=vars(parsed_args)['config_path']
        force_config=vars(parsed_args)['force_config']
        self.log.debug('force_config:' + str(force_config))
        if config_path != None:
            # if the user specifies a path to an existing file, they can use that instead of the interactive process.
            self.log.debug('path to config file: ' + config_path)
            process_config.main(config_path)
        else:
            # if the user did not specify a config, ask them questions and then WRITE/update a config!
            pancancer_config_path=os.path.expanduser('~/.pancancer/simple_pancancer_config.json')
            config_data={}
            if os.path.isfile(pancancer_config_path):
                #Read the pancancer_config.json file if it exists
                with open(pancancer_config_path,'r') as pancancer_config_file:
                    config_data = json.load(pancancer_config_file)

            pancancer_config={}

            aws_config_path = os.path.expanduser('~/.aws/config')
            aws_key=''
            aws_secret_key=''
            prev_aws_key=''
            prev_aws_secret_key=''
            if os.path.isfile(aws_config_path):
                # read the file, get the values
                aws_config = configparser.ConfigParser()
                aws_config.read(aws_config_path)
                prev_aws_key = aws_config['default']['aws_access_key_id']
                prev_aws_secret_key = aws_config['default']['aws_secret_access_key']

            # AWS Key and Secret Key should only be asked for if for some reason they are NOT available (they *should* be, but stuff happens...)
            if force_config  or prev_aws_key.strip() == '' or prev_aws_secret_key.strip() == '':
                aws_key = self._get_config_value(aws_key,prev_aws_key,'What is your AWS Key')
                aws_secret_key = self._get_config_value(aws_secret_key,prev_aws_secret_key,'What is your AWS Secret Key')
                # If the AWS keys weren't available, ~/.aws/config is either not present or missing values, so re-write it.
                aws_config = configparser.ConfigParser()
                aws_config['default'] = {'aws_access_key_id':aws_key, 'aws_secret_access_key':aws_secret_key}
                with open(aws_config_path,'w') as aws_configfile:
                    aws_config.write(aws_configfile,space_around_delimiters=False)

                # Copy the AWS config file to ~/.gnos, because some workflows may need it to access S3 and it's just easier to do this automatically
                # then to tell the user to do it.
                shutil.copy2(aws_config_path,os.path.expanduser('~/.gnos/config'))
            else:
                aws_key = prev_aws_key
                aws_secret_key = prev_aws_secret_key

            # also: get fleet name, PEM key path, and key name from env. If everything went smoothly, these values should be in
            # /opt/from_host/config/pancancer.config
            # Actually, fleet name should be coming from process_config.py, not here, since it's passed into the container as an environment variable.
            # If it's NOT there, we should askt he user for these values.
            bootstrap_config_path='/opt/from_host/config/pancancer.config'
            pem_key_path=''
            prev_pem_key_path=''
            key_name=''
            prev_key_name=''
            prev_security_group = ''
            security_group = ''
            prev_fleet_size = ''
            fleet_size=''
            if os.path.isfile(bootstrap_config_path):
                self.log.debug('bootstrap config file exists at: '+bootstrap_config_path)
                # load simple config file (it should have been generated by install_bootrap) and get values.
                H = dict(line.strip().split('=') for line in open(bootstrap_config_path))
                if 'PEM_PATH' in H:
                    prev_pem_key_path = H['PEM_PATH']
                if 'KEY_NAME' in H:
                    prev_key_name = H['KEY_NAME']

                # Fleet size could be defined in the bootstrap config file which should come from the host machine, or it could be defined in the pancancer config file.
                # If there's nothing in pancancer config, we'll check in the bootstrap config, and if nothing there, default to 1
                if 'max_fleet_size' in config_data:
                    prev_fleet_size = config_data['max_fleet_size']
                else:
                    if 'FLEET_SIZE' in H:
                        prev_fleet_size = H['FLEET_SIZE']
                if str(prev_fleet_size).strip() == '':
                    prev_fleet_size = 1
                if force_config:
                    fleet_size = self._get_config_value(fleet_size, prev_fleet_size, 'How many VMs do you want in your fleet', alt_condition = lambda x: x.strip()=='' or x.isdigit() == False or int(x)<=0 )
                else:
                    fleet_size = prev_fleet_size

                if 'security_group' in config_data:
                    prev_security_group = config_data['security_group']
                else:
                    if 'SECURITY_GROUP' in H:
                        prev_security_group = H['SECURITY_GROUP']
                if prev_security_group.strip() == '':
                    prev_security_group = 'default'
                if force_config:
                    security_group = self._get_config_value(security_group, prev_security_group,'What AWS Security Group should the VMs belong to')
                else:
                    security_group = prev_security_group

                if prev_pem_key_path.strip() != '':
                    pem_key_path = prev_pem_key_path
                if prev_key_name.strip() != '':
                    key_name = prev_key_name

            else:
                self.log.debug('bootstrap config file cannot be found at: '+bootstrap_config_path+' so a new one will be written.')

            # Only ask the use about these if one or the other does not exist (indicates there might be a config value/file problem somewhere)
            if force_config or prev_pem_key_path.strip()=='' or prev_key_name.strip()=='' :
                pem_key_path = self._get_config_value(pem_key_path, prev_pem_key_path, 'What is the path to the AWS pem key file that your new VMs to use')
                key_name = self._get_config_value(key_name, prev_key_name, 'What is the Name of this key in AWS')
                #Now write the bootstrap config file, but only if we had to ask the user for help filling in the details.
                with open(bootstrap_config_path,'w') as bootstrap_config:
                    bootstrap_config.write('PEM_PATH='+pem_key_path+'\n')
                    bootstrap_config.write('KEY_NAME='+key_name+'\n')
                    bootstrap_config.write('FLEET_NAME='+os.environ['FLEET_NAME']+'\n')

            # Now, write the simple JSON config that will be used for the rest of pancancer system.
            pancancer_config = { 'max_fleet_size':fleet_size, 'path_to_key': pem_key_path,
                                'name_of_key':key_name, 'aws_secret_key':aws_secret_key,
                                'security_group': security_group, 'aws_key':aws_key }

            if not os.path.exists(os.path.expanduser('~/.pancancer')):
                os.makedirs(os.path.expanduser('~/.pancancer'))

            with open(pancancer_config_path,'w') as simple_pancancer_config:
                simple_pancancer_config.write(str(json.dumps(pancancer_config,sort_keys=True, indent=4) ))
                self.log.info('Your new pancancer system config file has been written to '+pancancer_config_path)
                self.log.info('The next time you want to run \"pancancer sysconfig\", you can use this file like this: ')
                self.log.info('pancancer sysconfig --config '+pancancer_config_path)

            # Now that a config is written, USE IT!!
            process_config.main(pancancer_config_path)
<|endoftext|>


<commit_before>"""Attempt to fill missing MRMS files

Sadly, the NCEP LDM MRMS feed sometimes misses important files! We need to
look at our archive and attempt to fill those holes.

Run from RUN_2AM.sh
"""
import datetime
import sys
import os
import requests
from pyiem.util import exponential_backoff
import tempfile
import subprocess
import logging

logging.basicConfig()
logger = logging.getLogger()
# logger.setLevel(logging.DEBUG)

PRODS = {'GaugeCorr_QPE_01H': datetime.timedelta(minutes=60),
         'PrecipFlag': datetime.timedelta(minutes=2),
         'PrecipRate': datetime.timedelta(minutes=2),
         'RadarOnly_QPE_01H': datetime.timedelta(minutes=60),
         'RadarOnly_QPE_24H': datetime.timedelta(minutes=60),
         'SeamlessHSR': datetime.timedelta(minutes=2)}

def fetch(prod, now):
    """We have work to do!"""
    uri = now.strftime(("http://mrms.ncep.noaa.gov/data/2D/" +
                        prod + "/MRMS_" + prod +
                        "_00.00_%Y%m%d-%H%M%S.grib2.gz"))

    logger.debug("Fetching %s" % (uri,))
    res = exponential_backoff(requests.get, uri, timeout=60)
    if res is None or res.status_code != 200:
        logger.error("fill_mrms_holes failed to dl: %s" % (uri,))
        return
    (tmpfd, tmpfn) = tempfile.mkstemp()
    os.write(tmpfd, res.content)
    os.close(tmpfd)

    pattern = now.strftime(("/data/realtime/outgoing/grib2/"
                            "MRMS_" + prod + "_00.00_" +
                            "%Y%m%d-%H%M%S.grib2.gz"))
    cmd = "/home/ldm/bin/pqinsert -p '%s' %s" % (pattern, tmpfn)
    subprocess.call(cmd, shell=True)
    os.remove(tmpfn)

def do(prod, sts, ets):
    """Process this stuff now!"""
    now = sts
    while now < ets:
        fn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/" +
                           prod + "/" + prod + "_00.00_%Y%m%d-%H%M%S.grib2.gz"
                           ))
        if not os.path.isfile(fn):
            logger.info("%s is missing for time: %s" % (prod, now))
            fetch(prod, now)
        now += PRODS.get(prod)

def main(argv):
    """Do Something"""
    ets = datetime.datetime.utcnow().replace(minute=0, second=0,
                                             microsecond=0)
    sts = ets - datetime.timedelta(hours=24)
    [do(prod, sts, ets) for prod in PRODS]

if __name__ == '__main__':
    main(sys.argv)
<commit_msg>less aggressively fill MRMS data holes<commit_after>"""Attempt to fill missing MRMS files

Sadly, the NCEP LDM MRMS feed sometimes misses important files! We need to
look at our archive and attempt to fill those holes.

Run from RUN_12Z.sh
"""
import datetime
import sys
import os
import requests
from pyiem.util import exponential_backoff
import tempfile
import subprocess
import logging

logging.basicConfig()
logger = logging.getLogger()
# logger.setLevel(logging.DEBUG)

PRODS = {'GaugeCorr_QPE_01H': datetime.timedelta(minutes=60),
         'PrecipFlag': datetime.timedelta(minutes=2),
         'PrecipRate': datetime.timedelta(minutes=2),
         'RadarOnly_QPE_01H': datetime.timedelta(minutes=60),
         'RadarOnly_QPE_24H': datetime.timedelta(minutes=60),
         'SeamlessHSR': datetime.timedelta(minutes=2)}

def fetch(prod, now):
    """We have work to do!"""
    uri = now.strftime(("http://mrms.ncep.noaa.gov/data/2D/" +
                        prod + "/MRMS_" + prod +
                        "_00.00_%Y%m%d-%H%M%S.grib2.gz"))

    logger.debug("Fetching %s" % (uri,))
    res = exponential_backoff(requests.get, uri, timeout=60)
    if res is None or res.status_code != 200:
        logger.error("fill_mrms_holes failed to dl: %s" % (uri,))
        return
    (tmpfd, tmpfn) = tempfile.mkstemp()
    os.write(tmpfd, res.content)
    os.close(tmpfd)

    pattern = now.strftime(("/data/realtime/outgoing/grib2/"
                            "MRMS_" + prod + "_00.00_" +
                            "%Y%m%d-%H%M%S.grib2.gz"))
    cmd = "/home/ldm/bin/pqinsert -p '%s' %s" % (pattern, tmpfn)
    subprocess.call(cmd, shell=True)
    os.remove(tmpfn)

def do(prod, sts, ets):
    """Process this stuff now!"""
    now = sts
    while now < ets:
        fn = now.strftime(("/mnt/a4/data/%Y/%m/%d/mrms/ncep/" +
                           prod + "/" + prod + "_00.00_%Y%m%d-%H%M%S.grib2.gz"
                           ))
        if not os.path.isfile(fn):
            logger.info("%s is missing for time: %s" % (prod, now))
            fetch(prod, now)
        now += PRODS.get(prod)

def main(argv):
    """Do Something"""
    ets = datetime.datetime.utcnow().replace(minute=0, second=0,
                                             microsecond=0)
    # Don't run too aggressively up until the current time
    ets -= datetime.timedelta(minutes=15)
    sts = ets - datetime.timedelta(hours=24)
    [do(prod, sts, ets) for prod in PRODS]

if __name__ == '__main__':
    main(sys.argv)
<|endoftext|>


<commit_before>import os
import fileinput
import re
import logging
import sys

logging.basicConfig(level=logging.DEBUG)

class FileSetReader:
    def __init__(self):
        self._clear()
        self._logger = logging.getLogger(self.__class__.__name__)

    def _clear(self):
        self._opened_file = None
        self._file_index = -1
        self._line = None

    def initialize(self, input_directory):
        self._clear()
        self._files = os.listdir(input_directory)
        self._files[:] = [f for f in self._files if re.match('.*\.csv$', f) != None]
        self._files[:] = sorted(self._files)
        self._files[:] = [input_directory + '/' + f for f in self._files]

        self._load_next_file()

    def next_line(self):
        self._line = self._opened_file.readline()
        if len(self._line) == 0:
            self._load_next_file()
            if self._opened_file == None:
                return None
            self._line = self._opened_file.readline()
        # self._logger.debug('Line read: %s', self._line.strip())
        return self._line

    def current_line(self):
        return self._line

    def _load_next_file(self):
        if self._opened_file != None:
            self._logger.debug('Closing file: %s', self._opened_file._filename)
            self._opened_file.close()
        self._file_index += 1
        if self._file_index < len(self._files):
            self._logger.debug('Opening file: %s', self._files[self._file_index])
            self._opened_file = fileinput.input(self._files[self._file_index])
        else:
            self._logger.debug('No more files to open.')
            self._opened_file = None

class FiltererOutput:
    def __init__(self, output_directory, lines_per_file=1000000):
        self.lines_per_file = lines_per_file
        self.output_dir = output_directory
        try:
            os.makedirs(self.output_dir)
        except OSError:
            raise Exception("Output directory '%s' already exists. Can't filter due to possible conflicts." % self.output_dir)
        self.clear()

    def clear(self):
        self.output_file_number = 0
        self.line_counter = 0
        self.out = None
        self.open_next_file()

    def open_next_file(self):
        if self.out is not None:
            self.out.close()
        self.output_file_number += 1
        self.out = open(("%s/%05d.csv" % (self.output_dir, self.output_file_number)), "w")
        self.line_counter = 0

    def print_line(self, line):
        if self.line_counter == self.lines_per_file:
            self.open_next_file()
        self.out.write(line + '\n')
        self.line_counter += 1

    def close(self):
        self.out.close()

class PreFilterTaskEvents:
    def __init__(self, input_directory, output_directory):
        self._fileset = FileSetReader()
        self.input_dir = input_directory
        self.output_dir = output_directory
        self._logger = logging.getLogger(self.__class__.__name__)
        self._output = FiltererOutput(output_directory, 10000)

    def filter(self):
        self._fileset.initialize(self.input_dir)

        line = self._fileset.next_line()
        while line is not None:
            # timestamp,missing_info,job_id,task_index,machine_index,
            # event_type,username,scheduling_class,priority,
            # resource_request_cpu,resource_request_ram,
            # resource_request_disk_space,different-machine_constraint
            data = line.split(',')
            timestamp = data[0]
            missing_info = data[1]
            job_id = data[2]
            task_index = data[3]
            event_type = data[5]
            resource_request_cpu = data[9]
            resource_request_mem = data[10]

            if event_type in ['1','2','3','4','5','6'] and missing_info == '':
                #timestamp,event_type,vm_name,cpu,mem
                try:
                    new_event = '{timestamp},{event_type},{vm_name},{cpu},{mem}'.strip().format(

                            timestamp= timestamp,
                            event_type= '0' if event_type == '1' else '1',
                            vm_name= '{}-{}'.format(job_id, task_index),
                            cpu= resource_request_cpu,
                            mem= resource_request_mem

                        )
                    self._output.print_line(new_event)
                except Exception as e:
                    self._logger.error('Couldnot output line: {}'.format(line))
                    self._logger.exception(e)
                    raise e

            line = self._fileset.next_line()

input_dir = sys.argv[1]
output_dir = sys.argv[2]

prefilter = PreFilterTaskEvents(input_dir, output_dir)

print('comecou')

prefilter.filter()

print('terminou')
<commit_msg>TaskEvents preprocessing<commit_after>import os
import fileinput
import re
import logging
import sys

logging.basicConfig(level=logging.DEBUG)

class FileSetReader:
    def __init__(self):
        self._clear()
        self._logger = logging.getLogger(self.__class__.__name__)

    def _clear(self):
        self._opened_file = None
        self._file_index = -1
        self._line = None

    def initialize(self, input_directory):
        self._clear()
        self._files = os.listdir(input_directory)
        self._files[:] = [f for f in self._files if re.match('.*\.csv$', f) != None]
        self._files[:] = sorted(self._files)
        self._files[:] = [input_directory + '/' + f for f in self._files]

        self._load_next_file()

    def next_line(self):
        self._line = self._opened_file.readline()
        if len(self._line) == 0:
            self._load_next_file()
            if self._opened_file == None:
                return None
            self._line = self._opened_file.readline()
        # self._logger.debug('Line read: %s', self._line.strip())
        return self._line

    def current_line(self):
        return self._line

    def _load_next_file(self):
        if self._opened_file != None:
            self._logger.debug('Closing file: %s', self._opened_file._filename)
            self._opened_file.close()
        self._file_index += 1
        if self._file_index < len(self._files):
            self._logger.debug('Opening file: %s', self._files[self._file_index])
            self._opened_file = fileinput.input(self._files[self._file_index])
        else:
            self._logger.debug('No more files to open.')
            self._opened_file = None

class FiltererOutput:
    def __init__(self, output_directory, lines_per_file=1000000):
        self.lines_per_file = lines_per_file
        self.output_dir = output_directory
        try:
            os.makedirs(self.output_dir)
        except OSError:
            raise Exception("Output directory '%s' already exists. Can't filter due to possible conflicts." % self.output_dir)
        self.clear()

    def clear(self):
        self.output_file_number = 0
        self.line_counter = 0
        self.out = None
        self.open_next_file()

    def open_next_file(self):
        if self.out is not None:
            self.out.close()
        self.output_file_number += 1
        self.out = open(("%s/%05d.csv" % (self.output_dir, self.output_file_number)), "w")
        self.line_counter = 0

    def print_line(self, line):
        if self.line_counter == self.lines_per_file:
            self.open_next_file()
        self.out.write(line + '\n')
        self.line_counter += 1

    def close(self):
        self.out.close()

class PreFilterTaskEvents:
    def __init__(self, input_directory, output_directory):
        self._fileset = FileSetReader()
        self.input_dir = input_directory
        self.output_dir = output_directory
        self._logger = logging.getLogger(self.__class__.__name__)
        self._output = FiltererOutput(output_directory, 10000)

    def filter(self):
        self._fileset.initialize(self.input_dir)

        line = self._fileset.next_line()
        while line is not None:
            # timestamp,missing_info,job_id,task_index,machine_index,
            # event_type,username,scheduling_class,priority,
            # resource_request_cpu,resource_request_ram,
            # resource_request_disk_space,different-machine_constraint
            data = line.split(',')
            timestamp = data[0]
            missing_info = data[1]
            job_id = data[2]
            task_index = data[3]
            event_type = data[5]
            resource_request_cpu = data[9]
            resource_request_mem = data[10]

            if event_type in ['1','2','3','4','5','6'] and missing_info == '':
                #timestamp,event_type,vm_name,cpu,mem
                try:
                    new_event = '{timestamp},{event_type},{vm_name},{cpu},{mem}'.strip().format(

                            timestamp= timestamp,
                            event_type= '0' if event_type == '1' else '1',
                            vm_name= '{}-{}'.format(job_id, task_index),
                            cpu= resource_request_cpu,
                            mem= resource_request_mem

                        )
                    self._output.print_line(new_event)
                except Exception as e:
                    self._logger.error('Couldnot output line: {}'.format(line))
                    self._logger.exception(e)
                    raise e

            line = self._fileset.next_line()

input_dir = sys.argv[1]
output_dir = sys.argv[2]

prefilter = PreFilterTaskEvents(input_dir, output_dir)

prefilter.filter()
<|endoftext|>


<commit_before>import time
import random
from datetime import datetime
from exceptions import TimeoutException

DEFAULT_RETRY_TIMEOUT = 60
DEFAULT_QUERIES_PER_SECOND = 10

def qps_retry(original_function=None, **options):
    """ Query Per Second retry decorator
        The intention of this decorator is to retry requests against third
        party services that has QPS restriction.
        Parameters:
            - timeout: Maximum number of seconds to retry
            - qps: Allowed queries per second. This parameter is used to
                   calculate the next time to retry the request
    """
    if original_function is not None:
        def wrapped_function(*args, **kwargs):
            timeout = options.get('timeout', DEFAULT_RETRY_TIMEOUT)
            qps = options.get('qps', DEFAULT_QUERIES_PER_SECOND)
            provider = options.get('provider', None)
            return QPSService(retry_timeout=timeout, queries_per_second=qps, provider=provider).call(original_function, *args, **kwargs)
        return wrapped_function
    else:
        def partial_wrapper(func):
            return qps_retry(func, **options)
        return partial_wrapper

class QPSService:

    def __init__(self, queries_per_second, retry_timeout, provider):
        self._queries_per_second = queries_per_second
        self._retry_timeout = retry_timeout
        self._provider = provider

    def call(self, fn, *args, **kwargs):
        start_time = datetime.now()
        attempt_number = 1
        while True:
            try:
                return fn(*args, **kwargs)
            except Exception as e:
                response = getattr(e, 'response', None)
                if response is not None:
                    if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403):
                        self.retry(start_time, attempt_number)
                    elif response.status_code == 429:
                        self.retry(start_time, attempt_number)
                    else:
                        raise e
                else:
                    raise e
            attempt_number += 1

    def retry(self, first_request_time, retry_count):
        elapsed = datetime.now() - first_request_time
        if elapsed.total_seconds() > self._retry_timeout:
            raise TimeoutException()

        # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
        # iteration.
        delay = (1.0/self._queries_per_second) * 1.5 ** retry_count

        # https://www.awsarchitectureblog.com/2015/03/backoff.html
        # https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py#L193
        sleep_time = delay * (random.random() + 0.5)

        time.sleep(sleep_time)
<commit_msg>TOMTOM uses 403 instead of 429 for rate limiting<commit_after>import time
import random
from datetime import datetime
from exceptions import TimeoutException

DEFAULT_RETRY_TIMEOUT = 60
DEFAULT_QUERIES_PER_SECOND = 10

TOMTOM_403_RATE_LIMIT_HEADER = 'Account Over Queries Per Second Limit'
TOMTOM_DETAIL_HEADER = 'X-Error-Detail-Header'

def qps_retry(original_function=None, **options):
    """ Query Per Second retry decorator
        The intention of this decorator is to retry requests against third
        party services that has QPS restriction.
        Parameters:
            - timeout: Maximum number of seconds to retry
            - qps: Allowed queries per second. This parameter is used to
                   calculate the next time to retry the request
    """
    if original_function is not None:
        def wrapped_function(*args, **kwargs):
            timeout = options.get('timeout', DEFAULT_RETRY_TIMEOUT)
            qps = options.get('qps', DEFAULT_QUERIES_PER_SECOND)
            provider = options.get('provider', None)
            return QPSService(retry_timeout=timeout, queries_per_second=qps, provider=provider).call(original_function, *args, **kwargs)
        return wrapped_function
    else:
        def partial_wrapper(func):
            return qps_retry(func, **options)
        return partial_wrapper

class QPSService:

    def __init__(self, queries_per_second, retry_timeout, provider):
        self._queries_per_second = queries_per_second
        self._retry_timeout = retry_timeout
        self._provider = provider

    def call(self, fn, *args, **kwargs):
        start_time = datetime.now()
        attempt_number = 1
        while True:
            try:
                return fn(*args, **kwargs)
            except Exception as e:
                response = getattr(e, 'response', None)
                if response is not None:
                    if self._provider is not None and self._provider == 'tomtom' and (response.status_code == 403):
                        if response.headers.get(TOMTOM_DETAIL_HEADER) != TOMTOM_403_RATE_LIMIT_HEADER:
                            raise e
                        self.retry(start_time, attempt_number)
                    elif response.status_code == 429:
                        self.retry(start_time, attempt_number)
                    else:
                        raise e
                else:
                    raise e
            attempt_number += 1

    def retry(self, first_request_time, retry_count):
        elapsed = datetime.now() - first_request_time
        if elapsed.total_seconds() > self._retry_timeout:
            raise TimeoutException()

        # inverse qps * (1.5 ^ i) is an increased sleep time of 1.5x per
        # iteration.
        delay = (1.0/self._queries_per_second) * 1.5 ** retry_count

        # https://www.awsarchitectureblog.com/2015/03/backoff.html
        # https://github.com/googlemaps/google-maps-services-python/blob/master/googlemaps/client.py#L193
        sleep_time = delay * (random.random() + 0.5)

        time.sleep(sleep_time)
<|endoftext|>


<commit_before>class rectangle():
	"""docstring for rectangle

	quick variable name glossary
	dim = dimension of the vector to the next surface
	pos = position of the calculation point
	posR = state of current rotation (angle between dimension vector and axis)
	max = maximal distance in some direction(this is used to quick check for collisions)
	motT = translational motion
	motR = rotational motion
	mass = mass
	material = material of the object (irrelevant for now but can be filled out for readability of output)
	"""
	def __init__(self, pos, posR, dim, motT, motR):
		self.pos = pos
		self.posR = posR
		self.dim = dim
		self.motT = motT
		self.motR = motR
		setPoint(pos, posR, dim)
		#this determines which of the points is the most far out
		self.xPointArray = [self.A[0], self.B[0], self.C[0], self.D[0], self.E[0], self.F[0], self.G[0], self.H[0]]
		self.xPointArray.sort()
		self.yPointArray = [self.A[1], self.B[1], self.C[1], self.D[1], self.E[1], self.F[1], self.G[1], self.H[1]]
		self.yPointArray.sort()
		self.zPointArray = [self.A[2], self.B[2], self.C[2], self.D[2], self.E[2], self.F[2], self.G[2], self.H[2]]
		self.zPointArray.sort()
		self.quickCollisionXDim = [self.xPointArray[0], self.xPointArray[7]]
		self.quickCollisionYDim = [self.yPointArray[0], self.yPointArray[7]]
		self.quickCollisionZDim = [self.zPointArray[0], self.zPointArray[7]]

	def setPoint(self, pos, posR, dim):
		self.A = [((pos[0] - dim[0]/2) * math.cos(posR[0])), ((pos[1] - dim[1]/2) * math.cos(posR[1])), ((pos[2] - dim[2]/2) * math.cos(posR[2]))]
		self.B = [((pos[0] + dim[0]/2) * math.cos(posR[0])), ((pos[1] - dim[1]/2) * math.cos(posR[1])), ((pos[2] - dim[2]/2) * math.cos(posR[2]))]
		self.C = [((pos[0] - dim[0]/2) * maht.cos(posR[0])), ((pos[1] + dim[1]/2) * math.cos(posR[1])), ((pos[2] - dim[2]/2) * math.cos(posR[2]))]
		self.D = [((pos[0] + dim[0]/2) * maht.cos(posR[0])), ((pos[1] + dim[1]/2) * math.cos(posR[1])), ((pos[2] - dim[2]/2) * math.cos(posR[2]))]
		self.E = [((pos[0] - dim[0]/2) * maht.cos(posR[0])), ((pos[1] - dim[1]/2) * math.cos(posR[1])), ((pos[2] + dim[2]/2) * math.cos(posR[2]))]
		self.F = [((pos[0] + dim[0]/2) * maht.cos(posR[0])), ((pos[1] - dim[1]/2) * math.cos(posR[1])), ((pos[2] + dim[2]/2) * math.cos(posR[2]))]
		self.G = [((pos[0] - dim[0]/2) * maht.cos(posR[0])), ((pos[1] + dim[1]/2) * math.cos(posR[1])), ((pos[2] + dim[2]/2) * math.cos(posR[2]))]
		self.H = [((pos[0] + dim[0]/2) * maht.cos(posR[0])), ((pos[1] + dim[1]/2) * math.cos(posR[1])), ((pos[2] + dim[2]/2) * math.cos(posR[2]))]

	def quickCollision(self, point):
		collision = False
		if point[0] > self.quickCollisionXDim[0] and point[0] < self.quickCollisionXDim[1]:
			if point[1] > self.quickCollisionYDim[0] and point[1] < self.quickCollisionYDim[1]:
				if point[2] > self.quickCollisionZDim[0] and point[2] < self.quickCollisionZDim[1]:
					collision = True
				else:
					collision = False
			else:
				pass
		else:
			pass
<commit_msg>created redefineQuickCollisionBundingBox method for rectangle class<commit_after>class rectangle():
	"""docstring for rectangle

	quick variable name glossary
	dim = dimension of the vector to the next surface
	pos = position of the calculation point
	posR = state of current rotation (angle between dimension vector and axis)
	max = maximal distance in some direction(this is used to quick check for collisions)
	motT = translational motion
	motR = rotational motion
	mass = mass
	material = material of the object (irrelevant for now but can be filled out for readability of output)
	"""
	def __init__(self, pos, posR, dim, motT, motR):
		self.pos = pos
		self.posR = posR
		self.dim = dim
		self.motT = motT
		self.motR = motR
		setPoint(pos, posR, dim)
		#this determines which of the points is the most far out
		self.xPointArray = [self.A[0], self.B[0], self.C[0], self.D[0], self.E[0], self.F[0], self.G[0], self.H[0]]
		self.xPointArray.sort()
		self.yPointArray = [self.A[1], self.B[1], self.C[1], self.D[1], self.E[1], self.F[1], self.G[1], self.H[1]]
		self.yPointArray.sort()
		self.zPointArray = [self.A[2], self.B[2], self.C[2], self.D[2], self.E[2], self.F[2], self.G[2], self.H[2]]
		self.zPointArray.sort()
		self.quickCollisionXDim = [self.xPointArray[0], self.xPointArray[7]]
		self.quickCollisionYDim = [self.yPointArray[0], self.yPointArray[7]]
		self.quickCollisionZDim = [self.zPointArray[0], self.zPointArray[7]]

	def setPoint(self):
		self.A = [((self.pos[0] - self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] - self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] - self.dim[2]/2) * math.cos(self.posR[2]))]
		self.B = [((self.pos[0] + self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] - self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] - self.dim[2]/2) * math.cos(self.posR[2]))]
		self.C = [((self.pos[0] - self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] + self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] - self.dim[2]/2) * math.cos(self.posR[2]))]
		self.D = [((self.pos[0] + self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] + self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] - self.dim[2]/2) * math.cos(self.posR[2]))]
		self.E = [((self.pos[0] - self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] - self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] + self.dim[2]/2) * math.cos(self.posR[2]))]
		self.F = [((self.pos[0] + self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] - self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] + self.dim[2]/2) * math.cos(self.posR[2]))]
		self.G = [((self.pos[0] - self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] + self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] + self.dim[2]/2) * math.cos(self.posR[2]))]
		self.H = [((self.pos[0] + self.dim[0]/2) * math.cos(self.posR[0])), ((self.pos[1] + self.dim[1]/2) * math.cos(self.posR[1])), ((self.pos[2] + self.dim[2]/2) * math.cos(self.posR[2]))]

	def quickCollision(self, quickCollisionXDimB, quickCollisionYDimB, quickCollisionZDimB):
		collision = False
		if self.quickCollisionXDim[0] < quickCollisionXDimB[1] and self.quickCollisionXDim[1] > quickCollisionXDimB[0]:
			if self.quickCollisionYDim[0] < quickCollisionYDimB[1] and self.quickCollisionYDim[1] > quickCollisionYDimB[0]:
				if self.quickCollisionZDim[0] < quickCollisionZDim[1] and self.quickCollisionZDim[1] > quickCollisionZDimB[0]:
					collision = True
				else:
					pass
			else:
				pass
		else:
			pass
		return collision

	def redefineQuickCollisionBoundingBox(self):
		setPoint()
		self.xPointArray = [self.A[0], self.B[0], self.C[0], self.D[0], self.E[0], self.F[0], self.G[0], self.H[0]]
		self.xPointArray.sort()
		self.yPointArray = [self.A[1], self.B[1], self.C[1], self.D[1], self.E[1], self.F[1], self.G[1], self.H[1]]
		self.yPointArray.sort()
		self.zPointArray = [self.A[2], self.B[2], self.C[2], self.D[2], self.E[2], self.F[2], self.G[2], self.H[2]]
		self.zPointArray.sort()
		self.quickCollisionXDim = [self.xPointArray[0], self.xPointArray[7]]
		self.quickCollisionYDim = [self.yPointArray[0], self.yPointArray[7]]
		self.quickCollisionZDim = [self.zPointArray[0], self.zPointArray[7]]<|endoftext|>


<commit_before>import pytest
import time

from plenum.server.batch_handlers.three_pc_batch import ThreePcBatch
from plenum.test.testing_utils import FakeSomething

from plenum.common.txn_util import reqToTxn

from common.exceptions import LogicError

from plenum.common.request import Request
from plenum.common.util import randomString
from plenum.common.constants import TARGET_NYM, NODE, DOMAIN_LEDGER_ID

from plenum.server.batch_handlers.domain_batch_handler import DomainBatchHandler
from plenum.server.request_handlers.node_handler import NodeHandler
from plenum.server.database_manager import DatabaseManager
from plenum.server.request_managers.write_request_manager import WriteRequestManager

@pytest.fixture(scope='function')
def db():
    return DatabaseManager()

@pytest.fixture(scope='function')
def write_req_manager(db):
    manager = WriteRequestManager(db)

    write_req_handler = NodeHandler(db, None)
    batch_req_handler = DomainBatchHandler(db)

    manager.register_req_handler(write_req_handler)
    manager.register_batch_handler(batch_req_handler)

    # We do not need to check request handler workability
    handler = manager.request_handlers[NODE][0]
    handler.static_validation = lambda request: 1
    handler.dynamic_validation = lambda request: 1
    handler.update_state = lambda txn, prev_result, is_committed: 1
    handler.apply_request = lambda request, batch_ts, prev_result: (1, 1, 1)
    handler.apply_forced_request = lambda request: 1
    handler.transform_txn_for_ledger = lambda txn: 1

    # Same for batches
    handler = manager.batch_handlers[DOMAIN_LEDGER_ID][0]
    handler.post_batch_applied = lambda batch, prev_handler_result: 1
    handler.commit_batch = lambda ledger_id, txn_count, state_root, txn_root, pp_time, prev_handler_result: 1
    handler.post_batch_rejected = lambda ledger_id, prev_handler_result: 1

    return manager

@pytest.fixture(scope='function')
def node_req():
    return Request(identifier=randomString(),
                   reqId=5,
                   operation={'type': NODE,
                              'dest': randomString(),
                              TARGET_NYM: randomString(),
                              'data': {}
                              }
                   )

@pytest.fixture(scope='function')
def three_pc_batch():
    return ThreePcBatch(DOMAIN_LEDGER_ID, 0, 0, 1, time.time(),
                        randomString(),
                        randomString(),
                        ['a', 'b', 'c'], ['d1', 'd2', 'd3'])

def test_write_request_manager_fails_to_handle(write_req_manager: WriteRequestManager,
                                               node_req):
    node_req.operation['type'] = 999

    with pytest.raises(LogicError):
        write_req_manager.static_validation(node_req)

    with pytest.raises(LogicError):
        write_req_manager.dynamic_validation(node_req)

    with pytest.raises(LogicError):
        write_req_manager.update_state(reqToTxn(node_req))

    with pytest.raises(LogicError):
        write_req_manager.apply_request(node_req, None)

    with pytest.raises(LogicError):
        write_req_manager.apply_forced_request(node_req)

    with pytest.raises(LogicError):
        write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))

def test_write_request_manager_handles_request(write_req_manager: WriteRequestManager,
                                               node_req):
    write_req_manager.static_validation(node_req)
    write_req_manager.dynamic_validation(node_req)
    write_req_manager.update_state(reqToTxn(node_req))
    write_req_manager.apply_request(node_req, None)
    write_req_manager.apply_forced_request(node_req)
    write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))

def test_write_request_manager_chain_of_responsib_apply(write_req_manager: WriteRequestManager,
                                                        node_req, db):
    write_req_manager.request_handlers[NODE] = []
    handlers = write_req_manager.request_handlers[NODE]
    check_list = [FakeSomething(), FakeSomething(), FakeSomething()]
    node_req.check_list = check_list
    for check in check_list:
        check.check_field = False

    def modify_check_list(request, batch_ts, prev_result):
        assert not all(check.check_field for check in check_list)
        f_check = next(check for check in request.check_list if check.check_field is False)
        f_check.check_field = True
        return 1, 1, 1

    for i in range(3):
        handler = NodeHandler(db, None)
        handler.apply_request = modify_check_list
        handlers.append(handler)

    write_req_manager.apply_request(node_req, 0)

    assert all(check.check_field for check in check_list)

def test_write_request_manager_fails_to_handle_batches(write_req_manager: WriteRequestManager,
                                                       three_pc_batch):
    nonexistent_lid = 999
    three_pc_batch.ledger_id = nonexistent_lid

    with pytest.raises(LogicError):
        write_req_manager.post_apply_batch(three_pc_batch)

    with pytest.raises(LogicError):
        write_req_manager.post_batch_rejected(three_pc_batch)

    with pytest.raises(LogicError):
        write_req_manager.commit_batch(nonexistent_lid,
                                       len(three_pc_batch.valid_digests),
                                       three_pc_batch.state_root,
                                       three_pc_batch.txn_root,
                                       three_pc_batch.pp_time)

def test_write_request_manager_handles_batches(write_req_manager: WriteRequestManager,
                                               three_pc_batch):
    write_req_manager.post_apply_batch(three_pc_batch)
    write_req_manager.commit_batch(three_pc_batch.ledger_id,
                                   len(three_pc_batch.valid_digests),
                                   three_pc_batch.state_root,
                                   three_pc_batch.txn_root,
                                   three_pc_batch.pp_time)
    write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)

def test_write_request_manager_chain_of_responsib_batch(write_req_manager: WriteRequestManager,
                                                        three_pc_batch, db):
    write_req_manager.batch_handlers[DOMAIN_LEDGER_ID] = []
    handlers = write_req_manager.batch_handlers[DOMAIN_LEDGER_ID]

    check_list = [FakeSomething(), FakeSomething(), FakeSomething()]

    def modify_check_list():
        assert not all(check.check_field for check in check_list)
        f_check = next(check for check in check_list if check.check_field is False)
        f_check.check_field = True

    def modify_check_list_post_apply(batch, prev_result):
        modify_check_list()
        return 1, 1, 1

    def modify_check_list_commit(ledger_id, txn_count, state_root, txn_root, pp_time, prev_handler_result):
        modify_check_list()
        return 1, 1, 1

    def modify_check_list_post_rejected(lid, prev_result):
        modify_check_list()
        return 1, 1, 1

    for i in range(3):
        handler = DomainBatchHandler(db)
        handler.post_batch_applied = modify_check_list_post_apply
        handler.commit_batch = modify_check_list_commit
        handler.post_batch_rejected = modify_check_list_post_rejected
        handlers.append(handler)

    for check in check_list:
        check.check_field = False
    write_req_manager.post_apply_batch(three_pc_batch)
    assert all(check.check_field for check in check_list)

    for check in check_list:
        check.check_field = False
    write_req_manager.commit_batch(three_pc_batch.ledger_id,
                                   len(three_pc_batch.valid_digests),
                                   three_pc_batch.state_root,
                                   three_pc_batch.txn_root,
                                   three_pc_batch.pp_time)
    assert all(check.check_field for check in check_list)

    for check in check_list:
        check.check_field = False
    write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
    assert all(check.check_field for check in check_list)
<commit_msg>INDY-2108: Fix tests<commit_after>import pytest
import time

from plenum.server.batch_handlers.three_pc_batch import ThreePcBatch
from plenum.test.testing_utils import FakeSomething

from plenum.common.txn_util import reqToTxn

from common.exceptions import LogicError

from plenum.common.request import Request
from plenum.common.util import randomString
from plenum.common.constants import TARGET_NYM, NODE, DOMAIN_LEDGER_ID

from plenum.server.batch_handlers.domain_batch_handler import DomainBatchHandler
from plenum.server.request_handlers.node_handler import NodeHandler
from plenum.server.database_manager import DatabaseManager
from plenum.server.request_managers.write_request_manager import WriteRequestManager

@pytest.fixture(scope='function')
def db():
    return DatabaseManager()

@pytest.fixture(scope='function')
def write_req_manager(db):
    manager = WriteRequestManager(db)

    write_req_handler = NodeHandler(db, None)
    batch_req_handler = DomainBatchHandler(db)

    manager.register_req_handler(write_req_handler)
    manager.register_batch_handler(batch_req_handler)

    # We do not need to check request handler workability
    handler = manager.request_handlers[NODE][0]
    handler.static_validation = lambda request: 1
    handler.dynamic_validation = lambda request: 1
    handler.update_state = lambda txn, updated_state, request, is_committed: 1
    handler.apply_request = lambda request, batch_ts, prev_result: (1, 1, 1)
    handler.apply_forced_request = lambda request: 1
    handler.transform_txn_for_ledger = lambda txn: 1

    # Same for batches
    handler = manager.batch_handlers[DOMAIN_LEDGER_ID][0]
    handler.post_batch_applied = lambda batch, prev_handler_result: 1
    handler.commit_batch = lambda ledger_id, txn_count, state_root, txn_root, pp_time, prev_handler_result: 1
    handler.post_batch_rejected = lambda ledger_id, prev_handler_result: 1

    return manager

@pytest.fixture(scope='function')
def node_req():
    return Request(identifier=randomString(),
                   reqId=5,
                   operation={'type': NODE,
                              'dest': randomString(),
                              TARGET_NYM: randomString(),
                              'data': {}
                              }
                   )

@pytest.fixture(scope='function')
def three_pc_batch():
    return ThreePcBatch(DOMAIN_LEDGER_ID, 0, 0, 1, time.time(),
                        randomString(),
                        randomString(),
                        ['a', 'b', 'c'], ['d1', 'd2', 'd3'])

def test_write_request_manager_fails_to_handle(write_req_manager: WriteRequestManager,
                                               node_req):
    node_req.operation['type'] = 999

    with pytest.raises(LogicError):
        write_req_manager.static_validation(node_req)

    with pytest.raises(LogicError):
        write_req_manager.dynamic_validation(node_req)

    with pytest.raises(LogicError):
        write_req_manager.update_state(reqToTxn(node_req))

    with pytest.raises(LogicError):
        write_req_manager.apply_request(node_req, None)

    with pytest.raises(LogicError):
        write_req_manager.apply_forced_request(node_req)

    with pytest.raises(LogicError):
        write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))

def test_write_request_manager_handles_request(write_req_manager: WriteRequestManager,
                                               node_req):
    write_req_manager.static_validation(node_req)
    write_req_manager.dynamic_validation(node_req)
    write_req_manager.update_state(reqToTxn(node_req))
    write_req_manager.apply_request(node_req, None)
    write_req_manager.apply_forced_request(node_req)
    write_req_manager.transform_txn_for_ledger(reqToTxn(node_req))

def test_write_request_manager_chain_of_responsib_apply(write_req_manager: WriteRequestManager,
                                                        node_req, db):
    write_req_manager.request_handlers[NODE] = []
    handlers = write_req_manager.request_handlers[NODE]
    check_list = [FakeSomething(), FakeSomething(), FakeSomething()]
    node_req.check_list = check_list
    for check in check_list:
        check.check_field = False

    def modify_check_list(request, batch_ts, prev_result):
        assert not all(check.check_field for check in check_list)
        f_check = next(check for check in request.check_list if check.check_field is False)
        f_check.check_field = True
        return 1, 1, 1

    for i in range(3):
        handler = NodeHandler(db, None)
        handler.apply_request = modify_check_list
        handlers.append(handler)

    write_req_manager.apply_request(node_req, 0)

    assert all(check.check_field for check in check_list)

def test_write_request_manager_fails_to_handle_batches(write_req_manager: WriteRequestManager,
                                                       three_pc_batch):
    nonexistent_lid = 999
    three_pc_batch.ledger_id = nonexistent_lid

    with pytest.raises(LogicError):
        write_req_manager.post_apply_batch(three_pc_batch)

    with pytest.raises(LogicError):
        write_req_manager.post_batch_rejected(three_pc_batch)

    with pytest.raises(LogicError):
        write_req_manager.commit_batch(nonexistent_lid,
                                       len(three_pc_batch.valid_digests),
                                       three_pc_batch.state_root,
                                       three_pc_batch.txn_root,
                                       three_pc_batch.pp_time)

def test_write_request_manager_handles_batches(write_req_manager: WriteRequestManager,
                                               three_pc_batch):
    write_req_manager.post_apply_batch(three_pc_batch)
    write_req_manager.commit_batch(three_pc_batch.ledger_id,
                                   len(three_pc_batch.valid_digests),
                                   three_pc_batch.state_root,
                                   three_pc_batch.txn_root,
                                   three_pc_batch.pp_time)
    write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)

def test_write_request_manager_chain_of_responsib_batch(write_req_manager: WriteRequestManager,
                                                        three_pc_batch, db):
    write_req_manager.batch_handlers[DOMAIN_LEDGER_ID] = []
    handlers = write_req_manager.batch_handlers[DOMAIN_LEDGER_ID]

    check_list = [FakeSomething(), FakeSomething(), FakeSomething()]

    def modify_check_list():
        assert not all(check.check_field for check in check_list)
        f_check = next(check for check in check_list if check.check_field is False)
        f_check.check_field = True

    def modify_check_list_post_apply(batch, prev_result):
        modify_check_list()
        return 1, 1, 1

    def modify_check_list_commit(ledger_id, txn_count, state_root, txn_root, pp_time, prev_handler_result):
        modify_check_list()
        return 1, 1, 1

    def modify_check_list_post_rejected(lid, prev_result):
        modify_check_list()
        return 1, 1, 1

    for i in range(3):
        handler = DomainBatchHandler(db)
        handler.post_batch_applied = modify_check_list_post_apply
        handler.commit_batch = modify_check_list_commit
        handler.post_batch_rejected = modify_check_list_post_rejected
        handlers.append(handler)

    for check in check_list:
        check.check_field = False
    write_req_manager.post_apply_batch(three_pc_batch)
    assert all(check.check_field for check in check_list)

    for check in check_list:
        check.check_field = False
    write_req_manager.commit_batch(three_pc_batch.ledger_id,
                                   len(three_pc_batch.valid_digests),
                                   three_pc_batch.state_root,
                                   three_pc_batch.txn_root,
                                   three_pc_batch.pp_time)
    assert all(check.check_field for check in check_list)

    for check in check_list:
        check.check_field = False
    write_req_manager.post_batch_rejected(three_pc_batch.ledger_id)
    assert all(check.check_field for check in check_list)
<|endoftext|>


<commit_before>from cherrypy import wsgiserver
from cherrypy.wsgiserver import ssl_builtin
from hbproject import wsgi as django_wsgi

if __name__ == "__main__":
    host = "127.0.0.1"
    port = 8080
    server = wsgiserver.CherryPyWSGIServer(
        (host, port), django_wsgi.application,
        server_name=None, numthreads=2, timeout=30, max=500)
    print("WSGI-hosting Server started on %s:%s" % (host, port))
    server.start()

<commit_msg>modified cherrypy_launcher to expose the static content (css, img, js...)<commit_after>import os
import cherrypy
from cherrypy import wsgiserver
from cherrypy.wsgiserver import ssl_builtin
from hbproject import wsgi as django_wsgi

PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))

class Root(object):
    pass

def make_static_config(static_dir_name):
    """
    All custom static configurations are set here, since most are common, it
    makes sense to generate them just once.
    """
    static_path = os.path.join('/', static_dir_name)
    path = os.path.join(PATH, static_dir_name)
    configuration = {static_path: {
        'tools.staticdir.on': True,
        'tools.staticdir.dir': path}
    }
    print configuration
    return cherrypy.tree.mount(Root(), '/', config=configuration)

if __name__ == "__main__":
    host = "0.0.0.0"
    port = 8000

    application = wsgiserver.WSGIPathInfoDispatcher({
        '/': django_wsgi.application,
        '/static': make_static_config('static')})

    server = wsgiserver.CherryPyWSGIServer(
        (host, port), application,
        server_name=None, numthreads=2, timeout=30, max=500)
    print("WSGI-hosting Server started on %s:%s" % (host, port))
    server.start()

<|endoftext|>


<commit_before>"""Support for monitoring the local system."""
import logging
import os
import socket
import sys

import psutil
import voluptuous as vol

from homeassistant.components.sensor import PLATFORM_SCHEMA
from homeassistant.const import (
    CONF_RESOURCES,
    CONF_TYPE,
    DATA_GIBIBYTES,
    DATA_MEBIBYTES,
    DATA_RATE_MEGABYTES_PER_SECOND,
    PERCENTAGE,
    STATE_OFF,
    STATE_ON,
    TEMP_CELSIUS,
)
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity import Entity
from homeassistant.util import slugify
import homeassistant.util.dt as dt_util

# mypy: allow-untyped-defs, no-check-untyped-defs

_LOGGER = logging.getLogger(__name__)

CONF_ARG = "arg"

if sys.maxsize > 2 ** 32:
    CPU_ICON = "mdi:cpu-64-bit"
else:
    CPU_ICON = "mdi:cpu-32-bit"

SENSOR_TYPES = {
    "disk_free": ["Disk free", DATA_GIBIBYTES, "mdi:harddisk", None],
    "disk_use": ["Disk use", DATA_GIBIBYTES, "mdi:harddisk", None],
    "disk_use_percent": ["Disk use (percent)", PERCENTAGE, "mdi:harddisk", None],
    "ipv4_address": ["IPv4 address", "", "mdi:server-network", None],
    "ipv6_address": ["IPv6 address", "", "mdi:server-network", None],
    "last_boot": ["Last boot", "", "mdi:clock", "timestamp"],
    "load_15m": ["Load (15m)", " ", "mdi:memory", None],
    "load_1m": ["Load (1m)", " ", "mdi:memory", None],
    "load_5m": ["Load (5m)", " ", "mdi:memory", None],
    "memory_free": ["Memory free", DATA_MEBIBYTES, "mdi:memory", None],
    "memory_use": ["Memory use", DATA_MEBIBYTES, "mdi:memory", None],
    "memory_use_percent": ["Memory use (percent)", PERCENTAGE, "mdi:memory", None],
    "network_in": ["Network in", DATA_MEBIBYTES, "mdi:server-network", None],
    "network_out": ["Network out", DATA_MEBIBYTES, "mdi:server-network", None],
    "packets_in": ["Packets in", " ", "mdi:server-network", None],
    "packets_out": ["Packets out", " ", "mdi:server-network", None],
    "throughput_network_in": [
        "Network throughput in",
        DATA_RATE_MEGABYTES_PER_SECOND,
        "mdi:server-network",
        None,
    ],
    "throughput_network_out": [
        "Network throughput out",
        DATA_RATE_MEGABYTES_PER_SECOND,
        "mdi:server-network",
        None,
    ],
    "process": ["Process", " ", CPU_ICON, None],
    "processor_use": ["Processor use", PERCENTAGE, CPU_ICON, None],
    "processor_temperature": ["Processor temperature", TEMP_CELSIUS, CPU_ICON, None],
    "swap_free": ["Swap free", DATA_MEBIBYTES, "mdi:harddisk", None],
    "swap_use": ["Swap use", DATA_MEBIBYTES, "mdi:harddisk", None],
    "swap_use_percent": ["Swap use (percent)", PERCENTAGE, "mdi:harddisk", None],
}

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {
        vol.Optional(CONF_RESOURCES, default={CONF_TYPE: "disk_use"}): vol.All(
            cv.ensure_list,
            [
                vol.Schema(
                    {
                        vol.Required(CONF_TYPE): vol.In(SENSOR_TYPES),
                        vol.Optional(CONF_ARG): cv.string,
                    }
                )
            ],
        )
    }
)

IO_COUNTER = {
    "network_out": 0,
    "network_in": 1,
    "packets_out": 2,
    "packets_in": 3,
    "throughput_network_out": 0,
    "throughput_network_in": 1,
}

IF_ADDRS_FAMILY = {"ipv4_address": socket.AF_INET, "ipv6_address": socket.AF_INET6}

# There might be additional keys to be added for different
# platforms / hardware combinations.
# Taken from last version of "glances" integration before they moved to
# a generic temperature sensor logic.
# https://github.com/home-assistant/core/blob/5e15675593ba94a2c11f9f929cdad317e27ce190/homeassistant/components/glances/sensor.py#L199
CPU_SENSOR_PREFIXES = [
    "amdgpu 1",
    "aml_thermal",
    "Core 0",
    "Core 1",
    "CPU Temperature",
    "CPU",
    "cpu-thermal 1",
    "cpu_thermal 1",
    "exynos-therm 1",
    "Package id 0",
    "Physical id 0",
    "radeon 1",
    "soc-thermal 1",
    "soc_thermal 1",
]

def setup_platform(hass, config, add_entities, discovery_info=None):
    """Set up the system monitor sensors."""
    dev = []
    for resource in config[CONF_RESOURCES]:
        # Initialize the sensor argument if none was provided.
        # For disk monitoring default to "/" (root) to prevent runtime errors, if argument was not specified.
        if CONF_ARG not in resource:
            if resource[CONF_TYPE].startswith("disk_"):
                resource[CONF_ARG] = "/"
            else:
                resource[CONF_ARG] = ""

        # Verify if we can retrieve CPU / processor temperatures.
        # If not, do not create the entity and add a warning to the log
        if resource[CONF_TYPE] == "processor_temperature":
            if SystemMonitorSensor.read_cpu_temperature() is None:
                _LOGGER.warning("Cannot read CPU / processor temperature information.")
                continue

        dev.append(SystemMonitorSensor(resource[CONF_TYPE], resource[CONF_ARG]))

    add_entities(dev, True)

class SystemMonitorSensor(Entity):
    """Implementation of a system monitor sensor."""

    def __init__(self, sensor_type, argument=""):
        """Initialize the sensor."""
        self._name = "{} {}".format(SENSOR_TYPES[sensor_type][0], argument)
        self._unique_id = slugify(f"{sensor_type}_{argument}")
        self.argument = argument
        self.type = sensor_type
        self._state = None
        self._unit_of_measurement = SENSOR_TYPES[sensor_type][1]
        self._available = True
        if sensor_type in ["throughput_network_out", "throughput_network_in"]:
            self._last_value = None
            self._last_update_time = None

    @property
    def name(self):
        """Return the name of the sensor."""
        return self._name.rstrip()

    @property
    def unique_id(self):
        """Return the unique ID."""
        return self._unique_id

    @property
    def device_class(self):
        """Return the class of this sensor."""
        return SENSOR_TYPES[self.type][3]

    @property
    def icon(self):
        """Icon to use in the frontend, if any."""
        return SENSOR_TYPES[self.type][2]

    @property
    def state(self):
        """Return the state of the device."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement of this entity, if any."""
        return self._unit_of_measurement

    @property
    def available(self):
        """Return True if entity is available."""
        return self._available

    def update(self):
        """Get the latest system information."""
        if self.type == "disk_use_percent":
            self._state = psutil.disk_usage(self.argument).percent
        elif self.type == "disk_use":
            self._state = round(psutil.disk_usage(self.argument).used / 1024 ** 3, 1)
        elif self.type == "disk_free":
            self._state = round(psutil.disk_usage(self.argument).free / 1024 ** 3, 1)
        elif self.type == "memory_use_percent":
            self._state = psutil.virtual_memory().percent
        elif self.type == "memory_use":
            virtual_memory = psutil.virtual_memory()
            self._state = round(
                (virtual_memory.total - virtual_memory.available) / 1024 ** 2, 1
            )
        elif self.type == "memory_free":
            self._state = round(psutil.virtual_memory().available / 1024 ** 2, 1)
        elif self.type == "swap_use_percent":
            self._state = psutil.swap_memory().percent
        elif self.type == "swap_use":
            self._state = round(psutil.swap_memory().used / 1024 ** 2, 1)
        elif self.type == "swap_free":
            self._state = round(psutil.swap_memory().free / 1024 ** 2, 1)
        elif self.type == "processor_use":
            self._state = round(psutil.cpu_percent(interval=None))
        elif self.type == "processor_temperature":
            self._state = self.read_cpu_temperature()
        elif self.type == "process":
            for proc in psutil.process_iter():
                try:
                    if self.argument == proc.name():
                        self._state = STATE_ON
                        return
                except psutil.NoSuchProcess as err:
                    _LOGGER.warning(
                        "Failed to load process with id: %s, old name: %s",
                        err.pid,
                        err.name,
                    )
            self._state = STATE_OFF
        elif self.type == "network_out" or self.type == "network_in":
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                counter = counters[self.argument][IO_COUNTER[self.type]]
                self._state = round(counter / 1024 ** 2, 1)
            else:
                self._state = None
        elif self.type == "packets_out" or self.type == "packets_in":
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                self._state = counters[self.argument][IO_COUNTER[self.type]]
            else:
                self._state = None
        elif (
            self.type == "throughput_network_out"
            or self.type == "throughput_network_in"
        ):
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                counter = counters[self.argument][IO_COUNTER[self.type]]
                now = dt_util.utcnow()
                if self._last_value and self._last_value < counter:
                    self._state = round(
                        (counter - self._last_value)
                        / 1000 ** 2
                        / (now - self._last_update_time).seconds,
                        3,
                    )
                else:
                    self._state = None
                self._last_update_time = now
                self._last_value = counter
            else:
                self._state = None
        elif self.type == "ipv4_address" or self.type == "ipv6_address":
            addresses = psutil.net_if_addrs()
            if self.argument in addresses:
                for addr in addresses[self.argument]:
                    if addr.family == IF_ADDRS_FAMILY[self.type]:
                        self._state = addr.address
            else:
                self._state = None
        elif self.type == "last_boot":
            self._state = dt_util.as_local(
                dt_util.utc_from_timestamp(psutil.boot_time())
            ).isoformat()
        elif self.type == "load_1m":
            self._state = round(os.getloadavg()[0], 2)
        elif self.type == "load_5m":
            self._state = round(os.getloadavg()[1], 2)
        elif self.type == "load_15m":
            self._state = round(os.getloadavg()[2], 2)

    @staticmethod
    def read_cpu_temperature():
        """Attempt to read CPU / processor temperature."""
        temps = psutil.sensors_temperatures()

        for name, entries in temps.items():
            i = 1
            for entry in entries:
                # In case the label is empty (e.g. on Raspberry PI 4),
                # construct it ourself here based on the sensor key name.
                if not entry.label:
                    _label = f"{name} {i}"
                else:
                    _label = entry.label

                if _label in CPU_SENSOR_PREFIXES:
                    return round(entry.current, 1)

                i += 1
<commit_msg>Add systemmonitor check for mandatory "arg" of sensors (#39687)<commit_after>"""Support for monitoring the local system."""
import logging
import os
import socket
import sys

import psutil
import voluptuous as vol

from homeassistant.components.sensor import PLATFORM_SCHEMA
from homeassistant.const import (
    CONF_RESOURCES,
    CONF_TYPE,
    DATA_GIBIBYTES,
    DATA_MEBIBYTES,
    DATA_RATE_MEGABYTES_PER_SECOND,
    PERCENTAGE,
    STATE_OFF,
    STATE_ON,
    TEMP_CELSIUS,
)
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity import Entity
from homeassistant.util import slugify
import homeassistant.util.dt as dt_util

# mypy: allow-untyped-defs, no-check-untyped-defs

_LOGGER = logging.getLogger(__name__)

CONF_ARG = "arg"

if sys.maxsize > 2 ** 32:
    CPU_ICON = "mdi:cpu-64-bit"
else:
    CPU_ICON = "mdi:cpu-32-bit"

# Schema: [name, unit of measurement, icon, device class, flag if mandatory arg]
SENSOR_TYPES = {
    "disk_free": ["Disk free", DATA_GIBIBYTES, "mdi:harddisk", None, False],
    "disk_use": ["Disk use", DATA_GIBIBYTES, "mdi:harddisk", None, False],
    "disk_use_percent": [
        "Disk use (percent)",
        PERCENTAGE,
        "mdi:harddisk",
        None,
        False,
    ],
    "ipv4_address": ["IPv4 address", "", "mdi:server-network", None, True],
    "ipv6_address": ["IPv6 address", "", "mdi:server-network", None, True],
    "last_boot": ["Last boot", "", "mdi:clock", "timestamp", False],
    "load_15m": ["Load (15m)", " ", CPU_ICON, None, False],
    "load_1m": ["Load (1m)", " ", CPU_ICON, None, False],
    "load_5m": ["Load (5m)", " ", CPU_ICON, None, False],
    "memory_free": ["Memory free", DATA_MEBIBYTES, "mdi:memory", None, False],
    "memory_use": ["Memory use", DATA_MEBIBYTES, "mdi:memory", None, False],
    "memory_use_percent": [
        "Memory use (percent)",
        PERCENTAGE,
        "mdi:memory",
        None,
        False,
    ],
    "network_in": ["Network in", DATA_MEBIBYTES, "mdi:server-network", None, True],
    "network_out": ["Network out", DATA_MEBIBYTES, "mdi:server-network", None, True],
    "packets_in": ["Packets in", " ", "mdi:server-network", None, True],
    "packets_out": ["Packets out", " ", "mdi:server-network", None, True],
    "throughput_network_in": [
        "Network throughput in",
        DATA_RATE_MEGABYTES_PER_SECOND,
        "mdi:server-network",
        None,
        True,
    ],
    "throughput_network_out": [
        "Network throughput out",
        DATA_RATE_MEGABYTES_PER_SECOND,
        "mdi:server-network",
        True,
    ],
    "process": ["Process", " ", CPU_ICON, None, True],
    "processor_use": ["Processor use", PERCENTAGE, CPU_ICON, None, False],
    "processor_temperature": [
        "Processor temperature",
        TEMP_CELSIUS,
        CPU_ICON,
        None,
        False,
    ],
    "swap_free": ["Swap free", DATA_MEBIBYTES, "mdi:harddisk", None, True],
    "swap_use": ["Swap use", DATA_MEBIBYTES, "mdi:harddisk", None, False],
    "swap_use_percent": ["Swap use (percent)", PERCENTAGE, "mdi:harddisk", None, False],
}

def check_required_arg(value):
    """Validate that the required "arg" for the sensor types that need it are set."""
    for sensor in value:
        sensor_type = sensor[CONF_TYPE]
        sensor_arg = sensor.get(CONF_ARG)

        if sensor_arg is None and SENSOR_TYPES[sensor_type][4]:
            raise vol.RequiredFieldInvalid(
                f"Mandatory 'arg' is missing for sensor type '{sensor_type}'."
            )

    return value

PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
    {
        vol.Optional(CONF_RESOURCES, default={CONF_TYPE: "disk_use"}): vol.All(
            cv.ensure_list,
            [
                vol.Schema(
                    {
                        vol.Required(CONF_TYPE): vol.In(SENSOR_TYPES),
                        vol.Optional(CONF_ARG): cv.string,
                    }
                )
            ],
            check_required_arg,
        )
    }
)

IO_COUNTER = {
    "network_out": 0,
    "network_in": 1,
    "packets_out": 2,
    "packets_in": 3,
    "throughput_network_out": 0,
    "throughput_network_in": 1,
}

IF_ADDRS_FAMILY = {"ipv4_address": socket.AF_INET, "ipv6_address": socket.AF_INET6}

# There might be additional keys to be added for different
# platforms / hardware combinations.
# Taken from last version of "glances" integration before they moved to
# a generic temperature sensor logic.
# https://github.com/home-assistant/core/blob/5e15675593ba94a2c11f9f929cdad317e27ce190/homeassistant/components/glances/sensor.py#L199
CPU_SENSOR_PREFIXES = [
    "amdgpu 1",
    "aml_thermal",
    "Core 0",
    "Core 1",
    "CPU Temperature",
    "CPU",
    "cpu-thermal 1",
    "cpu_thermal 1",
    "exynos-therm 1",
    "Package id 0",
    "Physical id 0",
    "radeon 1",
    "soc-thermal 1",
    "soc_thermal 1",
]

def setup_platform(hass, config, add_entities, discovery_info=None):
    """Set up the system monitor sensors."""
    dev = []
    for resource in config[CONF_RESOURCES]:
        # Initialize the sensor argument if none was provided.
        # For disk monitoring default to "/" (root) to prevent runtime errors, if argument was not specified.
        if CONF_ARG not in resource:
            if resource[CONF_TYPE].startswith("disk_"):
                resource[CONF_ARG] = "/"
            else:
                resource[CONF_ARG] = ""

        # Verify if we can retrieve CPU / processor temperatures.
        # If not, do not create the entity and add a warning to the log
        if resource[CONF_TYPE] == "processor_temperature":
            if SystemMonitorSensor.read_cpu_temperature() is None:
                _LOGGER.warning("Cannot read CPU / processor temperature information.")
                continue

        dev.append(SystemMonitorSensor(resource[CONF_TYPE], resource[CONF_ARG]))

    add_entities(dev, True)

class SystemMonitorSensor(Entity):
    """Implementation of a system monitor sensor."""

    def __init__(self, sensor_type, argument=""):
        """Initialize the sensor."""
        self._name = "{} {}".format(SENSOR_TYPES[sensor_type][0], argument)
        self._unique_id = slugify(f"{sensor_type}_{argument}")
        self.argument = argument
        self.type = sensor_type
        self._state = None
        self._unit_of_measurement = SENSOR_TYPES[sensor_type][1]
        self._available = True
        if sensor_type in ["throughput_network_out", "throughput_network_in"]:
            self._last_value = None
            self._last_update_time = None

    @property
    def name(self):
        """Return the name of the sensor."""
        return self._name.rstrip()

    @property
    def unique_id(self):
        """Return the unique ID."""
        return self._unique_id

    @property
    def device_class(self):
        """Return the class of this sensor."""
        return SENSOR_TYPES[self.type][3]

    @property
    def icon(self):
        """Icon to use in the frontend, if any."""
        return SENSOR_TYPES[self.type][2]

    @property
    def state(self):
        """Return the state of the device."""
        return self._state

    @property
    def unit_of_measurement(self):
        """Return the unit of measurement of this entity, if any."""
        return self._unit_of_measurement

    @property
    def available(self):
        """Return True if entity is available."""
        return self._available

    def update(self):
        """Get the latest system information."""
        if self.type == "disk_use_percent":
            self._state = psutil.disk_usage(self.argument).percent
        elif self.type == "disk_use":
            self._state = round(psutil.disk_usage(self.argument).used / 1024 ** 3, 1)
        elif self.type == "disk_free":
            self._state = round(psutil.disk_usage(self.argument).free / 1024 ** 3, 1)
        elif self.type == "memory_use_percent":
            self._state = psutil.virtual_memory().percent
        elif self.type == "memory_use":
            virtual_memory = psutil.virtual_memory()
            self._state = round(
                (virtual_memory.total - virtual_memory.available) / 1024 ** 2, 1
            )
        elif self.type == "memory_free":
            self._state = round(psutil.virtual_memory().available / 1024 ** 2, 1)
        elif self.type == "swap_use_percent":
            self._state = psutil.swap_memory().percent
        elif self.type == "swap_use":
            self._state = round(psutil.swap_memory().used / 1024 ** 2, 1)
        elif self.type == "swap_free":
            self._state = round(psutil.swap_memory().free / 1024 ** 2, 1)
        elif self.type == "processor_use":
            self._state = round(psutil.cpu_percent(interval=None))
        elif self.type == "processor_temperature":
            self._state = self.read_cpu_temperature()
        elif self.type == "process":
            for proc in psutil.process_iter():
                try:
                    if self.argument == proc.name():
                        self._state = STATE_ON
                        return
                except psutil.NoSuchProcess as err:
                    _LOGGER.warning(
                        "Failed to load process with id: %s, old name: %s",
                        err.pid,
                        err.name,
                    )
            self._state = STATE_OFF
        elif self.type == "network_out" or self.type == "network_in":
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                counter = counters[self.argument][IO_COUNTER[self.type]]
                self._state = round(counter / 1024 ** 2, 1)
            else:
                self._state = None
        elif self.type == "packets_out" or self.type == "packets_in":
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                self._state = counters[self.argument][IO_COUNTER[self.type]]
            else:
                self._state = None
        elif (
            self.type == "throughput_network_out"
            or self.type == "throughput_network_in"
        ):
            counters = psutil.net_io_counters(pernic=True)
            if self.argument in counters:
                counter = counters[self.argument][IO_COUNTER[self.type]]
                now = dt_util.utcnow()
                if self._last_value and self._last_value < counter:
                    self._state = round(
                        (counter - self._last_value)
                        / 1000 ** 2
                        / (now - self._last_update_time).seconds,
                        3,
                    )
                else:
                    self._state = None
                self._last_update_time = now
                self._last_value = counter
            else:
                self._state = None
        elif self.type == "ipv4_address" or self.type == "ipv6_address":
            addresses = psutil.net_if_addrs()
            if self.argument in addresses:
                for addr in addresses[self.argument]:
                    if addr.family == IF_ADDRS_FAMILY[self.type]:
                        self._state = addr.address
            else:
                self._state = None
        elif self.type == "last_boot":
            self._state = dt_util.as_local(
                dt_util.utc_from_timestamp(psutil.boot_time())
            ).isoformat()
        elif self.type == "load_1m":
            self._state = round(os.getloadavg()[0], 2)
        elif self.type == "load_5m":
            self._state = round(os.getloadavg()[1], 2)
        elif self.type == "load_15m":
            self._state = round(os.getloadavg()[2], 2)

    @staticmethod
    def read_cpu_temperature():
        """Attempt to read CPU / processor temperature."""
        temps = psutil.sensors_temperatures()

        for name, entries in temps.items():
            i = 1
            for entry in entries:
                # In case the label is empty (e.g. on Raspberry PI 4),
                # construct it ourself here based on the sensor key name.
                if not entry.label:
                    _label = f"{name} {i}"
                else:
                    _label = entry.label

                if _label in CPU_SENSOR_PREFIXES:
                    return round(entry.current, 1)

                i += 1
<|endoftext|>