<commit_before>import collections
from typing import Any, List, Optional

class MugenList(list):
    """
    A subclass of list with extra functionality, used internally
    """

    def __init__(self, items: Optional[List[Any]] = None):
        if items is None:
            items = []

        super().__init__(items)

    def __add__(self, rhs):
        return type(self)((super().__add__(rhs)))

    def __getitem__(self, item):
        result = list.__getitem__(self, item)
        if isinstance(item, slice):
            return type(self)(result)
        else:
            return result

    def pretty_repr(self, item_reprs: List[str] = None):
        if item_reprs is None:
            item_reprs = [repr(item) for item in self]

        repr_str = ""
        for index, element_repr in enumerate(item_reprs):
            if index != 0:
                repr_str += " "
            repr_str += element_repr
            if index != len(self) - 1:
                repr_str += ", \n"
        return f"[{repr_str}]"

    def lget(self, attr) -> List[Any]:
        """
        Returns
        -------
        A list of values for the attribute, taken from each element in the array
        """
        return [getattr(elem, attr) for elem in self]

    def flatten(self):
        """
        Returns
        -------
        A new, flattened version of this list.
        Flattens an arbitrarily nested list of objects
        """
        return type(self)(flatten(self))

def flatten(list: List[Any]) -> List[Any]:
    """
    Flattens an arbitrarily nested irregular list of objects
    """
    list_flattened = []

    for element in list:
        if isinstance(element, collections.Iterable) and not isinstance(
            element, (str, bytes)
        ):
            list_flattened.extend(flatten(element))
        else:
            list_flattened.append(element)

    return list_flattened
<commit_msg>Import ABC from collections.abc for Python 3.10 compatibility.<commit_after>import collections.abc
from typing import Any, List, Optional

class MugenList(list):
    """
    A subclass of list with extra functionality, used internally
    """

    def __init__(self, items: Optional[List[Any]] = None):
        if items is None:
            items = []

        super().__init__(items)

    def __add__(self, rhs):
        return type(self)((super().__add__(rhs)))

    def __getitem__(self, item):
        result = list.__getitem__(self, item)
        if isinstance(item, slice):
            return type(self)(result)
        else:
            return result

    def pretty_repr(self, item_reprs: List[str] = None):
        if item_reprs is None:
            item_reprs = [repr(item) for item in self]

        repr_str = ""
        for index, element_repr in enumerate(item_reprs):
            if index != 0:
                repr_str += " "
            repr_str += element_repr
            if index != len(self) - 1:
                repr_str += ", \n"
        return f"[{repr_str}]"

    def lget(self, attr) -> List[Any]:
        """
        Returns
        -------
        A list of values for the attribute, taken from each element in the array
        """
        return [getattr(elem, attr) for elem in self]

    def flatten(self):
        """
        Returns
        -------
        A new, flattened version of this list.
        Flattens an arbitrarily nested list of objects
        """
        return type(self)(flatten(self))

def flatten(list: List[Any]) -> List[Any]:
    """
    Flattens an arbitrarily nested irregular list of objects
    """
    list_flattened = []

    for element in list:
        if isinstance(element, collections.abc.Iterable) and not isinstance(
            element, (str, bytes)
        ):
            list_flattened.extend(flatten(element))
        else:
            list_flattened.append(element)

    return list_flattened
<|endoftext|>


<commit_before>"""Portal module"""
from logging import handlers
import logging
import os
import pkginfo
import sys
import requests_cache
from flask import Flask
import redis
from werkzeug.contrib.profiler import ProfilerMiddleware

from .audit import configure_audit_log
from .config import DefaultConfig
from .csrf import csrf, csrf_blueprint
from .database import db
from .dogpile import dogpile_cache
from .extensions import authomatic, recaptcha
from .extensions import babel, celery, mail, oauth, session, user_manager
from .models.app_text import app_text
from .models.coredata import configure_coredata
from .models.role import ROLE
from .views.assessment_engine import assessment_engine_api
from .views.audit import audit_api
from .views.auth import auth, capture_next_view_function
from .views.coredata import coredata_api
from .views.clinical import clinical_api
from .views.demographics import demographics_api
from .views.extend_flask_user import reset_password_view_function
from .views.fhir import fhir_api
from .views.filters import filters_blueprint
from .views.group import group_api
from .views.intervention import intervention_api
from .views.patient import patient_api
from .views.patients import patients
from .views.procedure import procedure_api
from .views.portal import portal, page_not_found, server_error
from .views.organization import org_api
from .views.scheduled_job import scheduled_job_api
from .views.tou import tou_api
from .views.truenth import truenth_api
from .views.user import user_api

SITE_CFG = 'site.cfg'
DEFAULT_BLUEPRINTS = (
    assessment_engine_api,
    audit_api,
    auth,
    coredata_api,
    clinical_api,
    csrf_blueprint,
    demographics_api,
    fhir_api,
    filters_blueprint,
    group_api,
    intervention_api,
    org_api,
    patient_api,
    patients,
    procedure_api,
    portal,
    scheduled_job_api,
    truenth_api,
    tou_api,
    user_api,)

def create_app(config=None, app_name=None, blueprints=None):
    """Returns the configured flask app"""
    if app_name is None:
        app_name = DefaultConfig.PROJECT
    if blueprints is None:
        blueprints = DEFAULT_BLUEPRINTS

    app = Flask(app_name, template_folder='templates',
                instance_relative_config=True)
    configure_app(app, config)
    configure_profiler(app)
    configure_csrf(app)
    configure_dogpile(app)
    configure_jinja(app)
    configure_error_handlers(app)
    configure_extensions(app)
    configure_blueprints(app, blueprints=DEFAULT_BLUEPRINTS)
    configure_logging(app)
    configure_audit_log(app)
    configure_metadata(app)
    configure_coredata(app)
    configure_cache(app)
    return app

def configure_app(app, config):
    """Load successive configs - overriding defaults"""
    app.config.from_object(DefaultConfig)
    app.config.from_pyfile(SITE_CFG, silent=True)
    app.config.from_pyfile('application.cfg', silent=True)

    if config:
        app.config.from_object(config)

def configure_profiler(app):
    if app.config.get('PROFILE'):
        app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])

def configure_csrf(app):
    """Initialize CSRF protection

    See `csrf.csrf_protect()` for implementation.  Not using default
    as OAuth API use needs exclusion.

    """
    csrf.init_app(app)

def configure_dogpile(app):
    """Initialize dogpile cache with config values"""

    # Bootstrap challenges with config values dependent on other
    # configuration values, which may be set in different
    # order depending on environment.

    # Regardless of configuration location, this should now be set.
    app.config['DOGPILE_CACHE_URLS'] = app.config['REDIS_URL']
    dogpile_cache.init_app(app)

def configure_jinja(app):
    app.jinja_env.globals.update(app_text=app_text)
    app.jinja_env.globals.update(ROLE=ROLE)

def configure_error_handlers(app):
    if not app.debug:
        app.register_error_handler(404, page_not_found)
        app.register_error_handler(500, server_error)

def configure_extensions(app):
    """Bind extensions to application"""
    # flask-sqlalchemy - the ORM / DB used
    db.init_app(app)
    if app.testing:
        session_options = {}
        db.session_options = session_options

    # flask-user

    ## The default login and register view functions fail to capture
    ## the next parameter in a reliable fashion.  Using a simple closure
    ## capture 'next' before redirecting to the real view function to
    ## manage the flask-user business logic

    from flask_user.views import login, register
    user_manager.init_app(
        app,
        reset_password_view_function=reset_password_view_function,
        register_view_function=capture_next_view_function(register),
        login_view_function=capture_next_view_function(login))

    # authomatic - OAuth lib between Portal and other external IdPs
    authomatic.init_app(app)

    # flask-oauthlib - OAuth between Portal and Interventions
    oauth.init_app(app)

    # flask-mail - Email communication
    mail.init_app(app)

    # flask-session - Server side sessions
    session.init_app(app)

    # celery - task queue for asynchronous tasks
    celery.init_app(app)

    # babel - i18n
    babel.init_app(app)

    # recaptcha - form verification
    recaptcha.init_app(app)

def configure_blueprints(app, blueprints):
    """Register blueprints with application"""
    for blueprint in blueprints:
        app.register_blueprint(blueprint)

def configure_logging(app):  # pragma: no cover
    """Configure logging."""
    if app.config.get('LOG_SQL'):
        sql_log_file = '/tmp/sql_log'
        sql_file_handler = handlers.RotatingFileHandler(sql_log_file,
                maxBytes=1000000, backupCount=20)
        sql_file_handler.setFormatter(logging.Formatter(
            '%(asctime)s %(thread)d: %(message)s'
        ))
        sql_log = logging.getLogger('sqlalchemy.engine')
        sql_log.setLevel(logging.INFO)
        sql_log.addHandler(sql_file_handler)

    if app.testing or not app.config.get('LOG_FOLDER', None):
        # Write logs to stdout by default and when testing
        return

    if not os.path.exists(app.config['LOG_FOLDER']):
        os.mkdir(app.config['LOG_FOLDER'])

    level = getattr(logging, app.config['LOG_LEVEL'].upper())

    info_log = os.path.join(app.config['LOG_FOLDER'], 'info.log')
    # For WSGI servers, the log file is only writable by www-data
    # This prevents users from being able to run other management
    # commands as themselves.  If current user can't write to the
    # info_log, bail out - relying on stdout/stderr
    try:
        with open(info_log, 'a+'):
            pass
    except IOError:
        print >> sys.stderr, "Can't open log file '%s', use stdout" %\
            info_log
        print >> sys.stderr,\
            "Set LOG_FOLDER to a writable directory in configuration file"
        return

    info_file_handler = handlers.RotatingFileHandler(info_log,
            maxBytes=1000000, backupCount=20)
    info_file_handler.setLevel(level)
    info_file_handler.setFormatter(logging.Formatter(
        '%(asctime)s %(levelname)s: %(message)s '
        '[in %(pathname)s:%(lineno)d]')
    )

    app.logger.setLevel(level)
    app.logger.addHandler(info_file_handler)

    # OAuth library logging tends to be helpful for connection
    # debugging
    for logger in ('oauthlib', 'flask_oauthlib'):
        log = logging.getLogger(logger)
        log.setLevel(level)
        log.addHandler(info_file_handler)

    from .tasks import logger as task_logger
    task_logger.setLevel(level)
    task_logger.addHandler(info_file_handler)

    # Configure Error Emails for high level log messages, only in prod mode
    ADMINS = app.config['ERROR_SENDTO_EMAIL']
    if not app.debug:
        mail_handler = handlers.SMTPHandler(
            '127.0.0.1',
            app.config['MAIL_DEFAULT_SENDER'],
            ADMINS,
            '{} Log Message'.format(app.config['SERVER_NAME']))
        mail_handler.setLevel(logging.ERROR)
        app.logger.addHandler(mail_handler)
        task_logger.addHandler(mail_handler)

    #app.logger.debug("initiate logging done at level %s, %d",
    #    app.config['LOG_LEVEL'], level)

def configure_metadata(app):
    """Add distribution metadata for display in templates"""
    metadata = pkginfo.Develop(os.path.join(app.root_path, ".."))

    # Get git hash from version if present
    # Todo: extend Develop instead of monkey patching
    if metadata.version and '+ng' in metadata.version:
        metadata.git_hash = metadata.version.split('+ng')[-1].split('.')[0]

    app.config.metadata = metadata

def configure_cache(app):
    """Configure requests-cache"""
    REQUEST_CACHE_URL = app.config.get("REQUEST_CACHE_URL")

    requests_cache.install_cache(
        cache_name=app.name,
        backend='redis',
        expire_after=1000,
        include_get_headers=True,
        old_data_on_error=True,
        connection=redis.StrictRedis.from_url(REQUEST_CACHE_URL),
    )
<commit_msg>Fix version generation when portal module installed outside git checkout<commit_after>"""Portal module"""
from logging import handlers
import logging
import os
import pkginfo
import sys
import requests_cache
from flask import Flask
import redis
from werkzeug.contrib.profiler import ProfilerMiddleware

from .audit import configure_audit_log
from .config import DefaultConfig
from .csrf import csrf, csrf_blueprint
from .database import db
from .dogpile import dogpile_cache
from .extensions import authomatic, recaptcha
from .extensions import babel, celery, mail, oauth, session, user_manager
from .models.app_text import app_text
from .models.coredata import configure_coredata
from .models.role import ROLE
from .views.assessment_engine import assessment_engine_api
from .views.audit import audit_api
from .views.auth import auth, capture_next_view_function
from .views.coredata import coredata_api
from .views.clinical import clinical_api
from .views.demographics import demographics_api
from .views.extend_flask_user import reset_password_view_function
from .views.fhir import fhir_api
from .views.filters import filters_blueprint
from .views.group import group_api
from .views.intervention import intervention_api
from .views.patient import patient_api
from .views.patients import patients
from .views.procedure import procedure_api
from .views.portal import portal, page_not_found, server_error
from .views.organization import org_api
from .views.scheduled_job import scheduled_job_api
from .views.tou import tou_api
from .views.truenth import truenth_api
from .views.user import user_api

SITE_CFG = 'site.cfg'
DEFAULT_BLUEPRINTS = (
    assessment_engine_api,
    audit_api,
    auth,
    coredata_api,
    clinical_api,
    csrf_blueprint,
    demographics_api,
    fhir_api,
    filters_blueprint,
    group_api,
    intervention_api,
    org_api,
    patient_api,
    patients,
    procedure_api,
    portal,
    scheduled_job_api,
    truenth_api,
    tou_api,
    user_api,)

def create_app(config=None, app_name=None, blueprints=None):
    """Returns the configured flask app"""
    if app_name is None:
        app_name = DefaultConfig.PROJECT
    if blueprints is None:
        blueprints = DEFAULT_BLUEPRINTS

    app = Flask(app_name, template_folder='templates',
                instance_relative_config=True)
    configure_app(app, config)
    configure_profiler(app)
    configure_csrf(app)
    configure_dogpile(app)
    configure_jinja(app)
    configure_error_handlers(app)
    configure_extensions(app)
    configure_blueprints(app, blueprints=DEFAULT_BLUEPRINTS)
    configure_logging(app)
    configure_audit_log(app)
    configure_metadata(app)
    configure_coredata(app)
    configure_cache(app)
    return app

def configure_app(app, config):
    """Load successive configs - overriding defaults"""
    app.config.from_object(DefaultConfig)
    app.config.from_pyfile(SITE_CFG, silent=True)
    app.config.from_pyfile('application.cfg', silent=True)

    if config:
        app.config.from_object(config)

def configure_profiler(app):
    if app.config.get('PROFILE'):
        app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[30])

def configure_csrf(app):
    """Initialize CSRF protection

    See `csrf.csrf_protect()` for implementation.  Not using default
    as OAuth API use needs exclusion.

    """
    csrf.init_app(app)

def configure_dogpile(app):
    """Initialize dogpile cache with config values"""

    # Bootstrap challenges with config values dependent on other
    # configuration values, which may be set in different
    # order depending on environment.

    # Regardless of configuration location, this should now be set.
    app.config['DOGPILE_CACHE_URLS'] = app.config['REDIS_URL']
    dogpile_cache.init_app(app)

def configure_jinja(app):
    app.jinja_env.globals.update(app_text=app_text)
    app.jinja_env.globals.update(ROLE=ROLE)

def configure_error_handlers(app):
    if not app.debug:
        app.register_error_handler(404, page_not_found)
        app.register_error_handler(500, server_error)

def configure_extensions(app):
    """Bind extensions to application"""
    # flask-sqlalchemy - the ORM / DB used
    db.init_app(app)
    if app.testing:
        session_options = {}
        db.session_options = session_options

    # flask-user

    ## The default login and register view functions fail to capture
    ## the next parameter in a reliable fashion.  Using a simple closure
    ## capture 'next' before redirecting to the real view function to
    ## manage the flask-user business logic

    from flask_user.views import login, register
    user_manager.init_app(
        app,
        reset_password_view_function=reset_password_view_function,
        register_view_function=capture_next_view_function(register),
        login_view_function=capture_next_view_function(login))

    # authomatic - OAuth lib between Portal and other external IdPs
    authomatic.init_app(app)

    # flask-oauthlib - OAuth between Portal and Interventions
    oauth.init_app(app)

    # flask-mail - Email communication
    mail.init_app(app)

    # flask-session - Server side sessions
    session.init_app(app)

    # celery - task queue for asynchronous tasks
    celery.init_app(app)

    # babel - i18n
    babel.init_app(app)

    # recaptcha - form verification
    recaptcha.init_app(app)

def configure_blueprints(app, blueprints):
    """Register blueprints with application"""
    for blueprint in blueprints:
        app.register_blueprint(blueprint)

def configure_logging(app):  # pragma: no cover
    """Configure logging."""
    if app.config.get('LOG_SQL'):
        sql_log_file = '/tmp/sql_log'
        sql_file_handler = handlers.RotatingFileHandler(sql_log_file,
                maxBytes=1000000, backupCount=20)
        sql_file_handler.setFormatter(logging.Formatter(
            '%(asctime)s %(thread)d: %(message)s'
        ))
        sql_log = logging.getLogger('sqlalchemy.engine')
        sql_log.setLevel(logging.INFO)
        sql_log.addHandler(sql_file_handler)

    if app.testing or not app.config.get('LOG_FOLDER', None):
        # Write logs to stdout by default and when testing
        return

    if not os.path.exists(app.config['LOG_FOLDER']):
        os.mkdir(app.config['LOG_FOLDER'])

    level = getattr(logging, app.config['LOG_LEVEL'].upper())

    info_log = os.path.join(app.config['LOG_FOLDER'], 'info.log')
    # For WSGI servers, the log file is only writable by www-data
    # This prevents users from being able to run other management
    # commands as themselves.  If current user can't write to the
    # info_log, bail out - relying on stdout/stderr
    try:
        with open(info_log, 'a+'):
            pass
    except IOError:
        print >> sys.stderr, "Can't open log file '%s', use stdout" %\
            info_log
        print >> sys.stderr,\
            "Set LOG_FOLDER to a writable directory in configuration file"
        return

    info_file_handler = handlers.RotatingFileHandler(info_log,
            maxBytes=1000000, backupCount=20)
    info_file_handler.setLevel(level)
    info_file_handler.setFormatter(logging.Formatter(
        '%(asctime)s %(levelname)s: %(message)s '
        '[in %(pathname)s:%(lineno)d]')
    )

    app.logger.setLevel(level)
    app.logger.addHandler(info_file_handler)

    # OAuth library logging tends to be helpful for connection
    # debugging
    for logger in ('oauthlib', 'flask_oauthlib'):
        log = logging.getLogger(logger)
        log.setLevel(level)
        log.addHandler(info_file_handler)

    from .tasks import logger as task_logger
    task_logger.setLevel(level)
    task_logger.addHandler(info_file_handler)

    # Configure Error Emails for high level log messages, only in prod mode
    ADMINS = app.config['ERROR_SENDTO_EMAIL']
    if not app.debug:
        mail_handler = handlers.SMTPHandler(
            '127.0.0.1',
            app.config['MAIL_DEFAULT_SENDER'],
            ADMINS,
            '{} Log Message'.format(app.config['SERVER_NAME']))
        mail_handler.setLevel(logging.ERROR)
        app.logger.addHandler(mail_handler)
        task_logger.addHandler(mail_handler)

    #app.logger.debug("initiate logging done at level %s, %d",
    #    app.config['LOG_LEVEL'], level)

def configure_metadata(app):
    """Add distribution metadata for display in templates"""
    metadata = pkginfo.Installed('portal')

    # Get git hash from version if present
    # Todo: extend Distribution base class instead of monkey patching
    if metadata.version and '+ng' in metadata.version:
        metadata.git_hash = metadata.version.split('+ng')[-1].split('.')[0]

    app.config.metadata = metadata

def configure_cache(app):
    """Configure requests-cache"""
    REQUEST_CACHE_URL = app.config.get("REQUEST_CACHE_URL")

    requests_cache.install_cache(
        cache_name=app.name,
        backend='redis',
        expire_after=1000,
        include_get_headers=True,
        old_data_on_error=True,
        connection=redis.StrictRedis.from_url(REQUEST_CACHE_URL),
    )
<|endoftext|>


<commit_before># -*- coding: utf8 -*-
import logging
import signal
import itertools
from Queue import Queue, Empty
from threading import Thread, current_thread
from pprint import pprint
from cStringIO import StringIO
from datetime import datetime

log = logging.getLogger(__name__)

class MultiWorker(object):
    def __init__(self, workers):
        self.workers_total = workers
        self._worker_threads = None

    def _setup(self):
        raise NotImplemented

    def _cleanup(self):
        raise NotImplemented

    def _process_item(self, item):
        raise NotImplemented

    def _item_generator(self):
        raise NotImplemented

    def _job_size(self):
        raise NotImplemented

    def _worker(self):
        my_worker_number = self._worker_threads.index(current_thread())

        while True:
            item = self._data_queue.get()
            log.debug('Worker %s: got item: %s.' % (my_worker_number, item))

            if item == 'QUEUE_DONE':
                log.debug('Worker %s done, exiting.' % my_worker_number)
                self._data_queue.task_done()
                break

            if self._need_to_stop():
                log.debug('Worker %s: need to stop, ignoring item.' % my_worker_number)
                self._data_queue.task_done()
                continue

            try:
                self._process_item(item)
                log.info('Worker %s: finished %s' % (my_worker_number, item))
            except Exception:
                self._had_errors = True
                itempp = StringIO()
                itempp.write('Worker %s: exception on item:\n' % my_worker_number)
                pprint(item, stream=itempp)
                log.exception(itempp.getvalue())

            log.debug('Worker %s: setting task done.' % my_worker_number)
            self._data_queue.task_done()
            processed = self._total_processed.next()
            log.debug('incremented total processed, total is %s' % processed)
            if processed % 10 == 0:
                time_elapsed = datetime.now() - self._start_time
                estimated = time_elapsed / processed * (self._job_size() - processed)
                log.info('Estimated time remaining to complete: %s' % estimated)

    def _setup_workers(self):
        if self._worker_threads is not None:
            return

        self._data_queue = Queue(maxsize=self.workers_total * 10)
        self._worker_threads = []

        log.debug('Setting up %s sync_data workers.' % self.workers_total)
        for i in xrange(self.workers_total):
            thread = Thread(target=self._worker)
            self._worker_threads.append(thread)
            thread.start()

    def _finish_workers(self, empty=False):
        if empty:
            log.debug('emptying queue.')
            while True:
                try:
                    item = self._data_queue.get(False)
                    log.debug('Removed item from queue without processing: %s' % item)
                    self._data_queue.task_done()
                except Empty:
                    break

        for i in xrange(len(self._worker_threads)):
            log.debug('sending queue done signal to worker %s' % i)
            self._data_queue.put('QUEUE_DONE')

        log.debug('blocking until pending finished')
        self._data_queue.join()
        log.debug('finished')
        self._worker_threads = None

    def _need_to_stop(self):
        return self._stop_all or (self._had_errors and self._stop_on_errors)

    def execute(self, stop_on_errors=True):
        self._setup_workers()

        if current_thread().getName() == 'MainThread':
            def rs(*args):
                log.critical('received interrupt, finalizing...')
                self._stop_all = True
            signal.signal(signal.SIGINT, rs)

        self._stop_all = False
        self._had_errors = False
        self._stop_on_errors = stop_on_errors
        self._start_time = datetime.now()
        self._total_processed = itertools.count(1)
        self._setup()

        try:
            for item in self._item_generator():
                if self._need_to_stop():
                    log.debug('stopped processing orders')
                    break

                self._data_queue.put(item)
                log.debug('added item %s to the queue' % item)

            if not self._need_to_stop():
                self._data_queue.join()

                total_processed = self._total_processed.next() - 1
                if total_processed < self._job_size():
                    raise Exception('total processed(%s) != expected job size(%s)' %
                        (total_processed, self._job_size()))

            self._finish_workers(empty=self._need_to_stop())
        except:
            self._finish_workers(empty=True)
            self._had_errors = True
            log.exception('Error')
            raise
        finally:
            self._cleanup()

            if self._had_errors:
                log.critical('Execution finished with errors.')

            if not self._need_to_stop():
                log.info('Execution finished successfully.')

        return not self._had_errors and not self._stop_all

from time import sleep
from random import randint

class ExampleProcessor(MultiWorker):
    def _setup(self):
        self._total_items_to_process = 500
        self._items_to_process = xrange(self._total_items_to_process)

    def _cleanup(self):
        print 'did some house cleaning.'

    def _process_item(self, item):
        # i have a 1 in 1000 chance of going wrong
        if randint(0, 1000) == 3:
            raise Exception('OMG!')

        print 'I did some processing on item: %s' % item
        sleep(.2)

    def _item_generator(self):
        return self._items_to_process

    def _job_size(self):
        return self._total_items_to_process

if __name__ == '__main__':
    import logging
    import sys

    format = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
    logging.basicConfig(level=logging.DEBUG, format=format)

    try:
        processor = ExampleProcessor(len(sys.argv) > 1 and int(sys.argv[1]) or 4)
        processor.execute()
    finally:
        logging.shutdown()
<commit_msg>moved time estimation to example<commit_after># -*- coding: utf8 -*-
import logging
import signal
import itertools
from Queue import Queue, Empty
from threading import Thread, current_thread
from pprint import pprint
from cStringIO import StringIO
from datetime import datetime

log = logging.getLogger(__name__)

class MultiWorker(object):
    def __init__(self, workers):
        self.workers_total = workers
        self._worker_threads = None

    def _setup(self):
        raise NotImplemented

    def _cleanup(self):
        raise NotImplemented

    def _process_item(self, item):
        raise NotImplemented

    def _item_generator(self):
        raise NotImplemented

    def _job_size(self):
        raise NotImplemented

    def _report_completed(self, total_processed):
        pass

    def _worker(self):
        my_worker_number = self._worker_threads.index(current_thread())

        while True:
            item = self._data_queue.get()
            log.debug('Worker %s: got item: %s.' % (my_worker_number, item))

            if item == 'QUEUE_DONE':
                log.debug('Worker %s done, exiting.' % my_worker_number)
                self._data_queue.task_done()
                break

            if self._need_to_stop():
                log.debug('Worker %s: need to stop, ignoring item.' % my_worker_number)
                self._data_queue.task_done()
                continue

            try:
                self._process_item(item)
                log.info('Worker %s: finished %s' % (my_worker_number, item))
            except Exception:
                self._had_errors = True
                itempp = StringIO()
                itempp.write('Worker %s: exception on item:\n' % my_worker_number)
                pprint(item, stream=itempp)
                log.exception(itempp.getvalue())

            log.debug('Worker %s: setting task done.' % my_worker_number)
            self._data_queue.task_done()
            processed = self._total_processed.next()
            log.debug('incremented total processed, total is %s' % processed)
            self._report_completed(processed)

    def _setup_workers(self):
        if self._worker_threads is not None:
            return

        self._data_queue = Queue(maxsize=self.workers_total * 10)
        self._worker_threads = []

        log.debug('Setting up %s sync_data workers.' % self.workers_total)
        for i in xrange(self.workers_total):
            thread = Thread(target=self._worker)
            self._worker_threads.append(thread)
            thread.start()

    def _finish_workers(self, empty=False):
        if empty:
            log.debug('emptying queue.')
            while True:
                try:
                    item = self._data_queue.get(False)
                    log.debug('Removed item from queue without processing: %s' % item)
                    self._data_queue.task_done()
                except Empty:
                    break

        for i in xrange(len(self._worker_threads)):
            log.debug('sending queue done signal to worker %s' % i)
            self._data_queue.put('QUEUE_DONE')

        log.debug('blocking until pending finished')
        self._data_queue.join()
        log.debug('finished')
        self._worker_threads = None

    def _need_to_stop(self):
        return self._stop_all or (self._had_errors and self._stop_on_errors)

    def execute(self, stop_on_errors=True):
        self._setup_workers()

        if current_thread().getName() == 'MainThread':
            def rs(*args):
                log.critical('received interrupt, finalizing...')
                self._stop_all = True
            signal.signal(signal.SIGINT, rs)

        self._stop_all = False
        self._had_errors = False
        self._stop_on_errors = stop_on_errors
        self._total_processed = itertools.count(1)
        self._setup()

        try:
            for item in self._item_generator():
                if self._need_to_stop():
                    log.debug('stopped processing orders')
                    break

                self._data_queue.put(item)
                log.debug('added item %s to the queue' % item)

            if not self._need_to_stop():
                self._data_queue.join()

                total_processed = self._total_processed.next() - 1
                if total_processed < self._job_size():
                    raise Exception('total processed(%s) != expected job size(%s)' %
                        (total_processed, self._job_size()))

            self._finish_workers(empty=self._need_to_stop())
        except:
            self._finish_workers(empty=True)
            self._had_errors = True
            log.exception('Error')
            raise
        finally:
            self._cleanup()

            if self._had_errors:
                log.critical('Execution finished with errors.')

            if not self._need_to_stop():
                log.info('Execution finished successfully.')

        return not self._had_errors and not self._stop_all

from time import sleep
from random import randint

# Example with estimated time to complete
class ExampleProcessor(MultiWorker):
    def _setup(self):
        self._start_time = datetime.now()
        self._last_report = None
        self._total_items_to_process = 500
        self._items_to_process = xrange(self._total_items_to_process)
        print 'started things up.'

    def _cleanup(self):
        print 'did some house cleaning.'

    def _process_item(self, item):
        # i have a 1 in 1000 chance of going wrong
        if randint(0, 1000) == 3:
            raise Exception('OMG!')

        print 'I did some processing on item: %s' % item
        sleep(.2)

    def _item_generator(self):
        return self._items_to_process

    def _job_size(self):
        return self._total_items_to_process

    def _report_completed(self, total_processed):
        # report estimated time every 3 seconds
        time_elapsed = datetime.now() - self._start_time
        if self._last_report is None:
            self._last_report = time_elapsed.seconds
            return

        if time_elapsed.seconds % 3 == 0 and time_elapsed.seconds != self._last_report:
            self._last_report = time_elapsed.seconds
            estimated = time_elapsed / total_processed * (self._job_size() - total_processed)
            print 'Estimated time remaining to complete: %s' % estimated

if __name__ == '__main__':
    import logging
    import sys

    format = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
    logging.basicConfig(level=logging.DEBUG, format=format)

    try:
        processor = ExampleProcessor(len(sys.argv) > 1 and int(sys.argv[1]) or 4)
        processor.execute()
    finally:
        logging.shutdown()
<|endoftext|>


<commit_before>"""
Old JobManager / container module. Likely to go away since using database for jobs.
"""

from cloudscheduler.job import Job

# We need to decide on logic for deciding if a job has been scheduled or not yet
# Unsure if we can rely on the metadata from the job but right now it just defaults to not scheduled

class JobManager:

    """
    Job Manager Class to assist with scheduling jobs.
    """
    def __init__(self):
        """
        Init for job manager.
        """

        self.unscheduled_jobs = {}
        self.unscheduled_jobs_by_user = {}
        self.scheduled_jobs = {}
        self.scheduled_jobs_by_user = {}

        self.user_list = list()

    # accepts a group of job dictionaries and shuffles them into the existing
    #  dictionaries based on their GlobalJobIds.
    def update_jobs(self, jobs):
        """
        update job states.
        :param jobs: list of jobs from condor?
        """
        redis_key_list = []
        for job in jobs:
            j = Job(**job)
            redis_key_list.append(j.GlobalJobId)
            if j.GlobalJobId in self.unscheduled_jobs:
                #update unscheduled entry
                self.unscheduled_jobs[j.GlobalJobId] = j
                self.unscheduled_jobs_by_user[j.User][j.GlobalJobId] = j

            elif j.GlobalJobId in self.scheduled_jobs:
                #update scheduled entry
                j.set_state(1)
                self.scheduled_jobs[j.GlobalJobId] = j
                self.scheduled_jobs_by_user[j.User][j.GlobalJobId] = j

            else:
                #brand new job, insert into unscheduled dicts
                self.unscheduled_jobs[j.GlobalJobId] = j
                self.unscheduled_jobs_by_user[j.User][j.GlobalJobId] = j
        # Clean up jobs that were not in the redis store
        for key in self.scheduled_jobs:
            if key not in redis_key_list:
                del self.scheduled_jobs[key]
                del self.scheduled_jobs_by_user[key]
        for key in self.unscheduled_jobs:
            if key not in redis_key_list:
                del self.unscheduled_jobs[key]
                del self.unscheduled_jobs_by_user[key]

    def schedule_job(self, jobid):
        """
        Mark a job as scheduled.
        :param jobid: id of job to mark
        """
        self.unscheduled_jobs[jobid].set_state(1)
        job = self.unscheduled_jobs[jobid]
        self.scheduled_jobs[jobid] = job
        self.scheduled_jobs_by_user[job.User][jobid] = job
        del self.unscheduled_jobs[jobid]
        del self.unscheduled_jobs_by_user[job.User][jobid]

    def unschedule_job(self, jobid):
        """
        Unmark a previously scheduled job.
        :param jobid: id of job to unmark
        """
        self.scheduled_jobs[jobid].set_state(0)
        job = self.scheduled_jobs[jobid]
        self.unscheduled_jobs[jobid] = job
        self.unscheduled_jobs_by_user[job.User][jobid] = job
        del self.scheduled_jobs[jobid]
        del self.scheduled_jobs_by_user[job.User][jobid]

    def get_jobs_user(self, user):
        """
        Fetch the jobs for specified user.
        :param user: user to get jobs for.
        :return: list of jobs belonging to user or None.
        """
        unsched = self.unscheduled_jobs_by_user[user]
        sched = self.scheduled_jobs_by_user[user]
        # python 3 expression for merging 2 dicts
        user_jobs = {**unsched, **sched}
        return user_jobs

    def get_scheduled_jobs_user(self, user):
        """
        Get the scheduled jobs for user.
        :param user: user name
        :return: job dict
        """
        return self.scheduled_jobs_by_user[user]

    def get_unscheduled_jobs_user(self, user):
        """
        Get the unscheduled jobs for a specific user.
        :param user: user name to get jobs for.
        :return: job dict
        """
        return self.unscheduled_jobs_by_user[user]

    def get_unscheduled_jobs(self):
        """
        Get the unscheduled jobs.
        :return:
        """
        return self.unscheduled_jobs

    def get_scheduled_jobs(self):
        """
        Get the scheduled jobs dictionary.
        :return:
        """
        return self.scheduled_jobs

    def get_all_jobs(self):
        """
        Return a dict containing all the jobs in system.
        :return: dict of jobs.
        """
        unsched = self.unscheduled_jobs
        sched = self.scheduled_jobs
        #python 3 expression for merging 2 dicts
        all_jobs = {**unsched, **sched}
        return all_jobs

    def update_users(self):
        """
        Update the user list with current users.
        """
        self.user_list = list(set(self.unscheduled_jobs_by_user.keys())
                              .union
                              (set(self.scheduled_jobs_by_user.keys())))

    def get_users(self):
        """
        Get a list of users in the system.
        :return: list of user names.
        """
        return self.user_list
<commit_msg>naming fixes<commit_after>"""
Old JobManager / container module. Likely to go away since using database for jobs.
"""

from cloudscheduler.job import Job

# We need to decide on logic for deciding if a job has been scheduled or not yet
# Unsure if we can rely on the metadata from the job but right now it just defaults to not scheduled

class JobManager:

    """
    Job Manager Class to assist with scheduling jobs.
    """
    def __init__(self):
        """
        Init for job manager.
        """

        self.unscheduled_jobs = {}
        self.unscheduled_jobs_by_user = {}
        self.scheduled_jobs = {}
        self.scheduled_jobs_by_user = {}

        self.user_list = list()

    # accepts a group of job dictionaries and shuffles them into the existing
    #  dictionaries based on their globaljobids.
    def update_jobs(self, jobs):
        """
        update job states.
        :param jobs: list of jobs from condor?
        """
        redis_key_list = []
        for job in jobs:
            j = Job(**job)
            redis_key_list.append(j.globaljobid)
            if j.globaljobid in self.unscheduled_jobs:
                #update unscheduled entry
                self.unscheduled_jobs[j.globaljobid] = j
                self.unscheduled_jobs_by_user[j.user][j.globaljobid] = j

            elif j.globaljobid in self.scheduled_jobs:
                #update scheduled entry
                j.set_state(1)
                self.scheduled_jobs[j.globaljobid] = j
                self.scheduled_jobs_by_user[j.user][j.globaljobid] = j

            else:
                #brand new job, insert into unscheduled dicts
                self.unscheduled_jobs[j.globaljobid] = j
                self.unscheduled_jobs_by_user[j.user][j.globaljobid] = j
        # Clean up jobs that were not in the redis store
        for key in self.scheduled_jobs:
            if key not in redis_key_list:
                del self.scheduled_jobs[key]
                del self.scheduled_jobs_by_user[key]
        for key in self.unscheduled_jobs:
            if key not in redis_key_list:
                del self.unscheduled_jobs[key]
                del self.unscheduled_jobs_by_user[key]

    def schedule_job(self, jobid):
        """
        Mark a job as scheduled.
        :param jobid: id of job to mark
        """
        self.unscheduled_jobs[jobid].set_state(1)
        job = self.unscheduled_jobs[jobid]
        self.scheduled_jobs[jobid] = job
        self.scheduled_jobs_by_user[job.user][jobid] = job
        del self.unscheduled_jobs[jobid]
        del self.unscheduled_jobs_by_user[job.user][jobid]

    def unschedule_job(self, jobid):
        """
        Unmark a previously scheduled job.
        :param jobid: id of job to unmark
        """
        self.scheduled_jobs[jobid].set_state(0)
        job = self.scheduled_jobs[jobid]
        self.unscheduled_jobs[jobid] = job
        self.unscheduled_jobs_by_user[job.user][jobid] = job
        del self.scheduled_jobs[jobid]
        del self.scheduled_jobs_by_user[job.user][jobid]

    def get_jobs_user(self, user):
        """
        Fetch the jobs for specified user.
        :param user: user to get jobs for.
        :return: list of jobs belonging to user or None.
        """
        unsched = self.unscheduled_jobs_by_user[user]
        sched = self.scheduled_jobs_by_user[user]
        # python 3 expression for merging 2 dicts
        user_jobs = {**unsched, **sched}
        return user_jobs

    def get_scheduled_jobs_user(self, user):
        """
        Get the scheduled jobs for user.
        :param user: user name
        :return: job dict
        """
        return self.scheduled_jobs_by_user[user]

    def get_unscheduled_jobs_user(self, user):
        """
        Get the unscheduled jobs for a specific user.
        :param user: user name to get jobs for.
        :return: job dict
        """
        return self.unscheduled_jobs_by_user[user]

    def get_unscheduled_jobs(self):
        """
        Get the unscheduled jobs.
        :return:
        """
        return self.unscheduled_jobs

    def get_scheduled_jobs(self):
        """
        Get the scheduled jobs dictionary.
        :return:
        """
        return self.scheduled_jobs

    def get_all_jobs(self):
        """
        Return a dict containing all the jobs in system.
        :return: dict of jobs.
        """
        unsched = self.unscheduled_jobs
        sched = self.scheduled_jobs
        #python 3 expression for merging 2 dicts
        all_jobs = {**unsched, **sched}
        return all_jobs

    def update_users(self):
        """
        Update the user list with current users.
        """
        self.user_list = list(set(self.unscheduled_jobs_by_user.keys())
                              .union
                              (set(self.scheduled_jobs_by_user.keys())))

    def get_users(self):
        """
        Get a list of users in the system.
        :return: list of user names.
        """
        return self.user_list
<|endoftext|>


<commit_before>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from cloudify.decorators import operation

from cloudstack_plugin.cloudstack_common import get_cloud_driver

__author__ = 'uri1803'

@operation
def create(ctx, **kwargs):
    """ Create network with rules.
    """

    cloud_driver = get_cloud_driver(ctx)

    network = {
        'description': None,
        'name': ctx.node_id,
    }

    ctx.logger.debug('reading network configuration.')
    network.update(ctx.properties['network'])

    network_name = network['name']
    zone = network['zone']
    location = get_location(cloud_driver, zone)
    netoffer = network['service_offering']
    network_offering = get_network_offering(cloud_driver, netoffer)
    firewall_config = network['firewall']

    # if network['vpc']:
    #     vpc = get_vpc_id(cloud_driver, network['vpc'])
    #     ctx.logger.info('DEBUG: VPC id: '.format(vpc.id))

    ctx.logger.info('Current node {0}{1}'.format(ctx.node_id, ctx.properties))

    ctx['network_id'] = ctx.node_id

    if not _network_exists(cloud_driver, network_name):

        if vpc:
            ctx.logger.info('creating network: {0} in VPC with ID: {1}'.format(network_name, vpc.id))
        else:
            ctx.logger.info('creating network: {0}'.format(network_name))

        net = cloud_driver.ex_create_network(
            display_text=network_name,
            name=network_name,
            network_offering=network_offering,
            location=location,
            gateway=network['gateway'],
            netmask=network['netmask'],
            vpc_id=vpc.id)

        # Create ACL for the network if it's is part of a VPC
        if vpc:

            acl_list = create_acl_list(cloud_driver, vpc.name, vpc.id, net.id)

            # Creat ingress ACL rules in ACLlist
            acl_ingress_ports = firewall_config['ingress']['ports']
            acl_ingress_protocol = firewall_config['ingress']['protocol']
            acl_ingress_cidr = firewall_config['ingress']['cidr']

            for port in acl_ingress_ports:
                create_acl(cloud_driver, acl_ingress_protocol, acl_list.id,
                           acl_ingress_cidr, port, port, "ingress")

             # Creat egress ACL rules in ACLlist
            acl_egress_ports = firewall_config['egress']['ports']
            acl_egress_protocol = firewall_config['egress']['protocol']
            acl_egress_cidr = firewall_config['egress']['cidr']

            for port in acl_egress_ports:
                create_acl(cloud_driver, acl_egress_protocol, acl_list.id,
                           acl_egress_cidr, port, port, "egress")

        else:
            # Create firewall rules for new network
            egress_rules = firewall_config['egress']
            egr_ports = egress_rules['ports']

            for port in egr_ports:
                _create_egr_rules(
                    ctx, cloud_driver, net.id, egress_rules['cidr'],
                    egress_rules['protocol'],
                    port, port)

    else:
        ctx.logger.info('using existing management network {0}'.
                        format(network_name))
        net = get_network(cloud_driver, network_name)

    ctx['network_id'] = net.id
    ctx['network_name'] = net.name

@operation
def delete(ctx, **kwargs):

    network_name = ctx.runtime_properties['network_name']
    cloud_driver = get_cloud_driver(ctx)
    network = get_network(cloud_driver, network_name)

    try:

        cloud_driver.ex_delete_network(network)
    except:
        ctx.logger.warn(
            'network {0} may not have been deleted'
                .format(ctx.runtime_properties['network_name']))
        pass

def _network_exists(cloud_driver, network_name):
    exists = get_network(cloud_driver, network_name)
    if not exists:
        return False
    return True

def get_network(cloud_driver, network_name):
    networks = [net for net in cloud_driver
        .ex_list_networks() if net.name == network_name]

    if networks.__len__() == 0:
        return None
    return networks[0]

def get_location(cloud_driver, location_name):
    locations = [location for location in cloud_driver
        .list_locations() if location.name == location_name]
    if locations.__len__() == 0:
        return None
    return locations[0]

def get_network_offering(cloud_driver, netoffer_name):
    netoffers = [offer for offer in cloud_driver
        .ex_list_network_offerings() if offer.name == netoffer_name]
    if netoffers.__len__() == 0:
        return None
    return netoffers[0]

def get_vpc_id(cloud_driver, vpc_name):
    vpcs = [vpc for vpc in cloud_driver
        .ex_list_vpcs() if vpc.name == vpc_name]
    if vpcs.__len__() == 0:
        return None
    return vpcs[0]

def create_acl_list(cloud_driver, name, vpc_id, network_id):
    acllist = cloud_driver.ex_create_network_acllist(
        name=name,
        vpc_id=vpc_id,
        description=name)

    # Replace the newly created ACL list on to the network
    cloud_driver.ex_replace_network_acllist(
        acl_id=acllist.id,
        network_id=network_id)

    return acllist

def create_acl(cloud_driver, protocol, acl_id,
               cidr_list, start_port, end_port, traffic_type):
    acl = cloud_driver.ex_create_network_acl(
        protocol=protocol,
        acl_id=acl_id,
        cidr_list=cidr_list,
        start_port=start_port,
        end_port=end_port,
        traffic_type=traffic_type)
    return acl

@operation
def _create_egr_rules(cloud_driver, network_id, cidr_list, protocol,
                      start_port, end_port):

    cloud_driver.ex_create_egress_firewall_rule(
        network_id=network_id,
        cidr_list=cidr_list,
        protocol=protocol,
        start_port=start_port,
        end_port=end_port)

# def _add_ingress_rule(ctx, node_name, protocol,
#                       cidr_list, start_port, end_port=None):
#
#     network_name = ctx.runtime_properties['network_name']
#     cloud_driver = get_cloud_driver(ctx)
#
#     ctx.logger.debug(
#         'creating port forward rule for {0} with details {1}'
#         .format(network_name, locals().values()))
#
#     cloud_driver.ex_create_port_forwarding_rule(
#         address=ip_address,
#         private_port=privateport,
#         public_port=publicport,
#         node=node,
#         protocol=protocol,
#         openfirewall=False)
<commit_msg>monir update<commit_after>########
# Copyright (c) 2014 GigaSpaces Technologies Ltd. All rights reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.

from cloudify.decorators import operation

from cloudstack_plugin.cloudstack_common import get_cloud_driver

__author__ = 'uri1803'

@operation
def create(ctx, **kwargs):
    """ Create network with rules.
    """

    cloud_driver = get_cloud_driver(ctx)

    network = {
        'description': None,
        'name': ctx.node_id,
    }

    ctx.logger.debug('reading network configuration.')
    network.update(ctx.properties['network'])

    network_name = network['name']
    zone = network['zone']
    location = get_location(cloud_driver, zone)
    netoffer = network['service_offering']
    network_offering = get_network_offering(cloud_driver, netoffer)
    firewall_config = network['firewall']

    # if network['vpc']:
    #     vpc = get_vpc_id(cloud_driver, network['vpc'])
    #     ctx.logger.info('DEBUG: VPC id: '.format(vpc.id))
    vpc = False

    ctx.logger.info('Current node {0}{1}'.format(ctx.node_id, ctx.properties))

    ctx['network_id'] = ctx.node_id

    if not _network_exists(cloud_driver, network_name):

        if vpc:
            ctx.logger.info('creating network: {0} in VPC with ID: {1}'.format(network_name, vpc.id))
        else:
            ctx.logger.info('creating network: {0}'.format(network_name))

        net = cloud_driver.ex_create_network(
            display_text=network_name,
            name=network_name,
            network_offering=network_offering,
            location=location,
            gateway=network['gateway'],
            netmask=network['netmask'],
            vpc_id=vpc.id)

        # Create ACL for the network if it's is part of a VPC
        if vpc:

            acl_list = create_acl_list(cloud_driver, vpc.name, vpc.id, net.id)

            # Creat ingress ACL rules in ACLlist
            acl_ingress_ports = firewall_config['ingress']['ports']
            acl_ingress_protocol = firewall_config['ingress']['protocol']
            acl_ingress_cidr = firewall_config['ingress']['cidr']

            for port in acl_ingress_ports:
                create_acl(cloud_driver, acl_ingress_protocol, acl_list.id,
                           acl_ingress_cidr, port, port, "ingress")

             # Creat egress ACL rules in ACLlist
            acl_egress_ports = firewall_config['egress']['ports']
            acl_egress_protocol = firewall_config['egress']['protocol']
            acl_egress_cidr = firewall_config['egress']['cidr']

            for port in acl_egress_ports:
                create_acl(cloud_driver, acl_egress_protocol, acl_list.id,
                           acl_egress_cidr, port, port, "egress")

        else:
            # Create firewall rules for new network
            egress_rules = firewall_config['egress']
            egr_ports = egress_rules['ports']

            for port in egr_ports:
                _create_egr_rules(
                    ctx, cloud_driver, net.id, egress_rules['cidr'],
                    egress_rules['protocol'],
                    port, port)

    else:
        ctx.logger.info('using existing management network {0}'.
                        format(network_name))
        net = get_network(cloud_driver, network_name)

    ctx['network_id'] = net.id
    ctx['network_name'] = net.name

@operation
def delete(ctx, **kwargs):

    network_name = ctx.runtime_properties['network_name']
    cloud_driver = get_cloud_driver(ctx)
    network = get_network(cloud_driver, network_name)

    try:

        cloud_driver.ex_delete_network(network)
    except:
        ctx.logger.warn(
            'network {0} may not have been deleted'
                .format(ctx.runtime_properties['network_name']))
        pass

def _network_exists(cloud_driver, network_name):
    exists = get_network(cloud_driver, network_name)
    if not exists:
        return False
    return True

def get_network(cloud_driver, network_name):
    networks = [net for net in cloud_driver
        .ex_list_networks() if net.name == network_name]

    if networks.__len__() == 0:
        return None
    return networks[0]

def get_location(cloud_driver, location_name):
    locations = [location for location in cloud_driver
        .list_locations() if location.name == location_name]
    if locations.__len__() == 0:
        return None
    return locations[0]

def get_network_offering(cloud_driver, netoffer_name):
    netoffers = [offer for offer in cloud_driver
        .ex_list_network_offerings() if offer.name == netoffer_name]
    if netoffers.__len__() == 0:
        return None
    return netoffers[0]

def get_vpc_id(cloud_driver, vpc_name):
    vpcs = [vpc for vpc in cloud_driver
        .ex_list_vpcs() if vpc.name == vpc_name]
    if vpcs.__len__() == 0:
        return None
    return vpcs[0]

def create_acl_list(cloud_driver, name, vpc_id, network_id):
    acllist = cloud_driver.ex_create_network_acllist(
        name=name,
        vpc_id=vpc_id,
        description=name)

    # Replace the newly created ACL list on to the network
    cloud_driver.ex_replace_network_acllist(
        acl_id=acllist.id,
        network_id=network_id)

    return acllist

def create_acl(cloud_driver, protocol, acl_id,
               cidr_list, start_port, end_port, traffic_type):
    acl = cloud_driver.ex_create_network_acl(
        protocol=protocol,
        acl_id=acl_id,
        cidr_list=cidr_list,
        start_port=start_port,
        end_port=end_port,
        traffic_type=traffic_type)
    return acl

@operation
def _create_egr_rules(cloud_driver, network_id, cidr_list, protocol,
                      start_port, end_port):

    cloud_driver.ex_create_egress_firewall_rule(
        network_id=network_id,
        cidr_list=cidr_list,
        protocol=protocol,
        start_port=start_port,
        end_port=end_port)

# def _add_ingress_rule(ctx, node_name, protocol,
#                       cidr_list, start_port, end_port=None):
#
#     network_name = ctx.runtime_properties['network_name']
#     cloud_driver = get_cloud_driver(ctx)
#
#     ctx.logger.debug(
#         'creating port forward rule for {0} with details {1}'
#         .format(network_name, locals().values()))
#
#     cloud_driver.ex_create_port_forwarding_rule(
#         address=ip_address,
#         private_port=privateport,
#         public_port=publicport,
#         node=node,
#         protocol=protocol,
#         openfirewall=False)
<|endoftext|>


<commit_before>import functools

def rgetattr(obj, attr, allow_null=True):
    attrs = attr.split('__')

    def _get_attr(o, a):
        try:
            return getattr(o, a)
        except AttributeError as e:
            if not allow_null:
                raise e

    return functools.reduce(_get_attr, [obj] + attrs)
<commit_msg>Added memoization utility func<commit_after>import six
from six.moves import reduce

def rgetattr(obj, attr, allow_null=True):
    attrs = attr.split('__')

    def _get_attr(o, a):
        try:
            return getattr(o, a)
        except AttributeError as e:
            if not allow_null:
                raise e

    return reduce(_get_attr, [obj] + attrs)

def memoized_property(fget):
    """
    Return a property attribute for new-style classes that only calls its getter
    on the first access. The result is stored and on subsequent accesses is
    returned, preventing the need to call the getter any more.

    Example:
    Following classes C, and D are equivalent.
    >>> class C(object):
    ...   @memoized_property
    ...   def name(self):
    ...     return expensive_function_call()
    >>> class D(object):
    ...   @property
    ...   def name(self):
    ...     if not hasattr(self, '_name'):
    ...       self._name = expensive_function_call()
    ...     return self._name
    """
    attr_name = '_{0}'.format(fget.__name__)

    @six.wraps(fget)
    def fget_memoized(self):
        if not hasattr(self, attr_name):
            setattr(self, attr_name, fget(self))
        return getattr(self, attr_name)

    return property(fget_memoized)
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

__version__ = '0.24.0-dev'
<commit_msg>changed dev version to get rid of warning when installing from checkout<commit_after># -*- coding: utf-8; -*-
#
# Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
# license agreements.  See the NOTICE file distributed with this work for
# additional information regarding copyright ownership.  Crate licenses
# this file to you under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.  You may
# obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
# License for the specific language governing permissions and limitations
# under the License.
#
# However, if you have executed another commercial license agreement
# with Crate these terms will supersede the license and you may use the
# software solely pursuant to the terms of the relevant commercial agreement.

__version__ = '0.24.0.dev0'
<|endoftext|>


<commit_before>from pylab import *
from mpl_toolkits.mplot3d import Axes3D

X = 100
Y = 0
Z = 0

ax = 0.1
ay = 1
az = 0.2

y = linspace(-100, 100, 1000)

x = (X + (y - Z)*ax/az)**2 + (Y + (y - Z)*ay/az)**2

def plot3d(x, y, z):
    f = figure()
    ax = f.add_subplot(111, projection='3d')
    ax.plot(x, y, z, '.')
    show()
<commit_msg>added make_ray function<commit_after>from pylab import *
from mpl_toolkits.mplot3d import Axes3D

# Load juno orbit 1
juno = np.loadtxt('juno-state-orbit1.txt')

# Load attitude file
attitude = np.loadtxt('juno-attitude.txt')

def project_state(state):
    """
    Project 3D state to the vertical ring plane
    using cylindrical coordinates.
    Input: n,3 matrix (X,Y,Z)
    Output: n,2 matrix (X,Y)
    """
    x = sqrt(state[:,0]**2 + state[:,1]**2)
    y = state[:,2]
    return np.vstack((x,y)).T

def make_ray(X, Y, Z, RA, DE, length):
    ax = cos(DE) * cos(RA)
    ay = cos(DE) * sin(RA)
    az = sin(DE)

    mu = linspace(0, length, length/1000)
    x = sqrt((X + mu*ax)**2 + (Y + mu*ay)**2)
    y = Z + mu*az

    return x,y

def plot3d(x, y, z):
    f = figure()
    ax = f.add_subplot(111, projection='3d')
    ax.plot(x, y, z, '.')
    show()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (2016-2020) Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###

ANSIBLE_METADATA = {'status': ['stableinterface'],
                    'supported_by': 'community',
                    'metadata_version': '1.1'}

DOCUMENTATION = '''
---
module: oneview_scope_facts
short_description: Retrieve facts about one or more of the OneView Scopes.
description:
    - Retrieve facts about one or more of the Scopes from OneView.
version_added: "2.3"
requirements:
    - "python >= 2.7.9"
    - "hpeOneView >= 3.0.0"
author: "Mariana Kreisig (@marikrg)"
options:
    config:
      description:
        - Path to a .json configuration file containing the OneView client configuration.
          The configuration file is optional. If the file path is not provided, the configuration will be loaded from
          environment variables.
      required: false
    name:
      description:
        - Name of the scope.
      required: false
    params:
      description:
        - List of params to delimit, filter and sort the list of resources.
        - "params allowed:
           c(start): The first item to return, using 0-based indexing.
           c(count): The number of resources to return.
           c(query): A general query string to narrow the list of resources returned.
           c(sort): The sort order of the returned data set.
           c(view): Returns a specific subset of the attributes of the resource or collection, by specifying the name
           of a predefined view."
      required: false
notes:
    - This resource is available for API version 300 or later.
extends_documentation_fragment:
    - oneview
'''

EXAMPLES = '''
- name: Gather facts about all Scopes
  oneview_scope_facts:
    config: "{{ config_path }}"

- debug: var=scopes

- name: Gather paginated, filtered and sorted facts about Scopes
  oneview_scope_facts:
    config: "{{ config }}"
    params:
      start: 0
      count: 3
      sort: name:ascending
      query: name eq 'SampleScope'
  delegate_to: localhost

- debug: var=scopes

- name: Gather facts about a Scope by name
  oneview_scope_facts:
    config: "{{ config_path }}"
    name: "Name of the Scope"

- debug: var=scopes
'''

RETURN = '''
scopes:
    description: Has all the OneView facts about the Scopes.
    returned: Always, but can be null.
    type: dict
'''

from ansible.module_utils.oneview import OneViewModuleBase

class ScopeFactsModule(OneViewModuleBase):
    argument_spec = dict(
        name=dict(required=False, type='str'),
        params=dict(required=False, type='dict')
    )

    def __init__(self):
        super(ScopeFactsModule, self).__init__(additional_arg_spec=self.argument_spec)
        self.set_resource_object(self.oneview_client.scopes)

    def execute_module(self):
        if self.current_resource:
            scopes = [self.current_resource.data]
        else:
            scopes = self.resource_client.get_all(**self.facts_params)

        return dict(changed=False, ansible_facts=dict(scopes=scopes))

def main():
    ScopeFactsModule().run()

if __name__ == '__main__':
    main()
<commit_msg>change parent class<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (2016-2020) Hewlett Packard Enterprise Development LP
#
# Licensed under the Apache License, Version 2.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
###

ANSIBLE_METADATA = {'status': ['stableinterface'],
                    'supported_by': 'community',
                    'metadata_version': '1.1'}

DOCUMENTATION = '''
---
module: oneview_scope_facts
short_description: Retrieve facts about one or more of the OneView Scopes.
description:
    - Retrieve facts about one or more of the Scopes from OneView.
version_added: "2.3"
requirements:
    - "python >= 2.7.9"
    - "hpeOneView >= 5.4.0"
author: "Mariana Kreisig (@marikrg)"
options:
    config:
      description:
        - Path to a .json configuration file containing the OneView client configuration.
          The configuration file is optional. If the file path is not provided, the configuration will be loaded from
          environment variables.
      required: false
    name:
      description:
        - Name of the scope.
      required: false
    params:
      description:
        - List of params to delimit, filter and sort the list of resources.
        - "params allowed:
           c(start): The first item to return, using 0-based indexing.
           c(count): The number of resources to return.
           c(query): A general query string to narrow the list of resources returned.
           c(sort): The sort order of the returned data set.
           c(view): Returns a specific subset of the attributes of the resource or collection, by specifying the name
           of a predefined view."
      required: false
notes:
    - This resource is available for API version 300 or later.
extends_documentation_fragment:
    - oneview
'''

EXAMPLES = '''
- name: Gather facts about all Scopes
  oneview_scope_facts:
    config: "{{ config_path }}"

- debug: var=scopes

- name: Gather paginated, filtered and sorted facts about Scopes
  oneview_scope_facts:
    config: "{{ config }}"
    params:
      start: 0
      count: 3
      sort: name:ascending
      query: name eq 'SampleScope'
  delegate_to: localhost

- debug: var=scopes

- name: Gather facts about a Scope by name
  oneview_scope_facts:
    config: "{{ config_path }}"
    name: "Name of the Scope"

- debug: var=scopes
'''

RETURN = '''
scopes:
    description: Has all the OneView facts about the Scopes.
    returned: Always, but can be null.
    type: dict
'''

from ansible.module_utils.oneview import OneViewModule

class ScopeFactsModule(OneViewModule):
    argument_spec = dict(
        name=dict(required=False, type='str'),
        params=dict(required=False, type='dict')
    )

    def __init__(self):
        super(ScopeFactsModule, self).__init__(additional_arg_spec=self.argument_spec)
        self.set_resource_object(self.oneview_client.scopes)

    def execute_module(self):
        if self.current_resource:
            scopes = [self.current_resource.data]
        else:
            scopes = self.resource_client.get_all(**self.facts_params)

        return dict(changed=False, ansible_facts=dict(scopes=scopes))

def main():
    ScopeFactsModule().run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import fnmatch
import json
import os
import multiprocessing
import re
import sys

from collections import OrderedDict

try:
    from colorama import Fore
    from colorama import Style
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()
    Style = EmptyStringAttributes()

from .process import command_output
from .library import Library
from .universal_binary import UniversalBinary
from .platform import available_platforms, host_platform
from .generator import available_generators
from .target import Target
from .cd import current_directory

class Needy:
    def __init__(self, path, parameters):
        self.__path = path if os.path.isabs(path) else os.path.normpath(os.path.join(current_directory(), path))
        self.__parameters = parameters

        self.__needs_directory = os.path.join(self.__path, 'needs')
        self.__needs_file = self.find_needs_file(self.__path)

        directory = self.__path
        while directory:
            if self.find_needs_file(directory):
                self.__needs_directory = os.path.join(directory, 'needs')
            directory = os.path.dirname(directory)
            if directory == os.sep:
                break

    def path(self):
        return self.__path

    def needs_file(self):
        return self.__needs_file

    @staticmethod
    def find_needs_file(directory):
        ret = None
        for name in ['needs.json', 'needs.yaml']:
            path = os.path.join(directory, name)
            if os.path.isfile(path):
                if ret:
                    raise RuntimeError('More than one needs file is present.')
                ret = path
        return ret

    def needs_configuration(self, target=None):
        configuration = ''
        with open(self.needs_file(), 'r') as needs_file:
            configuration = needs_file.read()

        try:
            from jinja2 import Environment, PackageLoader
            env = Environment()
            env.filters['dirname'] = os.path.dirname
            template = env.from_string(configuration)
            configuration = template.render(
                platform=target.platform.identifier() if target else None,
                architecture=target.architecture if target else None,
                host_platform=host_platform().identifier(),
                needs_file=self.needs_file()
            )
        except ImportError:
            if re.compile('{%.*%}').search(configuration) or re.compile('{{.*}}').search(configuration) or re.compile('{#.*#}').search(configuration):
                raise RuntimeError('The needs file appears to contain Jinja templating. Please install the jinja2 Python package.')

        name, extension = os.path.splitext(self.needs_file())

        if extension == '.json':
            return json.loads(configuration, object_pairs_hook=OrderedDict)

        if extension == '.yaml':
            try:
                import yaml
                class OrderedLoader(yaml.SafeLoader):
                    pass
                def construct_mapping(loader, node):
                    loader.flatten_mapping(node)
                    return OrderedDict(loader.construct_pairs(node))
                OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
                return yaml.load(configuration, OrderedLoader)
            except ImportError:
                raise RuntimeError('The needs are defined in a YAML file. Please install the pyyaml Python package.')

            return None

        raise NotImplementedError('file parsing not implemented')

    def needs_directory(self):
        return self.__needs_directory

    def parameters(self):
        return self.__parameters

    def build_concurrency(self):
        if self.parameters().concurrency > 0:
            return self.parameters().concurrency
        return multiprocessing.cpu_count()

    def platform(self, identifier):
        platform = host_platform() if identifier == 'host' else available_platforms().get(identifier, None)
        if platform is not None:
            return platform(self.__parameters)

        raise ValueError('unknown platform (%s)' % identifier)

    def target(self, identifier):
        parts = identifier.split(':')
        platform = self.platform(parts[0])
        return Target(platform, parts[1] if len(parts) > 1 else platform.default_architecture())

    def recursive(self, path):
        return Needy(path, self.parameters()) if self.find_needs_file(path) else None

    def libraries_to_build(self, target, filters=None):
        needs_configuration = self.needs_configuration(target)

        if 'libraries' not in needs_configuration:
            return []

        names = []

        for name, library_configuration in needs_configuration['libraries'].iteritems():
            if filters:
                match = False
                for filter in filters:
                    if fnmatch.fnmatchcase(name, filter):
                        match = True
                        break
                if not match:
                    continue
            names.append(name)

        graph = {}
        libraries = {}

        while len(names):
            name = names.pop()
            directory = os.path.join(self.__needs_directory, name)
            library = Library(target, needs_configuration['libraries'][name], directory, self)
            libraries[name] = library
            if 'dependencies' not in library.configuration():
                graph[name] = set()
                continue
            str_or_list = library.configuration()['dependencies']
            dependencies = str_or_list if isinstance(str_or_list, list) else [str_or_list]
            graph[name] = set(dependencies)
            for dependency in dependencies:
                if dependency not in graph:
                    names.append(dependency)

        s = []

        for name, dependencies in graph.iteritems():
            if len(dependencies) == 0:
                s.append(name)

        ret = []

        while len(s):
            name = s.pop()
            ret.append((name, libraries[name]))
            for n, deps in graph.iteritems():
                if name not in deps:
                    continue
                deps.remove(name)
                if len(deps) == 0:
                    s.append(n)

        for name, deps in graph.iteritems():
            if len(deps):
                raise ValueError('circular dependency detected')

        return ret

    def universal_binary_configuration(self, universal_binary):
        needs_configuration = self.needs_configuration()

        if 'universal-binaries' not in needs_configuration:
            raise ValueError('no universal binaries defined')

        if universal_binary not in needs_configuration['universal-binaries']:
            raise ValueError('unknown universal binary ({})'.format(universal_binary))

        return needs_configuration['universal-binaries'][universal_binary]

    def libraries(self, target_or_universal_binary, filters=None):
        libraries = dict()
        targets = []

        if isinstance(target_or_universal_binary, Target):
            targets = [target_or_universal_binary]
        else:
            configuration = self.universal_binary_configuration(target_or_universal_binary)
            for platform, architectures in configuration.iteritems():
                for architecture in architectures:
                    targets.append(Target(self.platform(platform), architecture))

        for target in targets:
            for name, library in self.libraries_to_build(target, filters):
                if name not in libraries:
                    libraries[name] = list()
                libraries[name].append(library)

        return libraries

    def include_paths(self, target_or_universal_binary, filters=None):
        ret = []
        for name, libraries in self.libraries(target_or_universal_binary, filters).iteritems():
            if isinstance(target_or_universal_binary, Target):
                for library in libraries:
                    ret.append(library.include_path())
            else:
                ub = UniversalBinary(target_or_universal_binary, libraries, self)
                ret.append(ub.include_path())
            needy = self.recursive(libraries[0].source_directory())
            if needy:
                ret.extend(needy.include_paths(target))
        return ret

    def library_paths(self, target_or_universal_binary, filters=None):
        ret = []
        for name, libraries in self.libraries(target_or_universal_binary, filters).iteritems():
            if isinstance(target_or_universal_binary, Target):
                for library in libraries:
                    ret.append(library.library_path())
            else:
                ub = UniversalBinary(target_or_universal_binary, libraries, self)
                ret.append(ub.library_path())
            needy = self.recursive(libraries[0].source_directory())
            if needy:
                ret.extend(needy.library_paths(target))
        return ret

    def build_directory(self, library, target_or_universal_binary):
        directory = os.path.join(self.__needs_directory, library)
        if isinstance(target_or_universal_binary, Target):
            l = Library(target_or_universal_binary, None, directory, self)
            return l.build_directory()
        l = Library(None, None, directory, self)
        b = UniversalBinary(target_or_universal_binary, [l], self)
        return b.build_directory()

    def satisfy_target(self, target, filters=None):
        needs_configuration = self.needs_configuration(target)

        if 'libraries' not in needs_configuration:
            return

        print('Satisfying needs in %s' % self.path())

        try:
            for name, library in self.libraries_to_build(target, filters):
                if library.has_up_to_date_build():
                    self.__print_status(Fore.GREEN, 'UP-TO-DATE', name)
                else:
                    self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                    library.build()
                    self.__print_status(Fore.GREEN, 'SUCCESS', name)
        except Exception as e:
            self.__print_status(Fore.RED, 'ERROR')
            print(e)
            raise

    def satisfy_universal_binary(self, universal_binary, filters=None):
        try:
            print('Satisfying universal binary %s in %s' % (universal_binary, self.path()))
            configuration = self.universal_binary_configuration(universal_binary)

            libraries = dict()

            for platform, architectures in configuration.iteritems():
                for architecture in architectures:
                    target = Target(self.platform(platform), architecture)
                    for name, library in self.libraries_to_build(target, filters):
                        if name not in libraries:
                            libraries[name] = list()
                        libraries[name].append(library)
                        if library.has_up_to_date_build():
                            self.__print_status(Fore.GREEN, 'UP-TO-DATE', '{} for {} {}'.format(name, target.platform.identifier(), target.architecture))
                        else:
                            self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                            library.build()
                            self.__print_status(Fore.GREEN, 'SUCCESS', name)

            for name, libs in libraries.iteritems():
                binary = UniversalBinary(universal_binary, libs, self)
                if binary.is_up_to_date():
                    self.__print_status(Fore.GREEN, 'UP-TO-DATE', name)
                else:
                    self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                    binary.build()
                    self.__print_status(Fore.GREEN, 'SUCCESS', name)
        except Exception as e:
            self.__print_status(Fore.RED, 'ERROR')
            print(e)
            raise

    def __print_status(self, color, status, name=None):
        print(color + Style.BRIGHT + '[' + status + ']' + Style.RESET_ALL + Fore.RESET + (' %s' % name if name else ''))

    def generate(self, files):
        if not os.path.exists(self.needs_directory()):
            os.makedirs(self.needs_directory())
        for generator in available_generators():
            if generator.identifier() in files:
                generator().generate(self)
<commit_msg>Add needs_directory for Jinja templates<commit_after>import fnmatch
import json
import os
import multiprocessing
import re
import sys

from collections import OrderedDict

try:
    from colorama import Fore
    from colorama import Style
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()
    Style = EmptyStringAttributes()

from .process import command_output
from .library import Library
from .universal_binary import UniversalBinary
from .platform import available_platforms, host_platform
from .generator import available_generators
from .target import Target
from .cd import current_directory

class Needy:
    def __init__(self, path, parameters):
        self.__path = path if os.path.isabs(path) else os.path.normpath(os.path.join(current_directory(), path))
        self.__parameters = parameters

        self.__needs_directory = os.path.join(self.__path, 'needs')
        self.__needs_file = self.find_needs_file(self.__path)

        directory = self.__path
        while directory:
            if self.find_needs_file(directory):
                self.__needs_directory = os.path.join(directory, 'needs')
            directory = os.path.dirname(directory)
            if directory == os.sep:
                break

    def path(self):
        return self.__path

    def needs_file(self):
        return self.__needs_file

    @staticmethod
    def find_needs_file(directory):
        ret = None
        for name in ['needs.json', 'needs.yaml']:
            path = os.path.join(directory, name)
            if os.path.isfile(path):
                if ret:
                    raise RuntimeError('More than one needs file is present.')
                ret = path
        return ret

    def needs_configuration(self, target=None):
        configuration = ''
        with open(self.needs_file(), 'r') as needs_file:
            configuration = needs_file.read()

        try:
            from jinja2 import Environment, PackageLoader
            env = Environment()
            env.filters['dirname'] = os.path.dirname
            template = env.from_string(configuration)
            configuration = template.render(
                platform=target.platform.identifier() if target else None,
                architecture=target.architecture if target else None,
                host_platform=host_platform().identifier(),
                needs_file=self.needs_file(),
                needs_directory=self.needs_directory()
            )
        except ImportError:
            if re.compile('{%.*%}').search(configuration) or re.compile('{{.*}}').search(configuration) or re.compile('{#.*#}').search(configuration):
                raise RuntimeError('The needs file appears to contain Jinja templating. Please install the jinja2 Python package.')

        name, extension = os.path.splitext(self.needs_file())

        if extension == '.json':
            return json.loads(configuration, object_pairs_hook=OrderedDict)

        if extension == '.yaml':
            try:
                import yaml
                class OrderedLoader(yaml.SafeLoader):
                    pass
                def construct_mapping(loader, node):
                    loader.flatten_mapping(node)
                    return OrderedDict(loader.construct_pairs(node))
                OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
                return yaml.load(configuration, OrderedLoader)
            except ImportError:
                raise RuntimeError('The needs are defined in a YAML file. Please install the pyyaml Python package.')

            return None

        raise NotImplementedError('file parsing not implemented')

    def needs_directory(self):
        return self.__needs_directory

    def parameters(self):
        return self.__parameters

    def build_concurrency(self):
        if self.parameters().concurrency > 0:
            return self.parameters().concurrency
        return multiprocessing.cpu_count()

    def platform(self, identifier):
        platform = host_platform() if identifier == 'host' else available_platforms().get(identifier, None)
        if platform is not None:
            return platform(self.__parameters)

        raise ValueError('unknown platform (%s)' % identifier)

    def target(self, identifier):
        parts = identifier.split(':')
        platform = self.platform(parts[0])
        return Target(platform, parts[1] if len(parts) > 1 else platform.default_architecture())

    def recursive(self, path):
        return Needy(path, self.parameters()) if self.find_needs_file(path) else None

    def libraries_to_build(self, target, filters=None):
        needs_configuration = self.needs_configuration(target)

        if 'libraries' not in needs_configuration:
            return []

        names = []

        for name, library_configuration in needs_configuration['libraries'].iteritems():
            if filters:
                match = False
                for filter in filters:
                    if fnmatch.fnmatchcase(name, filter):
                        match = True
                        break
                if not match:
                    continue
            names.append(name)

        graph = {}
        libraries = {}

        while len(names):
            name = names.pop()
            directory = os.path.join(self.__needs_directory, name)
            library = Library(target, needs_configuration['libraries'][name], directory, self)
            libraries[name] = library
            if 'dependencies' not in library.configuration():
                graph[name] = set()
                continue
            str_or_list = library.configuration()['dependencies']
            dependencies = str_or_list if isinstance(str_or_list, list) else [str_or_list]
            graph[name] = set(dependencies)
            for dependency in dependencies:
                if dependency not in graph:
                    names.append(dependency)

        s = []

        for name, dependencies in graph.iteritems():
            if len(dependencies) == 0:
                s.append(name)

        ret = []

        while len(s):
            name = s.pop()
            ret.append((name, libraries[name]))
            for n, deps in graph.iteritems():
                if name not in deps:
                    continue
                deps.remove(name)
                if len(deps) == 0:
                    s.append(n)

        for name, deps in graph.iteritems():
            if len(deps):
                raise ValueError('circular dependency detected')

        return ret

    def universal_binary_configuration(self, universal_binary):
        needs_configuration = self.needs_configuration()

        if 'universal-binaries' not in needs_configuration:
            raise ValueError('no universal binaries defined')

        if universal_binary not in needs_configuration['universal-binaries']:
            raise ValueError('unknown universal binary ({})'.format(universal_binary))

        return needs_configuration['universal-binaries'][universal_binary]

    def libraries(self, target_or_universal_binary, filters=None):
        libraries = dict()
        targets = []

        if isinstance(target_or_universal_binary, Target):
            targets = [target_or_universal_binary]
        else:
            configuration = self.universal_binary_configuration(target_or_universal_binary)
            for platform, architectures in configuration.iteritems():
                for architecture in architectures:
                    targets.append(Target(self.platform(platform), architecture))

        for target in targets:
            for name, library in self.libraries_to_build(target, filters):
                if name not in libraries:
                    libraries[name] = list()
                libraries[name].append(library)

        return libraries

    def include_paths(self, target_or_universal_binary, filters=None):
        ret = []
        for name, libraries in self.libraries(target_or_universal_binary, filters).iteritems():
            if isinstance(target_or_universal_binary, Target):
                for library in libraries:
                    ret.append(library.include_path())
            else:
                ub = UniversalBinary(target_or_universal_binary, libraries, self)
                ret.append(ub.include_path())
            needy = self.recursive(libraries[0].source_directory())
            if needy:
                ret.extend(needy.include_paths(target))
        return ret

    def library_paths(self, target_or_universal_binary, filters=None):
        ret = []
        for name, libraries in self.libraries(target_or_universal_binary, filters).iteritems():
            if isinstance(target_or_universal_binary, Target):
                for library in libraries:
                    ret.append(library.library_path())
            else:
                ub = UniversalBinary(target_or_universal_binary, libraries, self)
                ret.append(ub.library_path())
            needy = self.recursive(libraries[0].source_directory())
            if needy:
                ret.extend(needy.library_paths(target))
        return ret

    def build_directory(self, library, target_or_universal_binary):
        directory = os.path.join(self.__needs_directory, library)
        if isinstance(target_or_universal_binary, Target):
            l = Library(target_or_universal_binary, None, directory, self)
            return l.build_directory()
        l = Library(None, None, directory, self)
        b = UniversalBinary(target_or_universal_binary, [l], self)
        return b.build_directory()

    def satisfy_target(self, target, filters=None):
        needs_configuration = self.needs_configuration(target)

        if 'libraries' not in needs_configuration:
            return

        print('Satisfying needs in %s' % self.path())

        try:
            for name, library in self.libraries_to_build(target, filters):
                if library.has_up_to_date_build():
                    self.__print_status(Fore.GREEN, 'UP-TO-DATE', name)
                else:
                    self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                    library.build()
                    self.__print_status(Fore.GREEN, 'SUCCESS', name)
        except Exception as e:
            self.__print_status(Fore.RED, 'ERROR')
            print(e)
            raise

    def satisfy_universal_binary(self, universal_binary, filters=None):
        try:
            print('Satisfying universal binary %s in %s' % (universal_binary, self.path()))
            configuration = self.universal_binary_configuration(universal_binary)

            libraries = dict()

            for platform, architectures in configuration.iteritems():
                for architecture in architectures:
                    target = Target(self.platform(platform), architecture)
                    for name, library in self.libraries_to_build(target, filters):
                        if name not in libraries:
                            libraries[name] = list()
                        libraries[name].append(library)
                        if library.has_up_to_date_build():
                            self.__print_status(Fore.GREEN, 'UP-TO-DATE', '{} for {} {}'.format(name, target.platform.identifier(), target.architecture))
                        else:
                            self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                            library.build()
                            self.__print_status(Fore.GREEN, 'SUCCESS', name)

            for name, libs in libraries.iteritems():
                binary = UniversalBinary(universal_binary, libs, self)
                if binary.is_up_to_date():
                    self.__print_status(Fore.GREEN, 'UP-TO-DATE', name)
                else:
                    self.__print_status(Fore.CYAN, 'OUT-OF-DATE', name)
                    binary.build()
                    self.__print_status(Fore.GREEN, 'SUCCESS', name)
        except Exception as e:
            self.__print_status(Fore.RED, 'ERROR')
            print(e)
            raise

    def __print_status(self, color, status, name=None):
        print(color + Style.BRIGHT + '[' + status + ']' + Style.RESET_ALL + Fore.RESET + (' %s' % name if name else ''))

    def generate(self, files):
        if not os.path.exists(self.needs_directory()):
            os.makedirs(self.needs_directory())
        for generator in available_generators():
            if generator.identifier() in files:
                generator().generate(self)
<|endoftext|>