<commit_before>import argparse
import builtins
import re
from collections import OrderedDict
from enum import Enum
from inspect import Parameter

from .exc import CommandError
from .util import load_json_item, load_json_value

# Marker used to indicate that a bool arg's inverse option should be
# disabled.
DISABLE = object()

class ArgConfig:

    """Configuration for an arg.

    This can be used as a function parameter annotation to explicitly
    configure an arg, overriding default behavior.

    Args:

        short_option (str): Short option like ``-x`` to use instead of
            the default, which is derived from the first character of
            the arg name.
        long_option (str): Long option like ``--xyz`` to use instead of
            the default, which is derived from the arg name.
        type (type): Type to use instead of guessing based on the arg's
            default value.
        choices (sequence): Sequence of allowed choices for the arg.
        action (Action): ``argparse`` Action.
        help (str): Help string for the arg.
        inverse_help (str): Inverse help string for the arg (for the
            ``--no-xyz`` variation of boolean args).

    .. note:: For convenience, regular dicts can be used to annotate
        args instead instead; they will be converted to instances of
        this class automatically.

    """

    short_option_regex = re.compile(r'^-\w$')

    def __init__(self, *, short_option=None, long_option=None, inverse_option=None, type=None,
                 choices=None, action=None, help=None, inverse_help=None):
        if short_option is not None:
            if not self.short_option_regex.search(short_option):
                message = 'Expected short option with form -x, not "{short_option}"'
                message = message.format_map(locals())
                raise ValueError(message)

        if type is not None:
            if not isinstance(type, builtins.type):
                message = 'Expected type, not {type.__class__.__name__}'.format_map(locals())
                raise ValueError(message)

        self.short_option = short_option
        self.long_option = long_option
        self.inverse_option = inverse_option
        self.type = type
        self.choices = choices
        self.action = action
        self.help = help
        self.inverse_help = inverse_help

    def __repr__(self):
        options = self.short_option, self.long_option, self.inverse_option
        options = (option for option in options if option)
        options = ', '.join(options)
        return 'arg({options})'.format_map(locals())

arg = ArgConfig

class Arg:

    """Encapsulates an arg belonging to a command.

    Attributes:

        command (Command): Command this arg belongs to.
        parameter (Parameter): Function parameter this arg is derived
            from.
        name (str): Normalized arg name.
        type (type): Type of the arg. By default, a positional arg will
            be parsed as str and an optional/keyword arg will be parsed
            as the type of its default value (or as str if the default
            value is None).
        default (object): Default value for the arg.
        choices (sequence): A sequence of allowed choices for the arg.
        help (str): Help string for the arg.
        inverse_help (str): Inverse help string for the arg (for the
            ``--no-xyz`` variation of boolean args).
        short_option (str): Short command line option.
        long_option (str): Long command line option.
        inverse_option (str): Inverse option for boolean args.

    """

    empty = Parameter.empty

    def __init__(self, *,
                 command,
                 parameter,
                 name,
                 type,
                 default,
                 choices,
                 help,
                 inverse_help,
                 short_option,
                 long_option,
                 inverse_option,
                 action):

        self.command = command

        self.parameter = parameter
        self.parameter_name = parameter.name
        self.is_positional = default is self.empty
        self.is_optional = not self.is_positional
        self.is_keyword_only = parameter.kind is parameter.KEYWORD_ONLY

        self.name = name

        if type is None:
            type = str if default in (None, self.empty) else default.__class__
        elif not isinstance(type, builtins.type):
            message = 'Expected type, not {arg_type.__class__.__name__}'.format_map(locals())
            raise TypeError(message)

        self.type = type
        self.is_bool = issubclass(type, bool)
        self.is_dict = issubclass(type, dict)
        self.is_enum = issubclass(type, Enum)
        self.is_list = issubclass(type, list)
        self.is_tuple = issubclass(type, tuple)
        self.is_bool_or = issubclass(type, bool_or)
        self.takes_value = self.is_positional or (self.is_optional and not self.is_bool)
        self.default = default

        if not choices:
            if self.is_enum:
                choices = type
            elif self.is_bool_or and issubclass(type.type, Enum):
                choices = type.type

        self.choices = choices
        self.help = help
        self.inverse_help = inverse_help

        self.short_option = short_option
        self.long_option = long_option
        self.inverse_option = inverse_option if (self.is_bool or self.is_bool_or) else None

        options = (self.short_option, self.long_option, self.inverse_option)
        self.options = tuple(option for option in options if (option and option is not DISABLE))

        self.action = action

    def __str__(self):
        string = '{kind} arg: {self.name}{default} ({self.type.__name__})'
        kind = 'Positional' if self.is_positional else 'Optional'
        has_default = self.default not in (self.empty, None)
        default = '[={self.default}]'.format_map(locals()) if has_default else ''
        return string.format_map(locals())

class HelpArg(Arg):

    def __init__(self, *, command):
        parameter = Parameter('help', Parameter.POSITIONAL_OR_KEYWORD, default=False)
        super().__init__(
            command=command,
            parameter=parameter,
            name='help',
            type=bool,
            default=False,
            choices=None,
            help=None,
            inverse_help=None,
            short_option='-h',
            long_option='--help',
            inverse_option=None,
            action=None,
        )

class bool_or:

    """Used to indicate that an arg can be a flag or an option.

    Use like this::

        @command
        def local(config, cmd, hide: {'type': bool_or(str)} = False):
            "Run the specified command, possibly hiding its output."

    Allows for this::

        run local --hide all     # Hide everything
        run local --hide         # Hide everything with less effort
        run local --hide stdout  # Hide stdout only
        run local --no-hide      # Don't hide anything

    """

    type = None

    def __new__(cls, other_type):
        if not isinstance(other_type, type):
            message = 'Expected type, not {other_type.__class__.__name__}'.format_map(locals())
            raise TypeError(message)
        name = 'BoolOr{name}'.format(name=other_type.__name__.title())
        return type(name, (cls, ), {'type': other_type})

class BoolOrAction(argparse.Action):

    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            value = True
        setattr(namespace, self.dest, value)

class DictAddAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, OrderedDict())
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for item in values:
            try:
                name, value = load_json_item(item)
            except ValueError:
                raise CommandError('Bad format for {self.option_strings[0]}'.format_map(locals()))

            self.add_item(items, name, value)

    def add_item(self, items, name, value):
        items[name] = value

class NestedDictAddAction(DictAddAction):

    def add_item(self, items, name, value):
        segments = name.split('.')
        for segment in segments[:-1]:
            items = items.setdefault(segment, {})
        items[segments[-1]] = value

class ListAppendAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, [])
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for value in values:
            value = load_json_value(value)
            items.append(value)

class TupleAppendAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, ())
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for value in values:
            value = load_json_value(value)
            items += (value,)

        setattr(namespace, self.dest, items)
<commit_msg>Fix ArgConfig docstring<commit_after>import argparse
import builtins
import re
from collections import OrderedDict
from enum import Enum
from inspect import Parameter

from .exc import CommandError
from .util import load_json_item, load_json_value

# Marker used to indicate that a bool arg's inverse option should be
# disabled.
DISABLE = object()

class ArgConfig:

    """Configuration for an arg.

    This can be used as a function parameter annotation to explicitly
    configure an arg, overriding default behavior.

    Args:

        short_option (str): Short option like ``-x`` to use instead of
            the default, which is derived from the first character of
            the arg name.
        long_option (str): Long option like ``--xyz`` to use instead of
            the default, which is derived from the arg name.
        inverse_option (str): Inverse long option like ``--no-xyz`` for
            bool to use instead of the default, which is derived from
            the arg name. This can be set to :data:`DISABLED` to elide
            the inverse option (which is useful for bool options that
            are negative by default).
        type (type): Type to use instead of guessing based on the arg's
            default value.
        choices (sequence): Sequence of allowed choices for the arg.
        action (Action): ``argparse`` Action.
        help (str): Help string for the arg.
        inverse_help (str): Inverse help string for the arg (for the
            ``--no-xyz`` variation of boolean args).

    .. note:: For convenience, regular dicts can be used to annotate
        args instead; they will be converted to instances of this class
        automatically.

    """

    short_option_regex = re.compile(r'^-\w$')

    def __init__(self, *, short_option=None, long_option=None, inverse_option=None, type=None,
                 choices=None, action=None, help=None, inverse_help=None):
        if short_option is not None:
            if not self.short_option_regex.search(short_option):
                message = 'Expected short option with form -x, not "{short_option}"'
                message = message.format_map(locals())
                raise ValueError(message)

        if type is not None:
            if not isinstance(type, builtins.type):
                message = 'Expected type, not {type.__class__.__name__}'.format_map(locals())
                raise ValueError(message)

        self.short_option = short_option
        self.long_option = long_option
        self.inverse_option = inverse_option
        self.type = type
        self.choices = choices
        self.action = action
        self.help = help
        self.inverse_help = inverse_help

    def __repr__(self):
        options = self.short_option, self.long_option, self.inverse_option
        options = (option for option in options if option)
        options = ', '.join(options)
        return 'arg({options})'.format_map(locals())

arg = ArgConfig

class Arg:

    """Encapsulates an arg belonging to a command.

    Attributes:

        command (Command): Command this arg belongs to.
        parameter (Parameter): Function parameter this arg is derived
            from.
        name (str): Normalized arg name.
        type (type): Type of the arg. By default, a positional arg will
            be parsed as str and an optional/keyword arg will be parsed
            as the type of its default value (or as str if the default
            value is None).
        default (object): Default value for the arg.
        choices (sequence): A sequence of allowed choices for the arg.
        help (str): Help string for the arg.
        inverse_help (str): Inverse help string for the arg (for the
            ``--no-xyz`` variation of boolean args).
        short_option (str): Short command line option.
        long_option (str): Long command line option.
        inverse_option (str): Inverse option for boolean args.

    """

    empty = Parameter.empty

    def __init__(self, *,
                 command,
                 parameter,
                 name,
                 type,
                 default,
                 choices,
                 help,
                 inverse_help,
                 short_option,
                 long_option,
                 inverse_option,
                 action):

        self.command = command

        self.parameter = parameter
        self.parameter_name = parameter.name
        self.is_positional = default is self.empty
        self.is_optional = not self.is_positional
        self.is_keyword_only = parameter.kind is parameter.KEYWORD_ONLY

        self.name = name

        if type is None:
            type = str if default in (None, self.empty) else default.__class__
        elif not isinstance(type, builtins.type):
            message = 'Expected type, not {arg_type.__class__.__name__}'.format_map(locals())
            raise TypeError(message)

        self.type = type
        self.is_bool = issubclass(type, bool)
        self.is_dict = issubclass(type, dict)
        self.is_enum = issubclass(type, Enum)
        self.is_list = issubclass(type, list)
        self.is_tuple = issubclass(type, tuple)
        self.is_bool_or = issubclass(type, bool_or)
        self.takes_value = self.is_positional or (self.is_optional and not self.is_bool)
        self.default = default

        if not choices:
            if self.is_enum:
                choices = type
            elif self.is_bool_or and issubclass(type.type, Enum):
                choices = type.type

        self.choices = choices
        self.help = help
        self.inverse_help = inverse_help

        self.short_option = short_option
        self.long_option = long_option
        self.inverse_option = inverse_option if (self.is_bool or self.is_bool_or) else None

        options = (self.short_option, self.long_option, self.inverse_option)
        self.options = tuple(option for option in options if (option and option is not DISABLE))

        self.action = action

    def __str__(self):
        string = '{kind} arg: {self.name}{default} ({self.type.__name__})'
        kind = 'Positional' if self.is_positional else 'Optional'
        has_default = self.default not in (self.empty, None)
        default = '[={self.default}]'.format_map(locals()) if has_default else ''
        return string.format_map(locals())

class HelpArg(Arg):

    def __init__(self, *, command):
        parameter = Parameter('help', Parameter.POSITIONAL_OR_KEYWORD, default=False)
        super().__init__(
            command=command,
            parameter=parameter,
            name='help',
            type=bool,
            default=False,
            choices=None,
            help=None,
            inverse_help=None,
            short_option='-h',
            long_option='--help',
            inverse_option=None,
            action=None,
        )

class bool_or:

    """Used to indicate that an arg can be a flag or an option.

    Use like this::

        @command
        def local(config, cmd, hide: {'type': bool_or(str)} = False):
            "Run the specified command, possibly hiding its output."

    Allows for this::

        run local --hide all     # Hide everything
        run local --hide         # Hide everything with less effort
        run local --hide stdout  # Hide stdout only
        run local --no-hide      # Don't hide anything

    """

    type = None

    def __new__(cls, other_type):
        if not isinstance(other_type, type):
            message = 'Expected type, not {other_type.__class__.__name__}'.format_map(locals())
            raise TypeError(message)
        name = 'BoolOr{name}'.format(name=other_type.__name__.title())
        return type(name, (cls, ), {'type': other_type})

class BoolOrAction(argparse.Action):

    def __call__(self, parser, namespace, value, option_string=None):
        if value is None:
            value = True
        setattr(namespace, self.dest, value)

class DictAddAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, OrderedDict())
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for item in values:
            try:
                name, value = load_json_item(item)
            except ValueError:
                raise CommandError('Bad format for {self.option_strings[0]}'.format_map(locals()))

            self.add_item(items, name, value)

    def add_item(self, items, name, value):
        items[name] = value

class NestedDictAddAction(DictAddAction):

    def add_item(self, items, name, value):
        segments = name.split('.')
        for segment in segments[:-1]:
            items = items.setdefault(segment, {})
        items[segments[-1]] = value

class ListAppendAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, [])
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for value in values:
            value = load_json_value(value)
            items.append(value)

class TupleAppendAction(argparse.Action):

    def __call__(self, parser, namespace, values, option_string=None):
        if not hasattr(namespace, self.dest):
            setattr(namespace, self.dest, ())
        items = getattr(namespace, self.dest)

        if not isinstance(values, list):
            values = [values]

        for value in values:
            value = load_json_value(value)
            items += (value,)

        setattr(namespace, self.dest, items)
<|endoftext|>


<commit_before>from numpy import array, zeros, append

def block_iterator(data, block_size=1024, hop_size=1024):
    """
    Iterates over the given data in blocks.

    :param block_size: int
    :param hop_size: int
    :return: iterator
    """
    i = 0
    while i < len(data):
        yield array(data[i:i+block_size])
        i += hop_size

def process(data, transformation, block_size=1024, hop_size=None):
    """Applies transformation on audio data blockwise."""
    if not hop_size:
        hop_size = block_size

    processed_data = []

    random_parameter = 0  # TODO random walk
    previous_block = zeros(block_size)
    for current_block in block_iterator(data, block_size=block_size, hop_size=hop_size):
        processed_block = transformation(current_block, previous_block, random_parameter)
        processed_data += list(processed_block)

    return array(processed_data)

def process_multi(data, *args):
    """Like :func:`process` but can handle multichannel audio data."""
    if len(data.shape) == 1:
        return process(data, *args)
    elif len(data.shape) == 2:
        processed_data = []
        for i in range(data.shape[1]):
            processed_channel = process(data[:, i], *args)
            processed_data.append(processed_channel)
        return array(processed_data).transpose()

<commit_msg>fix: previous_block was not set and kwargs were missing<commit_after>from numpy import array, zeros, append

def block_iterator(data, block_size=1024, hop_size=1024):
    """
    Iterates over the given data in blocks.

    :param block_size: int
    :param hop_size: int
    :return: iterator
    """
    i = 0
    while i < len(data):
        yield array(data[i:i+block_size])
        i += hop_size

def process(data, transformation, block_size=1024, hop_size=None):
    """Applies transformation on audio data blockwise."""
    if not hop_size:
        hop_size = block_size

    processed_data = []

    random_parameter = 0  # TODO random walk
    previous_block = zeros(block_size)
    for current_block in block_iterator(data, block_size=block_size, hop_size=hop_size):
        processed_block = transformation(current_block, previous_block, random_parameter)
        processed_data += list(processed_block)
        previous_block = current_block
    return array(processed_data)

def process_multi(data, *args, **kwargs):
    """Like :func:`process` but can handle multichannel audio data."""
    if len(data.shape) == 1:
        return process(data, *args)
    elif len(data.shape) == 2:
        processed_data = []
        for i in range(data.shape[1]):
            processed_channel = process(data[:, i], *args, **kwargs)
            processed_data.append(processed_channel)
        return array(processed_data).transpose()

<|endoftext|>


<commit_before>"""
Copyright 2017-present, Airbnb Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import re

from app_integrations import LOGGER
from app_integrations.apps.app_base import app, AppIntegration

@app
class OneLoginApp(AppIntegration):
    """OneLogin StreamAlert App"""
    _ONELOGIN_EVENTS_URL = 'https://api.{}.onelogin.com/api/1/events'
    _ONELOGIN_TOKEN_URL = 'https://api.{}.onelogin.com/auth/oauth2/v2/token'
    _ONELOGIN_RATE_LIMIT_URL = 'https://api.{}.onelogin.com/auth/rate_limit'
    # OneLogin API returns 50 events per page
    _MAX_EVENTS_LIMIT = 50

    # Define our authorization headers variable
    def __init__(self, config):
        super(OneLoginApp, self).__init__(config)
        self._auth_headers = None
        self._next_page_url = None
        self._rate_limit_sleep = 0

    @classmethod
    def _type(cls):
        return 'events'

    @classmethod
    def service(cls):
        return 'onelogin'

    @classmethod
    def date_formatter(cls):
        """OneLogin API expects the ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ"""
        return '%Y-%m-%dT%H:%M:%SZ'

    def _token_endpoint(self):
        """Get the endpoint URL to retrieve tokens

        Returns:
            str: Full URL to generate tokens for the OneLogin API
        """
        return self._ONELOGIN_TOKEN_URL.format(self._config.auth['region'])

    def _events_endpoint(self):
        """Get the endpoint URL to retrieve events

        Returns:
            str: Full URL to retrieve events in the OneLogin API
        """
        return self._ONELOGIN_EVENTS_URL.format(self._config.auth['region'])

    def _rate_limit_endpoint(self):
        """Get the endpoint URL to retrieve rate limit details

        Returns:
            str: Full URL to retrieve rate limit details in the OneLogin API
        """
        return self._ONELOGIN_RATE_LIMIT_URL.format(self._config.auth['region'])

    def _generate_headers(self):
        """Each request will request a new token to call the resources APIs.

        More details to be found here:
            https://developers.onelogin.com/api-docs/1/oauth20-tokens/generate-tokens-2

        Returns:
            str: Bearer token to be used to call the OneLogin resource APIs
        """
        if self._auth_headers:
            return True

        authorization = 'client_id: {}, client_secret: {}'.format(
            self._config.auth['client_id'], self._config.auth['client_secret'])

        headers_token = {'Authorization': authorization,
                         'Content-Type': 'application/json'}

        result, response = self._make_post_request(self._token_endpoint(),
                                                   headers_token,
                                                   {'grant_type':'client_credentials'})

        if not result:
            return False

        if not response:
            LOGGER.error('Response invalid generating headers for service \'%s\'',
                         self.type())
            return False

        bearer = 'bearer:{}'.format(response.get('access_token'))
        self._auth_headers = {'Authorization': bearer}

        return True

    def _gather_logs(self):
        """Gather the authentication log events."""
        if not self._generate_headers():
            return False

        return self._get_onelogin_events()

    def _set_rate_limit_sleep(self):
        """Get the number of seconds we need to sleep until we are clear to continue"""
        # Make sure we have authentication headers
        if not self._auth_headers:
            self._rate_limit_sleep = 0
            LOGGER.error('No authentication headers for service \'%s\'',
                         self.type())
            return

        result, response = self._make_get_request(self._rate_limit_endpoint(),
                                                  self._auth_headers)

        if not result:
            self._rate_limit_sleep = 0
            return

        # Making sure we have a valid response
        if not response:
            LOGGER.error('Response invalid getting rate limit data for service \'%s\'',
                         self.type())
            self._rate_limit_sleep = 0
            return

        self._rate_limit_sleep = response.get('data')['X-RateLimit-Reset']
        LOGGER.info('Rate limit sleep set for \'%s\': %d', self.type(), self._rate_limit_sleep)

    def _get_onelogin_events(self):
        """Get all events from the endpoint for this timeframe

        Returns:
            [
                {
                    'id': <int:id>,
                    'created_at': <str:created_at>,
                    'account_id': <int:account_id>,
                    'user_id': <int:user_id>,
                    'event_type_id': <int:event_type_id>,
                    'notes': <str:notes>,
                    'ipaddr': <str:ipaddr>,
                    'actor_user_id': <int:actor_user_id>,
                    'assuming_acting_user_id': null,
                    'role_id': <int:role_id>,
                    'app_id': <int:app_id>,
                    'group_id': <int:group_id>,
                    'otp_device_id': <int:otp_device_id>,
                    'policy_id': <int:policy_id>,
                    'actor_system': <str:actor_system>,
                    'custom_message': <str:custom_message>,
                    'role_name': <str:role_name>,
                    'app_name': <str:app_name>,
                    'group_name': <str:group_name>,
                    'actor_user_name': <str:actor_user_name>,
                    'user_name': <str:user_name>,
                    'policy_name': <str:policy_name>,
                    'otp_device_name': <str:otp_device_name>,
                    'operation_name': <str:operation_name>,
                    'directory_sync_run_id': <int:directory_sync_run_id>,
                    'directory_id': <int:directory_id>,
                    'resolution': <str:resolution>,
                    'client_id': <int:client_id>,
                    'resource_type_id': <int:resource_type_id>,
                    'error_description': <str:error_description>
                }
            ]
        """
        # Make sure we have authentication headers
        if not self._auth_headers:
            LOGGER.error('No authentication headers for service \'%s\'',
                         self.type())
            return False

        # Are we just getting events or getting paginated events?
        if self._next_page_url:
            params = None
            request_url = self._next_page_url
        else:
            params = {'since': self._last_timestamp}
            request_url = self._events_endpoint()

        result, response = self._make_get_request(request_url, self._auth_headers, params)

        if not result:
            # If we hit the rate limit, update the sleep time
            if response.get('status'):
                r_status = response.get('status')
                if r_status['code'] == 400 and r_status['message'] == 'rate_limit_exceeded':
                    self._set_rate_limit_sleep()

            return False

        # Fail if response is invalid
        if not response:
            LOGGER.error('Response is invalid for service \'%s\'', self.type())
            return False

        # Set pagination link, if there is any
        self._next_page_url = response['pagination']['next_link']
        self._more_to_poll = bool(self._next_page_url)

        # Adjust the last seen event, if the events list is not empty
        if not response['data']:
            LOGGER.info('Empty list of events for service \'%s\'', self.type())
            return False

        self._last_timestamp = response['data'][-1]['created_at']

        # Return the list of events to the caller so they can be send to the batcher
        return response['data']

    @classmethod
    def _required_auth_info(cls):
        return {
            'region':
                {
                    'description': ('the region for the OneLogin API. This should be'
                                    'just "en" or "us".'),
                    'format': re.compile(r'^(en|us)$')
                },
            'client_secret':
                {
                    'description': ('the client secret for the OneLogin API. This '
                                    'should be a string of 64 alphanumeric characters'),
                    'format': re.compile(r'^[a-z0-9]{64}$')
                },
            'client_id':
                {
                    'description': ('the client id for the OneLogin API. This '
                                    'should be a string of 64 alphanumeric characters'),
                    'format': re.compile(r'^[a-z0-9]{64}$')
                }
            }

    def _sleep_seconds(self):
        """Return the number of seconds this polling function should sleep for
        between requests to avoid failed requests. OneLogin tokens allows for 5000 requests
        every hour, but if the rate limit is reached, we can retrieve how long until we are clear.

        More information about this here:
            https://developers.onelogin.com/api-docs/1/oauth20-tokens/get-rate-limit

        Returns:
            int: Number of seconds that this function should sleep for between requests
        """
        return self._rate_limit_sleep
<commit_msg>Making sure response is valid<commit_after>"""
Copyright 2017-present, Airbnb Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import re

from app_integrations import LOGGER
from app_integrations.apps.app_base import app, AppIntegration

@app
class OneLoginApp(AppIntegration):
    """OneLogin StreamAlert App"""
    _ONELOGIN_EVENTS_URL = 'https://api.{}.onelogin.com/api/1/events'
    _ONELOGIN_TOKEN_URL = 'https://api.{}.onelogin.com/auth/oauth2/v2/token'
    _ONELOGIN_RATE_LIMIT_URL = 'https://api.{}.onelogin.com/auth/rate_limit'
    # OneLogin API returns 50 events per page
    _MAX_EVENTS_LIMIT = 50

    # Define our authorization headers variable
    def __init__(self, config):
        super(OneLoginApp, self).__init__(config)
        self._auth_headers = None
        self._next_page_url = None
        self._rate_limit_sleep = 0

    @classmethod
    def _type(cls):
        return 'events'

    @classmethod
    def service(cls):
        return 'onelogin'

    @classmethod
    def date_formatter(cls):
        """OneLogin API expects the ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ"""
        return '%Y-%m-%dT%H:%M:%SZ'

    def _token_endpoint(self):
        """Get the endpoint URL to retrieve tokens

        Returns:
            str: Full URL to generate tokens for the OneLogin API
        """
        return self._ONELOGIN_TOKEN_URL.format(self._config.auth['region'])

    def _events_endpoint(self):
        """Get the endpoint URL to retrieve events

        Returns:
            str: Full URL to retrieve events in the OneLogin API
        """
        return self._ONELOGIN_EVENTS_URL.format(self._config.auth['region'])

    def _rate_limit_endpoint(self):
        """Get the endpoint URL to retrieve rate limit details

        Returns:
            str: Full URL to retrieve rate limit details in the OneLogin API
        """
        return self._ONELOGIN_RATE_LIMIT_URL.format(self._config.auth['region'])

    def _generate_headers(self):
        """Each request will request a new token to call the resources APIs.

        More details to be found here:
            https://developers.onelogin.com/api-docs/1/oauth20-tokens/generate-tokens-2

        Returns:
            str: Bearer token to be used to call the OneLogin resource APIs
        """
        if self._auth_headers:
            return True

        authorization = 'client_id: {}, client_secret: {}'.format(
            self._config.auth['client_id'], self._config.auth['client_secret'])

        headers_token = {'Authorization': authorization,
                         'Content-Type': 'application/json'}

        result, response = self._make_post_request(self._token_endpoint(),
                                                   headers_token,
                                                   {'grant_type':'client_credentials'})

        if not result:
            return False

        if not response:
            LOGGER.error('Response invalid generating headers for service \'%s\'',
                         self.type())
            return False

        bearer = 'bearer:{}'.format(response.get('access_token'))
        self._auth_headers = {'Authorization': bearer}

        return True

    def _gather_logs(self):
        """Gather the authentication log events."""
        if not self._generate_headers():
            return False

        return self._get_onelogin_events()

    def _set_rate_limit_sleep(self):
        """Get the number of seconds we need to sleep until we are clear to continue"""
        # Make sure we have authentication headers
        if not self._auth_headers:
            self._rate_limit_sleep = 0
            LOGGER.error('No authentication headers for service \'%s\'',
                         self.type())
            return

        result, response = self._make_get_request(self._rate_limit_endpoint(),
                                                  self._auth_headers)

        if not result:
            self._rate_limit_sleep = 0
            return

        # Making sure we have a valid response
        if not response:
            LOGGER.error('Response invalid getting rate limit data for service \'%s\'',
                         self.type())
            self._rate_limit_sleep = 0
            return

        self._rate_limit_sleep = response.get('data')['X-RateLimit-Reset']
        LOGGER.info('Rate limit sleep set for \'%s\': %d', self.type(), self._rate_limit_sleep)

    def _get_onelogin_events(self):
        """Get all events from the endpoint for this timeframe

        Returns:
            [
                {
                    'id': <int:id>,
                    'created_at': <str:created_at>,
                    'account_id': <int:account_id>,
                    'user_id': <int:user_id>,
                    'event_type_id': <int:event_type_id>,
                    'notes': <str:notes>,
                    'ipaddr': <str:ipaddr>,
                    'actor_user_id': <int:actor_user_id>,
                    'assuming_acting_user_id': null,
                    'role_id': <int:role_id>,
                    'app_id': <int:app_id>,
                    'group_id': <int:group_id>,
                    'otp_device_id': <int:otp_device_id>,
                    'policy_id': <int:policy_id>,
                    'actor_system': <str:actor_system>,
                    'custom_message': <str:custom_message>,
                    'role_name': <str:role_name>,
                    'app_name': <str:app_name>,
                    'group_name': <str:group_name>,
                    'actor_user_name': <str:actor_user_name>,
                    'user_name': <str:user_name>,
                    'policy_name': <str:policy_name>,
                    'otp_device_name': <str:otp_device_name>,
                    'operation_name': <str:operation_name>,
                    'directory_sync_run_id': <int:directory_sync_run_id>,
                    'directory_id': <int:directory_id>,
                    'resolution': <str:resolution>,
                    'client_id': <int:client_id>,
                    'resource_type_id': <int:resource_type_id>,
                    'error_description': <str:error_description>
                }
            ]
        """
        # Make sure we have authentication headers
        if not self._auth_headers:
            LOGGER.error('No authentication headers for service \'%s\'',
                         self.type())
            return False

        # Are we just getting events or getting paginated events?
        if self._next_page_url:
            params = None
            request_url = self._next_page_url
        else:
            params = {'since': self._last_timestamp}
            request_url = self._events_endpoint()

        result, response = self._make_get_request(request_url, self._auth_headers, params)

        if not result:
            # If we hit the rate limit, update the sleep time
            if response and response.get('status'):
                r_status = response.get('status')
                if r_status['code'] == 400 and r_status['message'] == 'rate_limit_exceeded':
                    self._set_rate_limit_sleep()

            return False

        # Fail if response is invalid
        if not response:
            LOGGER.error('Response is invalid for service \'%s\'', self.type())
            return False

        # Set pagination link, if there is any
        self._next_page_url = response['pagination']['next_link']
        self._more_to_poll = bool(self._next_page_url)

        # Adjust the last seen event, if the events list is not empty
        if not response['data']:
            LOGGER.info('Empty list of events for service \'%s\'', self.type())
            return False

        self._last_timestamp = response['data'][-1]['created_at']

        # Return the list of events to the caller so they can be send to the batcher
        return response['data']

    @classmethod
    def _required_auth_info(cls):
        return {
            'region':
                {
                    'description': ('the region for the OneLogin API. This should be'
                                    'just "en" or "us".'),
                    'format': re.compile(r'^(en|us)$')
                },
            'client_secret':
                {
                    'description': ('the client secret for the OneLogin API. This '
                                    'should be a string of 64 alphanumeric characters'),
                    'format': re.compile(r'^[a-z0-9]{64}$')
                },
            'client_id':
                {
                    'description': ('the client id for the OneLogin API. This '
                                    'should be a string of 64 alphanumeric characters'),
                    'format': re.compile(r'^[a-z0-9]{64}$')
                }
            }

    def _sleep_seconds(self):
        """Return the number of seconds this polling function should sleep for
        between requests to avoid failed requests. OneLogin tokens allows for 5000 requests
        every hour, but if the rate limit is reached, we can retrieve how long until we are clear.

        More information about this here:
            https://developers.onelogin.com/api-docs/1/oauth20-tokens/get-rate-limit

        Returns:
            int: Number of seconds that this function should sleep for between requests
        """
        return self._rate_limit_sleep
<|endoftext|>


<commit_before>##
## This file is auto-generated by ../s2client-api-build/buildpylib/update_version_file.py
##
import subprocess

def game_version():
    return "4.7.1.70326.1"

def read_command_output(cmd):
    lines = []
    handle = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    while True:
        line = handle.stdout.readline()
        if line != '':
            lines.append(line.rstrip())
        else:
            break
    return lines

def git_commit_hash():
    git_version_cmd = ['git', 'rev-parse', 'HEAD']
    lines = read_command_output(git_version_cmd)
    if len(lines) == 0 or len(lines) > 1:
        raise Exception("invalid output '{}' from '{}'".format(
            '\\n'.join(lines),
            ' '.join(git_version_cmd)))

    git_commit_hash = (''.join(lines))[0:8]

    return git_commit_hash

if __name__ == '__main__':
    print("Game version: {}".format(game_version()))
    print("Git commit hash: {}".format(git_commit_hash()))<commit_msg>Updating to version 4.8.0.71061.0<commit_after>##
## This file is auto-generated by ../s2client-api-build/buildpylib/update_version_file.py
##
import subprocess

def game_version():
    return "4.8.0.71061.0"

def read_command_output(cmd):
    lines = []
    handle = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    while True:
        line = handle.stdout.readline()
        if line != '':
            lines.append(line.rstrip())
        else:
            break
    return lines

def git_commit_hash():
    git_version_cmd = ['git', 'rev-parse', 'HEAD']
    lines = read_command_output(git_version_cmd)
    if len(lines) == 0 or len(lines) > 1:
        raise Exception("invalid output '{}' from '{}'".format(
            '\\n'.join(lines),
            ' '.join(git_version_cmd)))

    git_commit_hash = (''.join(lines))[0:8]

    return git_commit_hash

if __name__ == '__main__':
    print("Game version: {}".format(game_version()))
    print("Git commit hash: {}".format(git_commit_hash()))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import json
import django

from django_react_templatetags.mixins import RepresentationMixin

if django.VERSION >= (1, 10):
    from django.core.serializers.json import DjangoJSONEncoder
else:
    from django_react_templatetags.serializers import DjangoJSONEncoder

class ReactRepresentationJSONEncoder(DjangoJSONEncoder):
    '''
    Custom json encoder that adds support for RepresentationMixin
    '''
    def default(self, o):
        if isinstance(o, RepresentationMixin):
            return o.react_representation
        return super(ReactRepresentationJSONEncoder, self).default(o)
<commit_msg>Remove unecessary import<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import django

from django_react_templatetags.mixins import RepresentationMixin

if django.VERSION >= (1, 10):
    from django.core.serializers.json import DjangoJSONEncoder
else:
    from django_react_templatetags.serializers import DjangoJSONEncoder

class ReactRepresentationJSONEncoder(DjangoJSONEncoder):
    '''
    Custom json encoder that adds support for RepresentationMixin
    '''
    def default(self, o):
        if isinstance(o, RepresentationMixin):
            return o.react_representation
        return super(ReactRepresentationJSONEncoder, self).default(o)
<|endoftext|>


<commit_before>from django.conf.urls import patterns, include, url
from django.conf.urls.i18n import i18n_patterns

# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns( '',
  url( r'^jos2', 'wui.views.jos2' ),
  # url( r'^admin/', include( admin.site.urls ) ),
)

# These URLs accept the language prefix.
urlpatterns += i18n_patterns('',
  url(r'^(?P<unit>\w{3,15})/', 'wui.views.results'),
  url(r'^', 'wui.views.results'),
)
<commit_msg>Prevent any unexpected URL from rendering the whole website again.<commit_after>from django.conf.urls import patterns, include, url
from django.conf.urls.i18n import i18n_patterns

# Uncomment the next two lines to enable the admin:
# from django.contrib import admin
# admin.autodiscover()

urlpatterns = patterns( '',
  url( r'^jos2', 'wui.views.jos2' ),
  # url( r'^admin/', include( admin.site.urls ) ),
)

# These URLs accept the language prefix.
urlpatterns += i18n_patterns('',
  url(r'^$', 'wui.views.results'),
  url(r'^(?P<unit>\w{3,15})/', 'wui.views.results'),
)
<|endoftext|>


<commit_before>from doajtest.helpers import DoajTestCase
from parameterized import parameterized, param

from doajtest.fixtures import ApplicationFixtureFactory

from portality import models
from portality.bll import DOAJ
from portality.bll import exceptions

from werkzeug.datastructures import MultiDict
from portality.formcontext import formcontext

def load_formcontext_cases():
    application_source = ApplicationFixtureFactory.make_application_source()
    application = models.Suggestion(**application_source)

    application_form = ApplicationFixtureFactory.make_application_form_info()
    form_data = MultiDict(application_form)

    # generate the comparison functions
    application_publisher_source = comparator_closure(formcontext.PublisherUpdateRequest, True, True, False)
    application_publisher_form_data = comparator_closure(formcontext.PublisherUpdateRequest, True, True, True)

    return [
        param("unnown_type", "unknown", "publisher", raises=exceptions.NoSuchFormContext),
        param("application_unknown_role", "application", "unknown", none_result=True),
        param("application_publisher_source", "application", "publisher", source=application, comparator=application_publisher_source),
        param("application_publisher_form_data", "application", "publisher", form_data=form_data, source=application, comparator=application_publisher_form_data)
    ]

def comparator_closure(instance, form, source, form_data):
    def comparator(fc):
        assert isinstance(fc, instance)

        if form:
            assert fc.form is not None
        else:
            assert fc.form is None

        if source:
            assert fc.source is not None
        else:
            assert fc.source is None

        if form_data:
            assert fc.form_data is not None
        else:
            assert fc.form_data is None

    return comparator

class TestBLLFormContext(DoajTestCase):

    @parameterized.expand(load_formcontext_cases)
    def test_01_formcontext_factory(self, name, type, role, source=None, form_data=None, none_result=False, raises=None, comparator=None):
        doaj = DOAJ()

        if raises is not None:
            with self.assertRaises(raises):
                fc = doaj.formcontext(type, role, source=source, form_data=form_data)
        else:
            fc = doaj.formcontext(type, role, source=source, form_data=form_data)
            if none_result is True:
                assert fc is None
            else:
                if comparator is not None:
                    comparator(fc)
                else:
                    assert fc is not None<commit_msg>remove unnecessary formcontext test<commit_after><|endoftext|>


<commit_before>from uuid import uuid4 as uuid
import sys

from dino.environ import env
from dino.config import ApiActions
from dino.db.manager import AclManager

print("\n"*10)

if len(sys.argv) < 2 or sys.argv[1] not in {"list", "add"}:
    print("usage: DINO_ENVIRONMENT=<env> ./create_broadcast_groups.py list")
    print("usage: DINO_ENVIRONMENT=<env> ./create_broadcast_groups.py add <name> <acl_value>")
    print("\n"*10)
    sys.exit(1)

acl_manager = AclManager(env)

if sys.argv[1] == "list":
    room_acls = env.db.get_room_acls_for_action(ApiActions.AUTOJOIN)

    for room_id, acls in room_acls.items():
        # sometimes room_id is None, if no autojoin rooms exist
        if room_id is None or len(room_id.strip()) == 0:
            continue

        # acls = acl_manager.get_acls_room(room_id, encode_result=False)

        print("{}: {}".format(room_id, env.db.get_room_name(room_id)))
        print("type \t value".expandtabs(20))
        print("---- \t -----".expandtabs(20))

        if len(acls):
            for acl_type, acl_value in acls.items():
                print("{} \t {}".format(acl_type, acl_value).expandtabs(20))
        else:
            print('<no acls>')

        print()

    print("\n" * 10)
    sys.exit(0)

# add
new_acl_type = sys.argv[2]
new_acl_value = sys.argv[3]

room_id = str(uuid())
env.db.add_default_room(room_id)

channel_id = env.db.channel_for_room(room_id)
acl_manager.update_room_acl(channel_id, room_id, ApiActions.AUTOJOIN, new_acl_type, new_acl_value)

print("added room {} with acls {}={}".format(room_id, new_acl_type, new_acl_value))

print("\n"*10)<commit_msg>create all combinations<commit_after>from uuid import uuid4 as uuid
import sys

from dino.environ import env
from dino.config import ApiActions
from dino.db.manager import AclManager, RoomManager

print("\n"*10)

if len(sys.argv) < 2 or sys.argv[1] not in {"list", "add"}:
    print("usage: DINO_ENVIRONMENT=<env> ./create_broadcast_groups.py list")
    print("usage: DINO_ENVIRONMENT=<env> ./create_broadcast_groups.py add country=de,it,gb membership=normal,vip user_type=1,2,3")
    print("\n"*10)
    sys.exit(1)

acl_manager = AclManager(env)
room_manager = RoomManager(env)

room_acls = env.db.get_room_acls_for_action(ApiActions.AUTOJOIN)

if sys.argv[1] == "list":
    for room_id, acls in room_acls.items():
        if room_id is None or len(room_id.strip()) == 0:
            continue

        print("{}: {}".format(room_id, env.db.get_room_name(room_id)))
        print("type \t value".expandtabs(20))
        print("---- \t -----".expandtabs(20))
        if len(acls):
            for acl_type, acl_value in acls.items():
                print("{} \t {}".format(acl_type, acl_value).expandtabs(20))
        else:
            print('<no acls>')
        print()

    print("\n" * 10)

# add
else:
    channel_id = env.db.channel_for_room(room_acls.keys()[0])
    args = list()

    for arg in sys.argv[2:]:
        k, v = arg.split("=")
        v = v.split(",")
        args.append((k, v))

    for i in range(len(args[2][1])):
        for j in range(len(args[1][1])):
            for k in range(len(args[0][1])):
                room_id = str(uuid())

                # country=de,it,gb membership=normal,vip user_type=1,2,3
                room_name = "{}-{}-{}".format(args[0][1][k], args[1][1][j], args[2][1][i])
                room_manager.create_room(room_name, room_id, channel_id, "0")

                acl_manager.update_room_acl(channel_id, room_id, ApiActions.AUTOJOIN, args[0][0], args[0][1][k])
                acl_manager.update_room_acl(channel_id, room_id, ApiActions.AUTOJOIN, args[1][0], args[1][1][j])
                acl_manager.update_room_acl(channel_id, room_id, ApiActions.AUTOJOIN, args[2][0], args[2][1][i])

                print("added room {} with acls:".format(room_id))
                print("{}={}".format(args[0][0], args[0][1][k]))
                print("{}={}".format(args[1][0], args[1][1][j]))
                print("{}={}".format(args[2][0], args[2][1][i]))
                print()

    print("\n" * 10)
<|endoftext|>


<commit_before>#!/usr/bin/python2.7
import sys
import os
import time
from multiprocessing import Process, Lock, Queue, cpu_count
import struct
import numpy
from xml.dom.minidom import parse, parseString
import gdal
import osr
import gdalnumeric
from gdalconst import *

exitFlag = 0
queueLock = None
writeLock = None
workQueue = None
tasks = None

class Tasks(object):
	def __init__(self):
		self.tasks = []

	def add(self,task):
		self.tasks.append(task)

	def get(self,task):
		return self.tasks[task]

class RenderSubprocess(object):
    def __init__(self, processID, processName, queue, tasks):
        self.processID = processID
        self.processName = processName
        self.queue = queue
        self.tasks = tasks
        #Variable#
        self.strip = None
        self.strip_stretched = None
        self.task = None
        self.tries = 3

    def run(self):
    	while not exitFlag:
    		if self.strip is None:
    			queueLock.acquire()
        		if not workQueue.empty():
            			b, inBand, outBand, y0, y, r, t = self.task = self.tasks.get(self.queue.get())
            			queueLock.release()
            			#==#
            			if t==1:
			            	#print self.processName+" reading rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y:"+str(y)
            					print "r:"+str(r)

            			elif t==2:
			            	#print self.processName+" reading row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,(y0*r)+y,inBand.XSize,1,inBand.XSize,1)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y0:"+str(y0)
            					print "y:"+str(y)
            					print "r:"+str(r)
            					print "(y0*r)+y:"+str((y0*r)+y)

        		else:
            			queueLock.release()
            	elif self.strip_stretched is None:
            		self.strip_stretched = lut[self.strip]
        	elif (not self.strip is None) and (not self.strip_stretched is None):
        		if self.tries > 0:
        			writeLock.acquire()
        			b, inBand, outBand, y0, y, r, t = self.task
        			if t==1:
			            	#print self.processName+" writing rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
	            				outBand.WriteArray(self.strip_stretched,0,y*r)
            					self.strip = None
            					self.strip_stretched = None
            				except:
	            				self.tries = self.tries - 1
            					print "write failed.  "+str(self.tries)+" more tries."
            			elif t==2:
			            	#print self.processName+" writing row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					outBand.WriteArray(self.strip_stretched,0,(y0*r)+y)
            					self.strip = None
            					self.strip_stretched = None
            				except:
            					self.tries = self.tries -1
            					print "write failed.  "+str(self.tries)+" more tries."
            			writeLock.release()
            		else:
            			print "abandoning write"
            			self.strip = None
            			self.strip_stretched = None
            	else:
            	    print "abandoning task.  This should never happen."
            	    self.strip = None
            	    self.strip_stretched = None

        	time.sleep(1)

def execute(subprocess):
	subprocess.run()

class BreakPoint:

	def __init__(self, i = None, o = None):
		self.i = i #Should be double; even though it is most frequently an int
		self.o = o

class Line:

	def __init__(self, b0, b1):
		self.x0 = b0.i
		self.y0 = b0.o
		self.x1 = b1.i
		self.y1 = b1.o
		self.n = float(self.y1-self.y0)
		self.d = float(self.x1-self.x0)
		if self.d != 0:
			self.m = self.n/self.d
			self.b = self.y0 - self.m*self.x0
		else:
			self.m = None
			self.b = None

	def min(self):
		return self.x0

	def max(self):
		return self.x1

	def contains(self, x):
		return x>=self.x0 and x <=self.x1

	def calc(self, x):
		if self.d != 0:
			return int(round(self.m*x+self.b))
		elif self.d == 0 and self.y1 == self.y0:
			return self.y1
		else:
			return x

class Lines:

	def __init__(self, bps):
		self.lines = []
		for i in range(len(bps)-1):
			if (not (bps[i] is None)) and (not (bps[i+1] is None)):
				self.lines.append(Line(bps[i],bps[i+1]))

	def size(self):
		return len(self.lines)

	def min(self):
		return self.lines[0].min()

	def max(self):
		return self.lines[len(self.lines)-1].max()

	def calc(self, x):
		y = x
		for line in self.lines:
			if line.contains(x):
				y = line.calc(x)
				break
		return y

class LookUpTable:

	def __init__(self,bps):
		self.lines = Lines(bps)
		self.table = []
		self.min = int(self.lines.min())
		self.max = int(self.lines.max())

		for i in range(0,256):
			self.table.append(self.lines.calc(i))

class LookUpTable2:

	def __init__(self,bps_red,bps_green,bps_blue):
		self.lines_red = Lines(bps_red)
		self.lines_green = Lines(bps_green)
		self.lines_blue = Lines(bps_blue)
		self.table = []
		for i in range(0,256):
			self.table.append(int((self.lines_red.calc(i)+self.lines_green.calc(i)+self.lines_blue.calc(i))/3.0))

class LookUpTables:

	def __init__ (self, filename):
		self.bps_red = None
		self.bps_green = None
		self.bps_blue = None
		self.tables = None
		self.table = None

		self.file = None
		self.valid = True

		self.init_breakpoints(filename)
		self.init_tables()

	def init_breakpoints(self,filename):
                self.bps_red = []
                self.bps_green = []
                self.bps_blue = []

		if filename.endswith(".cbp") or filename.endswith(".txt"):
			lines = None
			file = open(filename,"r")
			with open(filename) as f:
				lines = f.readlines()

			band = -1
			for line in lines:
				line = line.strip()
				if line=="":
					continue
				elif line=="rgbversion8.3":
					continue
				elif line.startswith("#"):
					continue
				elif line=="RGB":
					band = band+1
				elif line.startswith("Band"):
					band = int(line.split()[1].strip(' \t\n\r'))-1
				elif line.startswith("Break") or line.startswith("-"):
					if band==0:
						self.bps_red.append(None)
					elif band==1:
						self.bps_green.append(None)
					elif band==2:
						self.bps_blue.append(None)
				else:
					terms = line.split()
					bp = BreakPoint(float(terms[0].strip(' \t\n\r')),float(terms[1].strip(' \t\n\r')))
					if band==0:
						self.bps_red.append(bp)
					elif band==1:
						self.bps_green.append(bp)
					elif band==2:
						self.bps_blue.append(bp)

		elif filename.endswith(".qml"):
			file = open(filename,"r")
			dom = parse(file)
			minRed = -1
			maxRed = -1
			minGreen = -1
			maxGreen = -1
			minBlue = -1
			maxBlue = -1
			for nRasterRenderer in dom.getElementsByTagName("rasterrenderer"):
				for node in nRasterRenderer.childNodes:
					#print node.toxml()
					if node.nodeType != node.TEXT_NODE:
						if node.tagName=="redContrastEnhancement":
							minRed = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxRed = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="greenContrastEnhancement":
							minGreen = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxGreen = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="blueContrastEnhancement":
							minBlue = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxBlue = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
			if minRed!=-1 and maxRed!=-1:
				self.bps_red.extend(self.buildBreakPoints(minRed,maxRed))
			if minGreen!=-1 and maxGreen!=-1:
				self.bps_green.extend(self.buildBreakPoints(minGreen,maxGreen))
                        if minBlue!=-1 and maxBlue!=-1:
				self.bps_blue.extend(self.buildBreakPoints(minBlue,maxBlue))

	def buildBreakPoints(self,minValue,maxValue):
		bps = []
		bps.append(BreakPoint(0.0,0.0))
		bps.append(BreakPoint(minValue,0.0))
		bps.append(BreakPoint(maxValue,255.0))
		bps.append(BreakPoint(255.0,255.0))
		return bps

	def init_tables(self):
		self.tables = []
		self.tables.append(LookUpTable(self.bps_red))
		self.tables.append(LookUpTable(self.bps_green))
		self.tables.append(LookUpTable(self.bps_blue))
		self.tables.append(LookUpTable2(self.bps_red,self.bps_green,self.bps_blue))

	def isValid(self):
		return self.valid;

def main():
	if(len(sys.argv)==6):
		inputFile = sys.argv[1]
		breakPointsFile = sys.argv[2]
		outputFile = sys.argv[3]
		rows = int(sys.argv[4])
		numberOfThreads = int(sys.argv[5])
		if numberOfThreads > 0:
			if(os.path.exists(inputFile) and os.path.exists(breakPointsFile)):
				if(not os.path.exists(outputFile)):
					inputDataset = gdal.Open(inputFile,GA_ReadOnly)
					lookUpTables = LookUpTables(breakPointsFile)
					if ((not inputDataset is None) and (lookUpTables.isValid())):
						outputFormat = "HFA"
						numberOfBands = 3
						w = inputDataset.RasterXSize
						h = inputDataset.RasterYSize
						outputDataset = initDataset(outputFile,outputFormat,w,h,numberOfBands)
						outputDataset.SetGeoTransform(list(inputDataset.GetGeoTransform()))
						outputDataset.SetProjection(inputDataset.GetProjection())

						if numberOfThreads == 1:
							for b in range(numberOfBands):
								print "Stretching Band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)

								r = rows
								for y in range(int(inBand.YSize/r)):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)],0,y*r)

								y0 = inBand.YSize/rows
								for y in range(inBand.YSize%r):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y0+y,inBand.XSize,1,inBand.XSize,1)],0,y0+y)
						elif numberOfThreads > 1:
							print "not fully implemented yet"
                                                        global exitFlag
                                                        global queueLock
                                                        global writeLock
                                                        global workQueue
                                                        global tasks
							#
                                                        exitFlag = 0
                                                        queueLock = Lock()
                                                        writeLock = Lock()
                                                        workQueue = Queue(0)
                                                        tasks = Tasks()
							#
							for b in range(inputBands):
								print "Adding tasks for band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)
								y0 = int(inBand.YSize/r)
								for y in range(int(inBand.YSize/r)):
									task = b+1, inBand, outBand, lut, y0, y, r, 1
									tasks.add(task)
								for y in range(inBand.YSize%r):
									task = b+1, inBand, outBand, lut, y0, y, r, 2
									tasks.add(task)

						inputDataset = None
						outputDataset = None
					else:
						print "Error Opening File"
				else:
					print "Output file already exists"
			else:
				print "Input file does not exist."
		else:
			print "You need at least 1 thread."
	else:
		print "Usage: cybergis-script-stretch.py <input_file> <breakpoints_file> <output_file> <rows> <threads>"

def initDataset(outputFile,f,w,h,b):
    driver = gdal.GetDriverByName(f)
    metadata = driver.GetMetadata()
    return driver.Create(outputFile,w,h,b,gdal.GDT_Byte,['ALPHA=YES'])

def initProcesses(count)
    print str(cpu_count())+" CPUs are available."
    processes = []
    processID = 1
    for processID in range(count):
        subprocess = RenderSubprocess(processID, ("Thread "+str(processID)), workQueue, tasks)
        process = Process(target=execute,args=(subprocess,))
        process.start()
        processes.append(process)
        processID += 1
    print "Initialized "+str(count)+" threads."
    return processes

main()
<commit_msg>Update cybergis-script-stretch.py<commit_after>#!/usr/bin/python2.7
import sys
import os
import time
from multiprocessing import Process, Lock, Queue, cpu_count
import struct
import numpy
from xml.dom.minidom import parse, parseString
import gdal
import osr
import gdalnumeric
from gdalconst import *

exitFlag = 0
queueLock = None
writeLock = None
workQueue = None
tasks = None

class Tasks(object):
	def __init__(self):
		self.tasks = []

	def add(self,task):
		self.tasks.append(task)

	def get(self,task):
		return self.tasks[task]

class RenderSubprocess(object):
    def __init__(self, processID, processName, queue, tasks):
        self.processID = processID
        self.processName = processName
        self.queue = queue
        self.tasks = tasks
        #Variable#
        self.strip = None
        self.strip_stretched = None
        self.task = None
        self.tries = 3

    def run(self):
    	while not exitFlag:
    		if self.strip is None:
    			queueLock.acquire()
        		if not workQueue.empty():
            			b, inBand, outBand, lut, y0, y, r, t = self.task = self.tasks.get(self.queue.get())
            			queueLock.release()
            			#==#
            			if t==1:
			            	#print self.processName+" reading rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y:"+str(y)
            					print "r:"+str(r)

            			elif t==2:
			            	#print self.processName+" reading row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					self.strip = inBand.ReadAsArray(0,(y0*r)+y,inBand.XSize,1,inBand.XSize,1)
            					self.tries = 10
            				except:
            					self.strip = None
            					self.strip_stretched = None
            					print "read failed.  will try again later."
            					print "y0:"+str(y0)
            					print "y:"+str(y)
            					print "r:"+str(r)
            					print "(y0*r)+y:"+str((y0*r)+y)

        		else:
            			queueLock.release()
            	elif (not self.strip is None) and (self.strip_stretched is None):
            		b, inBand, outBand, lut, y0, y, r, t = self.task
            		self.strip_stretched = lut[self.strip]
        	elif (not self.strip is None) and (not self.strip_stretched is None):
        		if self.tries > 0:
        			writeLock.acquire()
        			b, inBand, outBand, lut, y0, y, r, t = self.task
        			if t==1:
			            	#print self.processName+" writing rows "+str(y*r)+" to "+str((y*r)+r-1)+" in band "+str(b)+"."
            				try:
	            				outBand.WriteArray(self.strip_stretched,0,y*r)
            					self.strip = None
            					self.strip_stretched = None
            				except:
	            				self.tries = self.tries - 1
            					print "write failed.  "+str(self.tries)+" more tries."
            			elif t==2:
			            	#print self.processName+" writing row "+str((y0*r)+y)+" in band "+str(b)+"."
            				try:
            					outBand.WriteArray(self.strip_stretched,0,(y0*r)+y)
            					self.strip = None
            					self.strip_stretched = None
            				except:
            					self.tries = self.tries -1
            					print "write failed.  "+str(self.tries)+" more tries."
            			writeLock.release()
            		else:
            			print "abandoning write"
            			self.strip = None
            			self.strip_stretched = None
            	else:
            	    print "abandoning task.  This should never happen."
            	    self.strip = None
            	    self.strip_stretched = None

        	time.sleep(1)

def execute(subprocess):
	subprocess.run()

class BreakPoint:

	def __init__(self, i = None, o = None):
		self.i = i #Should be double; even though it is most frequently an int
		self.o = o

class Line:

	def __init__(self, b0, b1):
		self.x0 = b0.i
		self.y0 = b0.o
		self.x1 = b1.i
		self.y1 = b1.o
		self.n = float(self.y1-self.y0)
		self.d = float(self.x1-self.x0)
		if self.d != 0:
			self.m = self.n/self.d
			self.b = self.y0 - self.m*self.x0
		else:
			self.m = None
			self.b = None

	def min(self):
		return self.x0

	def max(self):
		return self.x1

	def contains(self, x):
		return x>=self.x0 and x <=self.x1

	def calc(self, x):
		if self.d != 0:
			return int(round(self.m*x+self.b))
		elif self.d == 0 and self.y1 == self.y0:
			return self.y1
		else:
			return x

class Lines:

	def __init__(self, bps):
		self.lines = []
		for i in range(len(bps)-1):
			if (not (bps[i] is None)) and (not (bps[i+1] is None)):
				self.lines.append(Line(bps[i],bps[i+1]))

	def size(self):
		return len(self.lines)

	def min(self):
		return self.lines[0].min()

	def max(self):
		return self.lines[len(self.lines)-1].max()

	def calc(self, x):
		y = x
		for line in self.lines:
			if line.contains(x):
				y = line.calc(x)
				break
		return y

class LookUpTable:

	def __init__(self,bps):
		self.lines = Lines(bps)
		self.table = []
		self.min = int(self.lines.min())
		self.max = int(self.lines.max())

		for i in range(0,256):
			self.table.append(self.lines.calc(i))

class LookUpTable2:

	def __init__(self,bps_red,bps_green,bps_blue):
		self.lines_red = Lines(bps_red)
		self.lines_green = Lines(bps_green)
		self.lines_blue = Lines(bps_blue)
		self.table = []
		for i in range(0,256):
			self.table.append(int((self.lines_red.calc(i)+self.lines_green.calc(i)+self.lines_blue.calc(i))/3.0))

class LookUpTables:

	def __init__ (self, filename):
		self.bps_red = None
		self.bps_green = None
		self.bps_blue = None
		self.tables = None
		self.table = None

		self.file = None
		self.valid = True

		self.init_breakpoints(filename)
		self.init_tables()

	def init_breakpoints(self,filename):
                self.bps_red = []
                self.bps_green = []
                self.bps_blue = []

		if filename.endswith(".cbp") or filename.endswith(".txt"):
			lines = None
			file = open(filename,"r")
			with open(filename) as f:
				lines = f.readlines()

			band = -1
			for line in lines:
				line = line.strip()
				if line=="":
					continue
				elif line=="rgbversion8.3":
					continue
				elif line.startswith("#"):
					continue
				elif line=="RGB":
					band = band+1
				elif line.startswith("Band"):
					band = int(line.split()[1].strip(' \t\n\r'))-1
				elif line.startswith("Break") or line.startswith("-"):
					if band==0:
						self.bps_red.append(None)
					elif band==1:
						self.bps_green.append(None)
					elif band==2:
						self.bps_blue.append(None)
				else:
					terms = line.split()
					bp = BreakPoint(float(terms[0].strip(' \t\n\r')),float(terms[1].strip(' \t\n\r')))
					if band==0:
						self.bps_red.append(bp)
					elif band==1:
						self.bps_green.append(bp)
					elif band==2:
						self.bps_blue.append(bp)

		elif filename.endswith(".qml"):
			file = open(filename,"r")
			dom = parse(file)
			minRed = -1
			maxRed = -1
			minGreen = -1
			maxGreen = -1
			minBlue = -1
			maxBlue = -1
			for nRasterRenderer in dom.getElementsByTagName("rasterrenderer"):
				for node in nRasterRenderer.childNodes:
					#print node.toxml()
					if node.nodeType != node.TEXT_NODE:
						if node.tagName=="redContrastEnhancement":
							minRed = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxRed = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="greenContrastEnhancement":
							minGreen = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxGreen = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
						elif node.tagName=="blueContrastEnhancement":
							minBlue = float(node.getElementsByTagName("minValue")[0].firstChild.nodeValue)
							maxBlue = float(node.getElementsByTagName("maxValue")[0].firstChild.nodeValue)
			if minRed!=-1 and maxRed!=-1:
				self.bps_red.extend(self.buildBreakPoints(minRed,maxRed))
			if minGreen!=-1 and maxGreen!=-1:
				self.bps_green.extend(self.buildBreakPoints(minGreen,maxGreen))
                        if minBlue!=-1 and maxBlue!=-1:
				self.bps_blue.extend(self.buildBreakPoints(minBlue,maxBlue))

	def buildBreakPoints(self,minValue,maxValue):
		bps = []
		bps.append(BreakPoint(0.0,0.0))
		bps.append(BreakPoint(minValue,0.0))
		bps.append(BreakPoint(maxValue,255.0))
		bps.append(BreakPoint(255.0,255.0))
		return bps

	def init_tables(self):
		self.tables = []
		self.tables.append(LookUpTable(self.bps_red))
		self.tables.append(LookUpTable(self.bps_green))
		self.tables.append(LookUpTable(self.bps_blue))
		self.tables.append(LookUpTable2(self.bps_red,self.bps_green,self.bps_blue))

	def isValid(self):
		return self.valid;

def main():
	if(len(sys.argv)==6):
		inputFile = sys.argv[1]
		breakPointsFile = sys.argv[2]
		outputFile = sys.argv[3]
		rows = int(sys.argv[4])
		numberOfThreads = int(sys.argv[5])
		if numberOfThreads > 0:
			if(os.path.exists(inputFile) and os.path.exists(breakPointsFile)):
				if(not os.path.exists(outputFile)):
					inputDataset = gdal.Open(inputFile,GA_ReadOnly)
					lookUpTables = LookUpTables(breakPointsFile)
					if ((not inputDataset is None) and (lookUpTables.isValid())):
						outputFormat = "HFA"
						numberOfBands = 3
						w = inputDataset.RasterXSize
						h = inputDataset.RasterYSize
						outputDataset = initDataset(outputFile,outputFormat,w,h,numberOfBands)
						outputDataset.SetGeoTransform(list(inputDataset.GetGeoTransform()))
						outputDataset.SetProjection(inputDataset.GetProjection())

						if numberOfThreads == 1:
							for b in range(numberOfBands):
								print "Stretching Band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)

								r = rows
								for y in range(int(inBand.YSize/r)):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y*r,inBand.XSize,r,inBand.XSize,r)],0,y*r)

								y0 = inBand.YSize/rows
								for y in range(inBand.YSize%r):
									outBand.WriteArray(lut[inBand.ReadAsArray(0,y0+y,inBand.XSize,1,inBand.XSize,1)],0,y0+y)
						elif numberOfThreads > 1:
							print "not fully implemented yet"
                                                        global exitFlag
                                                        global queueLock
                                                        global writeLock
                                                        global workQueue
                                                        global tasks
							#
                                                        exitFlag = 0
                                                        queueLock = Lock()
                                                        writeLock = Lock()
                                                        workQueue = Queue(0)
                                                        tasks = Tasks()
							#
							for b in range(inputBands):
								print "Adding tasks for band "+str(b+1)
								lut = numpy.array(lookUpTables.tables[b].table)
								inBand = inputDataset.GetRasterBand(b+1)
								outBand = outputDataset.GetRasterBand(b+1)
								y0 = int(inBand.YSize/r)
								for y in range(int(inBand.YSize/r)):
									task = b+1, inBand, outBand, lut, y0, y, r, 1
									tasks.add(task)
								for y in range(inBand.YSize%r):
									task = b+1, inBand, outBand, lut, y0, y, r, 2
									tasks.add(task)

						inputDataset = None
						outputDataset = None
					else:
						print "Error Opening File"
				else:
					print "Output file already exists"
			else:
				print "Input file does not exist."
		else:
			print "You need at least 1 thread."
	else:
		print "Usage: cybergis-script-stretch.py <input_file> <breakpoints_file> <output_file> <rows> <threads>"

def initDataset(outputFile,f,w,h,b):
    driver = gdal.GetDriverByName(f)
    metadata = driver.GetMetadata()
    return driver.Create(outputFile,w,h,b,gdal.GDT_Byte,['ALPHA=YES'])

def initProcesses(count)
    print str(cpu_count())+" CPUs are available."
    processes = []
    processID = 1
    for processID in range(count):
        subprocess = RenderSubprocess(processID, ("Thread "+str(processID)), workQueue, tasks)
        process = Process(target=execute,args=(subprocess,))
        process.start()
        processes.append(process)
        processID += 1
    print "Initialized "+str(count)+" threads."
    return processes

main()
<|endoftext|>


<commit_before>#ebtel_dem_analysis_main.py

#Will Barnes
#15 May 2015

#Import needed modules
import sys
import os
import argparse
import numpy as np
sys.path.append('/home/wtb2/Documents/EBTEL_analysis/src/')
import ebtel_dem as ebd
import ebtel_plot_em as ebpe
import ebtel_plot as ebp

#Declare parser object
parser = argparse.ArgumentParser(description='Script that performs DEM analysis for EBTEL-2fluid runs.')
#Add arguments to parser
parser.add_argument("-s","--species",help="Species to which the heating was applied for particular run.")
parser.add_argument("--root_dir",help="Root directory from which EBTEL output will be read.")
parser.add_argument("--root_dir_figs",help="Root directory to which figures will be output.")
#Declare the parser dictionary
args = parser.parse_args()

#set root directory
if args.root_dir:
    root_dir = args.root_dir
else:
    root_dir = '/data/datadrive2/EBTEL-2fluid_runs/'

if args.root_dir_figs:
    root_dir_figs = args.root_dir_figs
else:
    root_dir_figs = '/data/datadrive1/EBTEL-2fluid_figs/'

#set figure file format
figdir = '%s_heating_runs/alpha%s/'
figname = 'ebtel_L%.1f_tpulse%.1f_alpha%s_%s_heating'

#make parameter vectors
loop_length = np.array([20.0,40.0,60.0,80.0,100.0,120.0])
alpha = ['uniform',1.5,2.0,2.5]
#make waiting time vector
Tn = np.arange(250,5250,250)

#set limits on hot/cool slope calculation
slope_limits = {'cool_lower':6.0,'cool_upper':6.6,'hot_lower':6.7,'hot_upper':7.3}
t_cool_static = np.linspace(slope_limits['cool_lower'],slope_limits['cool_upper'],1000)
t_hot_static = np.linspace(slope_limits['hot_lower'],slope_limits['hot_upper'],1000)

#function to calculate variable hot slopes
def calc_upper_hot_lim(mean_temp,mean_em,delta_t):
    ninf_i = np.where(np.isinf(mean_em) == False) #find non-inf indices
    max_i = np.argmax(mean_em) #find index corresponding to max value
    hot_i = ninf_i[0][np.where(ninf_i[0]>max_i)] #indices for hot branch
    em_hot = mean_em[hot_i] #hot branch em
    delta_em_hot = np.fabs(np.diff(em_hot)) #delta(em) of hot branch
    delta_i = np.where(delta_em_hot>0.5)[0][0]
    lim_i = hot_i[delta_i - 1]-1
    t_upper = mean_temp[lim_i]
    t_lower = t_upper - delta_t
    return t_lower,t_upper

#set static parameters
tpulse = 100.0
solver = 'rka4'
delta_lim = 0.4

#declare instance of Plotter class
surf_plot = ebp.Plotter(format='pdf',fs=18.0,figsize=(8,8),dpi=1000)

#iterate over variable parameters
for i in range(len(alpha)):
    temp_max_save = []
    em_max_save = []
    for j in  range(len(loop_length)):
        #print status
        print("Processing L = %.1f, alpha = %s"%(loop_length[j],str(alpha[i])))
        #get data
        if loop_length[j] == 20.0 or alpha[i] == 'uniform':
            solver = 'euler'
        else:
            solver = 'rka4'

        dema = ebd.DEMAnalyzer(root_dir,args.species,alpha[i],loop_length[j],tpulse,solver,Tn=Tn,slope_limits=slope_limits)
        dema.process_raw()
        dema.em_statistics()
        #variable limit configuration and temperature fit array
        t_cool,t_hot = [],[]
        if alpha[i] is 'uniform' or args.species is 'ion':
            [(t_cool.append(t_cool_static),t_hot.append(t_hot_static)) for k in range(len(dema.temp_mean))]
            dema.many_slopes()
        else:
            hot_lower,hot_upper = [],[]
            cool_lower,cool_upper = [],[]
            for k in range(len(dema.temp_mean)):
                l,u = calc_upper_hot_lim(dema.temp_mean[k],dema.em_mean[k],delta_lim)
                hot_lower.append(l),hot_upper.append(u)
                cool_lower.append(slope_limits['cool_lower']),cool_upper.append(slope_limits['cool_upper'])
                t_cool.append(t_cool_static)
                t_hot.append(np.linspace(l,u,len(t_cool_static)))

            dema.many_slopes(slope_limits={'cool_lower':cool_lower,'cool_upper':cool_upper,'hot_lower':hot_lower,'hot_upper':hot_upper})

        dema.slope_statistics()
        dema.find_em_max()
        temp_max_save.append([np.mean(tmax) for tmax in dema.temp_max])
        em_max_save.append([np.mean(emmax) for emmax in dema.em_max])
        #build figure names and make sure directories exist
        if not os.path.exists(root_dir_figs + figdir%(args.species,str(alpha[i]))):
            os.makedirs(figdir%(root_dir_figs + args.species,str(alpha[i])))

        figname_temp = figdir%(args.species,str(alpha[i]))+figname%(loop_length[j],tpulse,str(alpha[i]),args.species)
        #plot data
        demp = ebpe.DEMPlotter(dema.temp_em,dema.em,alpha[i],Tn=Tn,format='pdf',fs=18.0,figsize=(8,8),dpi=1000)
        demp.plot_em_max(dema.temp_max,dema.em_max,print_fig_filename=root_dir_figs + figname_temp + '_TmaxVTn')
        demp.figsize = (demp.figsize[0],3/4*demp.figsize[1])
        demp.plot_em_slopes(dema.a_cool_mean,dema.a_cool_std,dema.a_hot_mean,dema.a_hot_std,print_fig_filename=root_dir_figs + figname_temp + '_hs_compare')
        demp.figsize = (demp.figsize[0],4/3*demp.figsize[1])
        fit_lines = {'t_cool':t_cool,'a_cool':dema.a_cool_mean,'b_cool':dema.b_cool_mean,'t_hot':t_hot,'a_hot':dema.a_hot_mean,'b_hot':dema.b_hot_mean}
        demp.plot_em_curves(dema.temp_mean,dema.em_mean,fit_lines=fit_lines,print_fig_filename=root_dir_figs+figname_temp+'_dem')
        #plot all em curves for given tn
        if alpha[i] is not 'uniform':
            if not os.path.exists(root_dir_figs+figname_temp+'_dem_mc/'):
                os.makedirs(root_dir_figs+figname_temp+'_dem_mc/')

            for k in range(len(Tn)):
                demp.plot_em_curve(k,print_fig_filename=root_dir_figs + figname_temp + '_dem_mc/' + figname%(loop_length[j],tpulse,str(alpha[i]),args.species) + '_'+str(k) + '_dem')

    #build surface plot
    surf_plot.plot_surface(Tn,loop_length,temp_max_save,vmin=6.0,vmax=6.8,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'$T_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 't_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
    surf_plot.plot_surface(Tn,loop_length,em_max_save,vmin=26.0,vmax=30.0,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'EM$_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 'em_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
<commit_msg>fix for switching limits in case of ion heating<commit_after>#ebtel_dem_analysis_main.py

#Will Barnes
#15 May 2015

#Import needed modules
import sys
import os
import argparse
import numpy as np
sys.path.append('/home/wtb2/Documents/EBTEL_analysis/src/')
import ebtel_dem as ebd
import ebtel_plot_em as ebpe
import ebtel_plot as ebp

#Declare parser object
parser = argparse.ArgumentParser(description='Script that performs DEM analysis for EBTEL-2fluid runs.')
#Add arguments to parser
parser.add_argument("-s","--species",help="Species to which the heating was applied for particular run.")
parser.add_argument("--root_dir",help="Root directory from which EBTEL output will be read.")
parser.add_argument("--root_dir_figs",help="Root directory to which figures will be output.")
#Declare the parser dictionary
args = parser.parse_args()

#set root directory
if args.root_dir:
    root_dir = args.root_dir
else:
    root_dir = '/data/datadrive2/EBTEL-2fluid_runs/'

if args.root_dir_figs:
    root_dir_figs = args.root_dir_figs
else:
    root_dir_figs = '/data/datadrive1/EBTEL-2fluid_figs/'

#set figure file format
figdir = '%s_heating_runs/alpha%s/'
figname = 'ebtel_L%.1f_tpulse%.1f_alpha%s_%s_heating'

#make parameter vectors
loop_length = np.array([20.0,40.0,60.0,80.0,100.0,120.0])
alpha = ['uniform',1.5,2.0,2.5]
#make waiting time vector
Tn = np.arange(250,5250,250)

#set limits on hot/cool slope calculation
slope_limits = {'cool_lower':6.0,'cool_upper':6.6,'hot_lower':6.7,'hot_upper':7.3}
t_cool_static = np.linspace(slope_limits['cool_lower'],slope_limits['cool_upper'],1000)
t_hot_static = np.linspace(slope_limits['hot_lower'],slope_limits['hot_upper'],1000)

#function to calculate variable hot slopes
def calc_upper_hot_lim(mean_temp,mean_em,delta_t):
    ninf_i = np.where(np.isinf(mean_em) == False) #find non-inf indices
    max_i = np.argmax(mean_em) #find index corresponding to max value
    hot_i = ninf_i[0][np.where(ninf_i[0]>max_i)] #indices for hot branch
    em_hot = mean_em[hot_i] #hot branch em
    delta_em_hot = np.fabs(np.diff(em_hot)) #delta(em) of hot branch
    delta_i = np.where(delta_em_hot>0.5)[0][0]
    lim_i = hot_i[delta_i - 1]-1
    t_upper = mean_temp[lim_i]
    t_lower = t_upper - delta_t
    return t_lower,t_upper

#set static parameters
tpulse = 100.0
solver = 'rka4'
delta_lim = 0.4

#declare instance of Plotter class
surf_plot = ebp.Plotter(format='pdf',fs=18.0,figsize=(8,8),dpi=1000)

#iterate over variable parameters
for i in range(len(alpha)):
    temp_max_save = []
    em_max_save = []
    for j in  range(len(loop_length)):
        #print status
        print("Processing L = %.1f, alpha = %s"%(loop_length[j],str(alpha[i])))
        #get data
        if loop_length[j] == 20.0 or alpha[i] == 'uniform':
            solver = 'euler'
        else:
            solver = 'rka4'

        dema = ebd.DEMAnalyzer(root_dir,args.species,alpha[i],loop_length[j],tpulse,solver,Tn=Tn,slope_limits=slope_limits)
        dema.process_raw()
        dema.em_statistics()
        #variable limit configuration and temperature fit array
        t_cool,t_hot = [],[]
        if alpha[i] == 'uniform' or args.species == 'ion':
            [(t_cool.append(t_cool_static),t_hot.append(t_hot_static)) for k in range(len(dema.temp_mean))]
            dema.many_slopes()
        else:
            hot_lower,hot_upper = [],[]
            cool_lower,cool_upper = [],[]
            for k in range(len(dema.temp_mean)):
                l,u = calc_upper_hot_lim(dema.temp_mean[k],dema.em_mean[k],delta_lim)
                hot_lower.append(l),hot_upper.append(u)
                cool_lower.append(slope_limits['cool_lower']),cool_upper.append(slope_limits['cool_upper'])
                t_cool.append(t_cool_static)
                t_hot.append(np.linspace(l,u,len(t_cool_static)))

            dema.many_slopes(slope_limits={'cool_lower':cool_lower,'cool_upper':cool_upper,'hot_lower':hot_lower,'hot_upper':hot_upper})

        dema.slope_statistics()
        dema.find_em_max()
        temp_max_save.append([np.mean(tmax) for tmax in dema.temp_max])
        em_max_save.append([np.mean(emmax) for emmax in dema.em_max])
        #build figure names and make sure directories exist
        if not os.path.exists(root_dir_figs + figdir%(args.species,str(alpha[i]))):
            os.makedirs(figdir%(root_dir_figs + args.species,str(alpha[i])))

        figname_temp = figdir%(args.species,str(alpha[i]))+figname%(loop_length[j],tpulse,str(alpha[i]),args.species)
        #plot data
        demp = ebpe.DEMPlotter(dema.temp_em,dema.em,alpha[i],Tn=Tn,format='pdf',fs=18.0,figsize=(8,8),dpi=1000)
        demp.plot_em_max(dema.temp_max,dema.em_max,print_fig_filename=root_dir_figs + figname_temp + '_TmaxVTn')
        demp.figsize = (demp.figsize[0],3/4*demp.figsize[1])
        demp.plot_em_slopes(dema.a_cool_mean,dema.a_cool_std,dema.a_hot_mean,dema.a_hot_std,print_fig_filename=root_dir_figs + figname_temp + '_hs_compare')
        demp.figsize = (demp.figsize[0],4/3*demp.figsize[1])
        fit_lines = {'t_cool':t_cool,'a_cool':dema.a_cool_mean,'b_cool':dema.b_cool_mean,'t_hot':t_hot,'a_hot':dema.a_hot_mean,'b_hot':dema.b_hot_mean}
        demp.plot_em_curves(dema.temp_mean,dema.em_mean,fit_lines=fit_lines,print_fig_filename=root_dir_figs+figname_temp+'_dem')
        #plot all em curves for given tn
        if alpha[i] is not 'uniform':
            if not os.path.exists(root_dir_figs+figname_temp+'_dem_mc/'):
                os.makedirs(root_dir_figs+figname_temp+'_dem_mc/')

            for k in range(len(Tn)):
                demp.plot_em_curve(k,print_fig_filename=root_dir_figs + figname_temp + '_dem_mc/' + figname%(loop_length[j],tpulse,str(alpha[i]),args.species) + '_'+str(k) + '_dem')

    #build surface plot
    surf_plot.plot_surface(Tn,loop_length,temp_max_save,vmin=6.0,vmax=6.8,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'$T_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 't_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
    surf_plot.plot_surface(Tn,loop_length,em_max_save,vmin=26.0,vmax=30.0,ylab=r'$L$ (Mm)',xlab=r'$T_n$ (s)',plot_title=r'EM$_{max}$ Surface, $\alpha=$' + str(alpha[i]),print_fig_filename=root_dir_figs + figdir%(args.species,str(alpha[i])) + 'em_max_surface_' + args.species + '_alpha' + str(alpha[i]) + '_tpulse' + str(tpulse) + '_' + solver)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
A Python script to set a font's OS/2 usWeightClass value so it conforms to
the Google Fonts specification. The font's filename is used to determine
the correct value.
"""
from __future__ import print_function
from gftools.fix import fix_weight_class, FontFixer
import sys

def main(font_path):
    FontFixer(font_path, verbose=True, fixes=[fix_weight_class]).fix()

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Please include a path to a font")
    else:
        main(sys.argv[1])

<commit_msg>The docs were a lie<commit_after>#!/usr/bin/env python3
# Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
A Python script to set a font's OS/2 usWeightClass value so it conforms to
the Google Fonts specification. The font's style name in the name record is
used to determine the correct value.
"""
from __future__ import print_function
from gftools.fix import fix_weight_class, FontFixer
import sys

def main(font_path):
    FontFixer(font_path, verbose=True, fixes=[fix_weight_class]).fix()

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Please include a path to a font")
    else:
        main(sys.argv[1])

<|endoftext|>