<commit_before>import collections
import logging

logger = logging.getLogger(__name__)

# noinspection PyUnresolvedReferences
class History(object):
    """
    History data.

    :ivar _history: raw(ish) history events
    :type _history: swf.models.history.History
    :ivar _activities: activity events
    :type _activities: collections.OrderedDict[str, dict[str, Any]]
    :ivar _child_workflows: child workflow events
    :type _child_workflows: collections.OrderedDict[str, dict[str, Any]]
    :ivar _external_workflows: external workflow events
    :type _external_workflows: collections.OrderedDict[str, dict[str, Any]]
    :ivar _signals: activity events
    :type _signals: collections.OrderedDict[str, dict[str, Any]]
    :ivar _tasks: ordered list of tasks/etc
    :type _tasks: list[dict[str, Any]]
    """

    def __init__(self, history):
        self._history = history
        self._activities = collections.OrderedDict()
        self._child_workflows = collections.OrderedDict()
        self._external_workflows = collections.OrderedDict()
        self._signals = collections.OrderedDict()
        self._tasks = []

    @property
    def activities(self):
        """
        :return: activities
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._activities

    @property
    def child_workflows(self):
        """
        :return: child WFs
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._child_workflows

    @property
    def external_workflows(self):
        """
        :return: external WFs
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._external_workflows

    @property
    def signals(self):
        """
        :return: signals
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._signals

    @property
    def tasks(self):
        return self._tasks

    @property
    def events(self):
        """

        :return:
        :rtype: list[swf.models.event.Event]
        """
        return self._history.events

    def parse_activity_event(self, events, event):
        """
        Aggregate all the attributes of an activity in a single entry.

        :param events:
        :type events: list[swf.models.event.Event]
        :param event:
        :type event: swf.models.event.Event
        """

        def get_activity():
            """
            Return a reference to the corresponding activity.
            :return: mutable activity
            :rtype: dict[str, Any]
            """
            scheduled_event = events[event.scheduled_event_id - 1]
            return self._activities[scheduled_event.activity_id]

        if event.state == 'scheduled':
            activity = {
                'type': 'activity',
                'id': event.activity_id,
                'name': event.activity_type['name'],
                'version': event.activity_type['version'],
                'state': event.state,
                'scheduled_id': event.id,
                'scheduled_timestamp': event.timestamp,
                'input': event.input,
                'task_list': event.task_list['name'],
            }
            if event.activity_id not in self._activities:
                self._activities[event.activity_id] = activity
                self._tasks.append(activity)
            else:
                # When the executor retries a task, it schedules it again.
                # We have to take care of not overriding some values set by the
                # previous execution of the task such as the number of retries
                # in ``retry``.  As the state of the event mutates, it
                # corresponds to the last execution.
                self._activities[event.activity_id].update(activity)
        elif event.state == 'schedule_failed':
            activity = {
                'type': 'activity',
                'state': event.state,
                'cause': event.cause,
                'activity_type': event.activity_type.copy(),
                'schedule_failed_timestamp': event.timestamp,
            }

            if event.activity_id not in self._activities:
                self._activities[event.activity_id] = activity
                self._tasks.append(activity)
            else:
                # When the executor retries a task, it schedules it again.
                # We have to take care of not overriding some values set by the
                # previous execution of the task such as the number of retries
                # in ``retry``.  As the state of the event mutates, it
                # corresponds to the last execution.
                self._activities[event.activity_id].update(activity)

        elif event.state == 'started':
            activity = get_activity()
            activity['state'] = event.state
            activity['identity'] = event.identity
            activity['started_id'] = event.id
            activity['started_timestamp'] = event.timestamp
        elif event.state == 'completed':
            activity = get_activity()
            activity['state'] = event.state
            activity['result'] = event.result
            activity['completed_id'] = event.id
            activity['completed_timestamp'] = event.timestamp
        elif event.state == 'timed_out':
            activity = get_activity()
            activity['state'] = event.state
            activity['timeout_type'] = event.timeout_type
            activity['timeout_value'] = getattr(
                events[activity['scheduled_id'] - 1],
                '{}_timeout'.format(event.timeout_type.lower()))
            activity['timed_out_id'] = event.id
            activity['timed_out_timestamp'] = event.timestamp
            if 'retry' not in activity:
                activity['retry'] = 0
            else:
                activity['retry'] += 1
        elif event.state == 'failed':
            activity = get_activity()
            activity['state'] = event.state
            activity['reason'] = event.reason
            activity['details'] = getattr(event, 'details', '')
            activity['failed_timestamp'] = event.timestamp
            if 'retry' not in activity:
                activity['retry'] = 0
            else:
                activity['retry'] += 1
        elif event.state == 'cancelled':
            activity = get_activity()
            activity['state'] = event.state
            if hasattr(event, 'details'):
                activity['details'] = event.details
            activity['cancelled_timestamp'] = event.timestamp

    def parse_child_workflow_event(self, events, event):
        """Aggregate all the attributes of a workflow in a single entry.

        See http://docs.aws.amazon.com/amazonswf/latest/apireference/API_HistoryEvent.html

        - StartChildWorkflowExecutionInitiated: A request was made to start a
          child workflow execution.
        - StartChildWorkflowExecutionFailed: Failed to process
          StartChildWorkflowExecution decision. This happens when the decision
          is not configured properly, for example the workflow type specified
          is not registered.
        - ChildWorkflowExecutionStarted: A child workflow execution was
          successfully started.
        - ChildWorkflowExecutionCompleted: A child workflow execution, started
          by this workflow execution, completed successfully and was closed.
        - ChildWorkflowExecutionFailed: A child workflow execution, started by
          this workflow execution, failed to complete successfully and was
          closed.
        - ChildWorkflowExecutionTimedOut: A child workflow execution, started
          by this workflow execution, timed out and was closed.
        - ChildWorkflowExecutionCanceled: A child workflow execution, started
          by this workflow execution, was canceled and closed.
        - ChildWorkflowExecutionTerminated: A child workflow execution, started
          by this workflow execution, was terminated.

        :param events:
        :type events: list[swf.models.event.Event]
        :param event:
        :type event: swf.models.event.Event
        """

        def get_workflow():
            initiated_event = events[event.initiated_event_id - 1]
            return self._child_workflows[initiated_event.workflow_id]

        if event.state == 'start_initiated':
            workflow = {
                'type': 'child_workflow',
                'id': event.workflow_id,
                'name': event.workflow_type['name'],
                'version': event.workflow_type['version'],
                'state': event.state,
                'initiated_event_id': event.id,
                'raw_input': event.raw.get('input'),
                'child_policy': event.child_policy,
                'control': getattr(event, 'control', None),
                'tag_list': getattr(event, 'tag_list', None),
                'task_list': event.task_list['name'],
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._child_workflows:
                self._child_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                # May have gotten a start_failed before (or retrying?)
                if self._child_workflows[event.workflow_id]['state'] == 'start_initiated':
                    # Should not happen anymore
                    logger.warning("start_initiated again for workflow {} (initiated @{}, we're @{})".format(
                        event.workflow_id,
                        self._child_workflows[event.workflow_id]['initiated_event_id'],
                        event.id
                    ))
                self._child_workflows[event.workflow_id].update(workflow)
        elif event.state == 'start_failed':
            workflow = {
                'type': 'child_workflow',
                'id': event.workflow_id,
                'state': event.state,
                'cause': event.cause,
                'name': event.workflow_type['name'],
                'version': event.workflow_type['version'],
                'control': getattr(event, 'control', None),
                'start_failed_id': event.id,
                'start_failed_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._child_workflows:
                self._child_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                self._child_workflows[event.workflow_id].update(workflow)
        elif event.state == 'started':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['started_id'] = event.id
            workflow['started_timestamp'] = event.timestamp
        elif event.state == 'completed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['result'] = event.result
            workflow['completed_id'] = event.id
            workflow['completed_timestamp'] = event.timestamp
        elif event.state == 'failed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['reason'] = getattr(event, 'reason', None)
            workflow['details'] = getattr(event, 'details', None)
            workflow['failed_id'] = event.id
            workflow['failed_timestamp'] = event.timestamp
            # FIXME add retry here too?
        elif event.state == 'timed_out':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['timeout_type'] = event.timeout_type
            workflow['timeout_value'] = getattr(
                events[workflow['initiated_event_id'] - 1],
                '{}_timeout'.format(event.timeout_type.lower()),
                None,
            )
            workflow['timed_out_id'] = event.id
            workflow['timed_out_timestamp'] = event.timestamp
            if 'retry' not in workflow:
                workflow['retry'] = 0
            else:
                workflow['retry'] += 1
        elif event.state == 'canceled':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['details'] = getattr(event, 'details', None)
            workflow['canceled_id'] = event.id
            workflow['canceled_timestamp'] = event.timestamp
        elif event.state == 'terminated':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['terminated_id'] = event.id
            workflow['terminated_timestamp'] = event.timestamp

    def parse_workflow_event(self, events, event):
        """
        Parse a workflow event.
        :param events:
        :param event:
        """
        if event.state == 'signaled':
            signal = {
                'type': 'signal',
                'signal_name': event.signal_name,
                'state': event.state,
                'external_initiated_event_id': getattr(event, 'external_initiated_event_id', None),
                'external_run_id': getattr(event, 'external_workflow_execution', {}).get('runId'),
                'external_workflow_id': getattr(event, 'external_workflow_execution', {}).get('workflowId'),
                'input': event.input,
                'timestamp': event.timestamp,
            }
            self._signals[event.signal_name] = signal
            self._tasks.append(signal)

    def parse_external_workflow_event(self, events, event):
        """
        Parse an external workflow event.
        :param events:
        :param event:
        """
        def get_workflow():
            initiated_event = events[event.initiated_event_id - 1]
            return self._external_workflows[initiated_event.workflow_id]

        if event.state == 'signal_initiated':
            workflow = {
                'type': 'external_workflow',
                'id': event.workflow_id,
                'run_id': getattr(event, 'run_id', None),
                'signal_name': event.signal_name,
                'state': event.state,
                'initiated_event_id': event.id,
                'raw_input': event.raw.get('input'),
                'control': getattr(event, 'control', None),
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._external_workflows:
                self._external_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                logger.warning("signal_initiated again for workflow {} (initiated @{}, we're @{})".format(
                    event.workflow_id,
                    self._external_workflows[event.workflow_id]['initiated_event_id'],
                    event.id
                ))
                self._external_workflows[event.workflow_id].update(workflow)
        elif event.state == 'signal_failed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['cause'] = event.cause
            workflow['control'] = getattr(event, 'control', None)
            workflow['signal_failed_timestamp'] = event.timestamp
        elif event.state == 'signaled':
            workflow = get_workflow()
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['signaled_event_id'] = event.id
            workflow['signaled_timestamp'] = event.timestamp
        elif event.state == 'request_cancel_initiated':
            workflow = {
                'type': 'external_workflow',
                'id': event.workflow_id,
                'run_id': getattr(event, 'run_id', None),
                'signal_name': event.signal_name,
                'state': event.state,
                'initiated_event_id': event.id,
                'raw_input': event.raw.get('input'),
                'control': getattr(event, 'control', None),
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._external_workflows:
                self._external_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                logger.warning("request_cancel_initiated again for workflow {} (initiated @{}, we're @{})".format(
                    event.workflow_id,
                    self._external_workflows[event.workflow_id]['initiated_event_id'],
                    event.id
                ))
                self._external_workflows[event.workflow_id].update(workflow)
        elif event.state == 'request_cancel_failed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['cause'] = event.cause
            workflow['control'] = getattr(event, 'control', None)
            workflow['request_cancel_failed_timestamp'] = event.timestamp
        elif event.state == 'cancel_requested':
            workflow = get_workflow()
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['cancel_requested_timestamp'] = event.timestamp

    TYPE_TO_PARSER = {
        'ActivityTask': parse_activity_event,
        'ChildWorkflowExecution': parse_child_workflow_event,
        'WorkflowExecution': parse_workflow_event,
        'ExternalWorkflowExecution': parse_external_workflow_event,
    }

    def parse(self):
        """
        Parse the events.
        Update the corresponding statuses.
        """

        events = self.events
        for event in events:
            parser = self.TYPE_TO_PARSER.get(event.type)
            if parser:
                parser(self, events, event)
<commit_msg>History: split external events<commit_after>import collections
import logging

logger = logging.getLogger(__name__)

# noinspection PyUnresolvedReferences
class History(object):
    """
    History data.

    :ivar _history: raw(ish) history events
    :type _history: swf.models.history.History
    :ivar _activities: activity events
    :type _activities: collections.OrderedDict[str, dict[str, Any]]
    :ivar _child_workflows: child workflow events
    :type _child_workflows: collections.OrderedDict[str, dict[str, Any]]
    :ivar _external_workflows_signaling: external workflow signaling events
    :type _external_workflows_signaling: collections.OrderedDict[str, dict[str, Any]]
    :ivar _external_workflows_canceling: external workflow canceling events
    :type _external_workflows_canceling: collections.OrderedDict[str, dict[str, Any]]
    :ivar _signals: activity events
    :type _signals: collections.OrderedDict[str, dict[str, Any]]
    :ivar _tasks: ordered list of tasks/etc
    :type _tasks: list[dict[str, Any]]
    """

    def __init__(self, history):
        self._history = history
        self._activities = collections.OrderedDict()
        self._child_workflows = collections.OrderedDict()
        self._external_workflows_signaling = collections.OrderedDict()
        self._external_workflows_canceling = collections.OrderedDict()
        self._signals = collections.OrderedDict()
        self._signaled_workflows = collections.defaultdict(list)
        self._tasks = []

    @property
    def activities(self):
        """
        :return: activities
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._activities

    @property
    def child_workflows(self):
        """
        :return: child WFs
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._child_workflows

    @property
    def external_workflows_signaling(self):
        """
        :return: external WFs
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._external_workflows_signaling

    @property
    def signals(self):
        """
        :return: signals
        :rtype: collections.OrderedDict[str, dict[str, Any]]
        """
        return self._signals

    @property
    def signaled_workflows(self):
        """
        :return: signaled workflows
        :rtype: defaultdict(list)
        """
        return self._signaled_workflows

    @property
    def tasks(self):
        return self._tasks

    @property
    def events(self):
        """

        :return:
        :rtype: list[swf.models.event.Event]
        """
        return self._history.events

    def parse_activity_event(self, events, event):
        """
        Aggregate all the attributes of an activity in a single entry.

        :param events:
        :type events: list[swf.models.event.Event]
        :param event:
        :type event: swf.models.event.Event
        """

        def get_activity():
            """
            Return a reference to the corresponding activity.
            :return: mutable activity
            :rtype: dict[str, Any]
            """
            scheduled_event = events[event.scheduled_event_id - 1]
            return self._activities[scheduled_event.activity_id]

        if event.state == 'scheduled':
            activity = {
                'type': 'activity',
                'id': event.activity_id,
                'name': event.activity_type['name'],
                'version': event.activity_type['version'],
                'state': event.state,
                'scheduled_id': event.id,
                'scheduled_timestamp': event.timestamp,
                'input': event.input,
                'task_list': event.task_list['name'],
            }
            if event.activity_id not in self._activities:
                self._activities[event.activity_id] = activity
                self._tasks.append(activity)
            else:
                # When the executor retries a task, it schedules it again.
                # We have to take care of not overriding some values set by the
                # previous execution of the task such as the number of retries
                # in ``retry``.  As the state of the event mutates, it
                # corresponds to the last execution.
                self._activities[event.activity_id].update(activity)
        elif event.state == 'schedule_failed':
            activity = {
                'type': 'activity',
                'state': event.state,
                'cause': event.cause,
                'activity_type': event.activity_type.copy(),
                'schedule_failed_timestamp': event.timestamp,
            }

            if event.activity_id not in self._activities:
                self._activities[event.activity_id] = activity
                self._tasks.append(activity)
            else:
                # When the executor retries a task, it schedules it again.
                # We have to take care of not overriding some values set by the
                # previous execution of the task such as the number of retries
                # in ``retry``.  As the state of the event mutates, it
                # corresponds to the last execution.
                self._activities[event.activity_id].update(activity)

        elif event.state == 'started':
            activity = get_activity()
            activity['state'] = event.state
            activity['identity'] = event.identity
            activity['started_id'] = event.id
            activity['started_timestamp'] = event.timestamp
        elif event.state == 'completed':
            activity = get_activity()
            activity['state'] = event.state
            activity['result'] = event.result
            activity['completed_id'] = event.id
            activity['completed_timestamp'] = event.timestamp
        elif event.state == 'timed_out':
            activity = get_activity()
            activity['state'] = event.state
            activity['timeout_type'] = event.timeout_type
            activity['timeout_value'] = getattr(
                events[activity['scheduled_id'] - 1],
                '{}_timeout'.format(event.timeout_type.lower()))
            activity['timed_out_id'] = event.id
            activity['timed_out_timestamp'] = event.timestamp
            if 'retry' not in activity:
                activity['retry'] = 0
            else:
                activity['retry'] += 1
        elif event.state == 'failed':
            activity = get_activity()
            activity['state'] = event.state
            activity['reason'] = event.reason
            activity['details'] = getattr(event, 'details', '')
            activity['failed_timestamp'] = event.timestamp
            if 'retry' not in activity:
                activity['retry'] = 0
            else:
                activity['retry'] += 1
        elif event.state == 'cancelled':
            activity = get_activity()
            activity['state'] = event.state
            if hasattr(event, 'details'):
                activity['details'] = event.details
            activity['cancelled_timestamp'] = event.timestamp

    def parse_child_workflow_event(self, events, event):
        """Aggregate all the attributes of a workflow in a single entry.

        See http://docs.aws.amazon.com/amazonswf/latest/apireference/API_HistoryEvent.html

        - StartChildWorkflowExecutionInitiated: A request was made to start a
          child workflow execution.
        - StartChildWorkflowExecutionFailed: Failed to process
          StartChildWorkflowExecution decision. This happens when the decision
          is not configured properly, for example the workflow type specified
          is not registered.
        - ChildWorkflowExecutionStarted: A child workflow execution was
          successfully started.
        - ChildWorkflowExecutionCompleted: A child workflow execution, started
          by this workflow execution, completed successfully and was closed.
        - ChildWorkflowExecutionFailed: A child workflow execution, started by
          this workflow execution, failed to complete successfully and was
          closed.
        - ChildWorkflowExecutionTimedOut: A child workflow execution, started
          by this workflow execution, timed out and was closed.
        - ChildWorkflowExecutionCanceled: A child workflow execution, started
          by this workflow execution, was canceled and closed.
        - ChildWorkflowExecutionTerminated: A child workflow execution, started
          by this workflow execution, was terminated.

        :param events:
        :type events: list[swf.models.event.Event]
        :param event:
        :type event: swf.models.event.Event
        """

        def get_workflow():
            initiated_event = events[event.initiated_event_id - 1]
            return self._child_workflows[initiated_event.workflow_id]

        if event.state == 'start_initiated':
            workflow = {
                'type': 'child_workflow',
                'id': event.workflow_id,
                'name': event.workflow_type['name'],
                'version': event.workflow_type['version'],
                'state': event.state,
                'initiated_event_id': event.id,
                'raw_input': event.raw.get('input'),
                'child_policy': event.child_policy,
                'control': getattr(event, 'control', None),
                'tag_list': getattr(event, 'tag_list', None),
                'task_list': event.task_list['name'],
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._child_workflows:
                self._child_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                # May have gotten a start_failed before (or retrying?)
                if self._child_workflows[event.workflow_id]['state'] == 'start_initiated':
                    # Should not happen anymore
                    logger.warning("start_initiated again for workflow {} (initiated @{}, we're @{})".format(
                        event.workflow_id,
                        self._child_workflows[event.workflow_id]['initiated_event_id'],
                        event.id
                    ))
                self._child_workflows[event.workflow_id].update(workflow)
        elif event.state == 'start_failed':
            workflow = {
                'type': 'child_workflow',
                'id': event.workflow_id,
                'state': event.state,
                'cause': event.cause,
                'name': event.workflow_type['name'],
                'version': event.workflow_type['version'],
                'control': getattr(event, 'control', None),
                'start_failed_id': event.id,
                'start_failed_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._child_workflows:
                self._child_workflows[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                self._child_workflows[event.workflow_id].update(workflow)
        elif event.state == 'started':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['started_id'] = event.id
            workflow['started_timestamp'] = event.timestamp
        elif event.state == 'completed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['result'] = event.result
            workflow['completed_id'] = event.id
            workflow['completed_timestamp'] = event.timestamp
        elif event.state == 'failed':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['reason'] = getattr(event, 'reason', None)
            workflow['details'] = getattr(event, 'details', None)
            workflow['failed_id'] = event.id
            workflow['failed_timestamp'] = event.timestamp
            # FIXME add retry here too?
        elif event.state == 'timed_out':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['timeout_type'] = event.timeout_type
            workflow['timeout_value'] = getattr(
                events[workflow['initiated_event_id'] - 1],
                '{}_timeout'.format(event.timeout_type.lower()),
                None,
            )
            workflow['timed_out_id'] = event.id
            workflow['timed_out_timestamp'] = event.timestamp
            if 'retry' not in workflow:
                workflow['retry'] = 0
            else:
                workflow['retry'] += 1
        elif event.state == 'canceled':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['details'] = getattr(event, 'details', None)
            workflow['canceled_id'] = event.id
            workflow['canceled_timestamp'] = event.timestamp
        elif event.state == 'terminated':
            workflow = get_workflow()
            workflow['state'] = event.state
            workflow['terminated_id'] = event.id
            workflow['terminated_timestamp'] = event.timestamp

    def parse_workflow_event(self, events, event):
        """
        Parse a workflow event.
        :param events:
        :param event:
        """
        if event.state == 'signaled':
            signal = {
                'type': 'signal',
                'name': event.signal_name,
                'state': event.state,
                'external_initiated_event_id': getattr(event, 'external_initiated_event_id', None),
                'external_run_id': getattr(event, 'external_workflow_execution', {}).get('runId'),
                'external_workflow_id': getattr(event, 'external_workflow_execution', {}).get('workflowId'),
                'input': event.input,
                'event_id': event.id,
                'timestamp': event.timestamp,
            }
            self._signals[event.signal_name] = signal
            self._tasks.append(signal)

    def parse_external_workflow_event(self, events, event):
        """
        Parse an external workflow event.
        :param events:
        :param event:
        """
        def get_workflow(workflows):
            initiated_event = events[event.initiated_event_id - 1]
            return workflows[initiated_event.workflow_id]

        control = getattr(event, 'control', None)
        if event.state == 'signal_execution_initiated':
            workflow = {
                'type': 'external_workflow',
                'id': event.workflow_id,
                'run_id': getattr(event, 'run_id', None),
                'name': event.signal_name,
                'state': event.state,
                'initiated_event_id': event.id,
                'raw_input': event.raw.get('input'),
                'control': control,
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._external_workflows_signaling:
                self._external_workflows_signaling[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                if self._external_workflows_signaling[event.workflow_id]['state'] == 'signal_initiated':
                    logger.warning("signal_initiated again for workflow {} (initiated @{}, we're @{})".format(
                        event.workflow_id,
                        self._external_workflows_signaling[event.workflow_id]['initiated_event_id'],
                        event.id
                    ))
                self._external_workflows_signaling[event.workflow_id].update(workflow)
        elif event.state == 'signal_execution_failed':
            # TODO put in _signals
            workflow = get_workflow(self._external_workflows_signaling)
            workflow['state'] = event.state
            workflow['cause'] = event.cause
            if control:
                workflow['control'] = control
            workflow['signal_failed_timestamp'] = event.timestamp
        elif event.state == 'execution_signaled':
            workflow = get_workflow(self._external_workflows_signaling)
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['signaled_event_id'] = event.id
            workflow['signaled_timestamp'] = event.timestamp
            self._signaled_workflows[workflow['name']].append(workflow)
            logger.warning('{}'.format(workflow))
        elif event.state == 'request_cancel_execution_initiated':
            workflow = {
                'type': 'external_workflow',
                'id': event.workflow_id,
                'run_id': getattr(event, 'run_id', None),
                'state': event.state,
                'control': control,
                'initiated_event_id': event.id,
                'initiated_event_timestamp': event.timestamp,
            }
            if event.workflow_id not in self._external_workflows_canceling:
                self._external_workflows_canceling[event.workflow_id] = workflow
                self._tasks.append(workflow)
            else:
                logger.warning("request_cancel_initiated again for workflow {} (initiated @{}, we're @{})".format(
                    event.workflow_id,
                    self._external_workflows_canceling[event.workflow_id]['initiated_event_id'],
                    event.id
                ))
                self._external_workflows_canceling[event.workflow_id].update(workflow)
        elif event.state == 'request_cancel_execution_failed':
            workflow = get_workflow(self._external_workflows_canceling)
            workflow['state'] = event.state
            workflow['cause'] = event.cause
            if control:
                workflow['control'] = control
            workflow['request_cancel_failed_timestamp'] = event.timestamp
        elif event.state == 'execution_cancel_requested':
            workflow = get_workflow(self._external_workflows_canceling)
            workflow['run_id'] = event.workflow_execution['runId']
            workflow['workflow_id'] = event.workflow_execution['workflowId']
            workflow['cancel_requested_timestamp'] = event.timestamp

    TYPE_TO_PARSER = {
        'ActivityTask': parse_activity_event,
        'ChildWorkflowExecution': parse_child_workflow_event,
        'WorkflowExecution': parse_workflow_event,
        'ExternalWorkflowExecution': parse_external_workflow_event,
    }

    def parse(self):
        """
        Parse the events.
        Update the corresponding statuses.
        """

        events = self.events
        for event in events:
            parser = self.TYPE_TO_PARSER.get(event.type)
            if parser:
                parser(self, events, event)
<|endoftext|>


<commit_before>"""
filterAnnotatedSV
~~~~~~~~~~~~~~~~~

:Description: This module will filter calls from the merged file
"""

'''
Created on Mar 17, 2015
Description: This module will filter calls from the merged file
@author: Ronak H Shah

::Inputs::
inputTxt: Filter Text File
outputDir: Output directory
outPrefix: Prefix of the output file
blacklistGenesFile: List of genes that should be eliminated
verbose: Mode
genesToKeepFile: List of genes that should be kept

::Output::
Filtered Output files
'''
import os
import pandas as pd
import logging
import checkparameters as cp
import re
import coloredlogs

# Initiate logger
logger = logging.getLogger('iCallSV.FilterDellyCalls')
coloredlogs.install(level='DEBUG')

def run(inputTxt, outputDir, outPrefix, blacklistGenesFile, verbose, genesToKeepFile="somefile.txt"):
    """
    This will ``filter sv calls`` from the final merged file.

    :param str inputTxt: str for the txt file to be filtered
    :param str outputDir: str for the output directory
    :param str outputPrefix: str prefix for the output File
    :param str blacklistGenesFile: str for the txt file containing blacklisted genes
    :param bool verbose: a boolean
    :param str genesToKeepFile: str for the txt file containing genes to keep
    :return: A str name of final sv file
    :rtype: str

    """
    cp.checkFile(inputTxt)
    cp.checkFile(blacklistGenesFile)
    cp.checkDir(outputDir)
    cp.checkEmpty(outPrefix, "Prefix for the output file")
    if(os.path.isfile(genesToKeepFile)):
        logger.info(
            "iCallSV::FilterFinalFile: Genes to Keep File Given %s and will be used.",
            genesToKeepFile)
        keepGenes = [line.strip() for line in open(genesToKeepFile, 'r')]
    else:
        keepGenes = None
    inputDF = pd.read_csv(inputTxt, sep="\t", header=0, keep_default_na='True')
    outputDF = inputDF.copy()
    #outputDF = pd.DataFrame(columns=inputDF.columns)
    outputFile = os.path.join(outputDir, outPrefix + "_final.txt")
    for index, row in inputDF.iterrows():
        gene1 = row.loc['Gene1']
        gene2 = row.loc['Gene2']
        site1 = row.loc['Site1Description']
        site2 = row.loc['Site2Description']
        # skip IGR records
        if("IGR" in site1 and "IGR" in site2):
            igrFlag = True
        else:
            igrFlag = False

        # check records from these gene
        if(keepGenes):
            keepGeneFlag = checkGeneListToKeep(gene1, gene2, keepGenes)
        else:
            keepGeneFlag = True
        # check records from these gene
        blacklistGenes = [line.strip() for line in open(blacklistGenesFile, 'r')]
        blacklistGeneFlag = checkBlackListGene(gene1, gene2, blacklistGenes)
        # skip record occurring within intron
        eventInIntronFlag = False
        if((gene1 == gene2) and ((igrFlag is False) or (blacklistGeneFlag is False)) and ("Intron" in site1 and "Intron" in site2)):
            eventInIntronFlag = checkEventInIntronFlag(gene1, gene2, site1, site2)
        else:
            pass

        if((keepGeneFlag is False) or (igrFlag) or (blacklistGeneFlag) or (eventInIntronFlag)):
            if(verbose):
                logger.warn(
                    "iCallSV::FilterFinalFile: Record: gene1:%s; gene2:%s; site1:%s; site2:%s; will be Filtered as keepGeneFlag:%s; IGR:%s; blackListGene:%s; Intronic Event:%s",
                    gene1,
                    gene2,
                    site1,
                    site2,
                    str(keepGeneFlag),
                    str(igrFlag),
                    str(blacklistGeneFlag),
                    str(eventInIntronFlag))
            outputDF = outputDF.drop(index)
        else:
            pass
    # Write The Final Output File
    outputDF.to_csv(outputFile, sep='\t', index=False)
    if(verbose):
        logger.info(
            "iCallSV::FilterFinalFile: Finished Filtering, Final data written in %s",
            outputFile)

    return(outputFile)

# Check if the gene is a Keep gene

def checkGeneListToKeep(gene1, gene2, keepGenes):
    if((gene1 in keepGenes) or (gene2 in keepGenes)):
        kgFlag = True
    else:
        kgFlag = False
    return(kgFlag)

# Check if the gene is a blacklist gene

def checkBlackListGene(gene1, gene2, blacklistGenes):
    """
    This will ``check for blacklisted genes``

    :param str gene1: str for the name of gene at breakpoint 1
    :param str gene2: str for the name of gene at breakpoint 2
    :param list blacklistGenes:  list containing blacklisted genes
    :param str genesToKeepFile: str for the txt file containing genes to keep
    :return: A boolean tag indicating True or False
    :rtype: bool

    """
    if((gene1 in blacklistGenes) or (gene2 in blacklistGenes)):
        bgFlag = True
    else:
        bgFlag = False
    return(bgFlag)

# Check if the event is in the intron only and not affecting splicing
def checkEventInIntronFlag(gene1, gene2, site1, site2):
    """
    This will ``Check if the event is in the intron only and not affecting
    splicing``

    :param str gene1: str for the name of gene at breakpoint 1
    :param str gene2: str for the name of gene at breakpoint 2
    :param str site1: str for the description of site in breakpoint 1
    :param str site2: str for the description of site in breakpoint 2
    :return: A boolean tag indicating True or False
    :rtype: bool

    """
    if(gene1 == gene2):
        (s1A, s1B) = site1.split(":")
        (s2A, s2B) = site2.split(":")
        (s1a, s1b, s1c, s1d) = s1B.split(" ")
        (s2a, s2b, s2c, s2d) = s2B.split(" ")
        if(("before" in site1 and "before" in site2) or ("after" in site1 and "after" in site2)):
            if(int(s1d) == int(s2d)):
                if("bp" in s1a):
                    s1location = re.findall(r'\d+', s1a)[0]
                    s2location = re.findall(r'\d+', s2a)[0]
                    if(int(s1location) < 5 or int(s2location) < 5):
                        eviFlag = False
                    else:
                        eviFlag = True
                else:
                    eviFlag = False
            else:
                eviFlag = False
        else:
            eviFlag = False
    else:
        eviFlag = False
    return(eviFlag)
<commit_msg>Debugging<commit_after>"""
filterAnnotatedSV
~~~~~~~~~~~~~~~~~

:Description: This module will filter calls from the merged file
"""

'''
Created on Mar 17, 2015
Description: This module will filter calls from the merged file
@author: Ronak H Shah

::Inputs::
inputTxt: Filter Text File
outputDir: Output directory
outPrefix: Prefix of the output file
blacklistGenesFile: List of genes that should be eliminated
verbose: Mode
genesToKeepFile: List of genes that should be kept

::Output::
Filtered Output files
'''
import os
import pandas as pd
import logging
import checkparameters as cp
import re
import coloredlogs

# Initiate logger
logger = logging.getLogger('iCallSV.FilterDellyCalls')
coloredlogs.install(level='DEBUG')

def run(inputTxt, outputDir, outPrefix, blacklistGenesFile, verbose, genesToKeepFile="somefile.txt"):
    """
    This will ``filter sv calls`` from the final merged file.

    :param str inputTxt: str for the txt file to be filtered
    :param str outputDir: str for the output directory
    :param str outputPrefix: str prefix for the output File
    :param str blacklistGenesFile: str for the txt file containing blacklisted genes
    :param bool verbose: a boolean
    :param str genesToKeepFile: str for the txt file containing genes to keep
    :return: A str name of final sv file
    :rtype: str

    """
    cp.checkFile(inputTxt)
    cp.checkFile(blacklistGenesFile)
    cp.checkDir(outputDir)
    cp.checkEmpty(outPrefix, "Prefix for the output file")
    if(os.path.isfile(genesToKeepFile)):
        logger.info(
            "iCallSV::FilterFinalFile: Genes to Keep File Given %s and will be used.",
            genesToKeepFile)
        keepGenes = [line.strip() for line in open(genesToKeepFile, 'r')]
    else:
        keepGenes = None
    inputDF = pd.read_table(inputTxt, keep_default_na='True')
    outputDF = inputDF.copy()
    #outputDF = pd.DataFrame(columns=inputDF.columns)
    outputFile = os.path.join(outputDir, outPrefix + "_final.txt")
    for index, row in inputDF.iterrows():
        gene1 = row.loc['Gene1']
        gene2 = row.loc['Gene2']
        site1 = row.loc['Site1Description']
        site2 = row.loc['Site2Description']
        # skip IGR records
        if("IGR" in site1 and "IGR" in site2):
            igrFlag = True
        else:
            igrFlag = False

        # check records from these gene
        if(keepGenes):
            keepGeneFlag = checkGeneListToKeep(gene1, gene2, keepGenes)
        else:
            keepGeneFlag = True
        # check records from these gene
        blacklistGenes = [line.strip() for line in open(blacklistGenesFile, 'r')]
        blacklistGeneFlag = checkBlackListGene(gene1, gene2, blacklistGenes)
        # skip record occurring within intron
        eventInIntronFlag = False
        if((gene1 == gene2) and ((igrFlag is False) or (blacklistGeneFlag is False)) and ("Intron" in site1 and "Intron" in site2)):
            eventInIntronFlag = checkEventInIntronFlag(gene1, gene2, site1, site2)
        else:
            pass

        if((keepGeneFlag is False) or (igrFlag) or (blacklistGeneFlag) or (eventInIntronFlag)):
            if(verbose):
                logger.warn(
                    "iCallSV::FilterFinalFile: Record: gene1:%s; gene2:%s; site1:%s; site2:%s; will be Filtered as keepGeneFlag:%s; IGR:%s; blackListGene:%s; Intronic Event:%s",
                    gene1,
                    gene2,
                    site1,
                    site2,
                    str(keepGeneFlag),
                    str(igrFlag),
                    str(blacklistGeneFlag),
                    str(eventInIntronFlag))
            outputDF = outputDF.drop(index)
        else:
            pass
    # Write The Final Output File
    outputDF.to_csv(outputFile, sep='\t', index=False)
    if(verbose):
        logger.info(
            "iCallSV::FilterFinalFile: Finished Filtering, Final data written in %s",
            outputFile)

    return(outputFile)

# Check if the gene is a Keep gene

def checkGeneListToKeep(gene1, gene2, keepGenes):
    if((gene1 in keepGenes) or (gene2 in keepGenes)):
        kgFlag = True
    else:
        kgFlag = False
    return(kgFlag)

# Check if the gene is a blacklist gene

def checkBlackListGene(gene1, gene2, blacklistGenes):
    """
    This will ``check for blacklisted genes``

    :param str gene1: str for the name of gene at breakpoint 1
    :param str gene2: str for the name of gene at breakpoint 2
    :param list blacklistGenes:  list containing blacklisted genes
    :param str genesToKeepFile: str for the txt file containing genes to keep
    :return: A boolean tag indicating True or False
    :rtype: bool

    """
    if((gene1 in blacklistGenes) or (gene2 in blacklistGenes)):
        bgFlag = True
    else:
        bgFlag = False
    return(bgFlag)

# Check if the event is in the intron only and not affecting splicing
def checkEventInIntronFlag(gene1, gene2, site1, site2):
    """
    This will ``Check if the event is in the intron only and not affecting
    splicing``

    :param str gene1: str for the name of gene at breakpoint 1
    :param str gene2: str for the name of gene at breakpoint 2
    :param str site1: str for the description of site in breakpoint 1
    :param str site2: str for the description of site in breakpoint 2
    :return: A boolean tag indicating True or False
    :rtype: bool

    """
    if(gene1 == gene2):
        (s1A, s1B) = site1.split(":")
        (s2A, s2B) = site2.split(":")
        (s1a, s1b, s1c, s1d) = s1B.split(" ")
        (s2a, s2b, s2c, s2d) = s2B.split(" ")
        if(("before" in site1 and "before" in site2) or ("after" in site1 and "after" in site2)):
            if(int(s1d) == int(s2d)):
                if("bp" in s1a):
                    s1location = re.findall(r'\d+', s1a)[0]
                    s2location = re.findall(r'\d+', s2a)[0]
                    if(int(s1location) < 5 or int(s2location) < 5):
                        eviFlag = False
                    else:
                        eviFlag = True
                else:
                    eviFlag = False
            else:
                eviFlag = False
        else:
            eviFlag = False
    else:
        eviFlag = False
    return(eviFlag)
<|endoftext|>


<commit_before>"""
sentry.search.django.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import re
import itertools

from collections import defaultdict

from django.utils.encoding import force_unicode

from sentry.search.base import SearchBackend

# Words which should not be indexed
STOP_WORDS = set(['the', 'of', 'to', 'and', 'a', 'in', 'is', 'it', 'you', 'that'])

# Do not index any words shorter than this
MIN_WORD_LENGTH = 3

# Consider these characters to be punctuation (they will be replaced with spaces prior to word extraction)
PUNCTUATION_CHARS = re.compile('[%s]' % re.escape(".,;:!?@$%^&*()-<>[]{}\\|/`~'\""))

class DjangoSearchBackend(SearchBackend):
    def _tokenize(self, text):
        """
        Given a string, returns a list of tokens.
        """
        if not text:
            return []

        text = PUNCTUATION_CHARS.sub(' ', text)

        words = [
            t[:128].lower() for t in text.split()
            if len(t) >= MIN_WORD_LENGTH and t.lower() not in STOP_WORDS
        ]

        return words

    def index(self, group, event):
        from sentry import app
        from sentry.search.django.models import SearchDocument, SearchToken

        document, created = SearchDocument.objects.get_or_create(
            project=event.project,
            group=group,
            defaults={
                'status': group.status,
                'total_events': 1,
                'date_added': group.first_seen,
                'date_changed': group.last_seen,
            }
        )
        if not created:
            app.buffer.incr(SearchDocument, {
                'total_events': 1,
            }, {
                'id': document.id,
            }, {
                'date_changed': group.last_seen,
                'status': group.status,
            })

            document.total_events += 1
            document.date_changed = group.last_seen
            document.status = group.status

        context = defaultdict(list)
        for interface in event.interfaces.itervalues():
            for k, v in interface.get_search_context(event).iteritems():
                context[k].extend(v)

        context['text'].extend([
            event.message,
            event.logger,
            event.server_name,
            event.culprit,
        ])

        token_counts = defaultdict(lambda: defaultdict(int))
        for field, values in context.iteritems():
            field = field.lower()
            if field == 'text':
                # we only tokenize the base text field
                values = itertools.chain(*[self._tokenize(force_unicode(v)) for v in values])
            else:
                values = [v.lower() for v in values]
            for value in values:
                if not value:
                    continue
                token_counts[field][value] += 1

        for field, tokens in token_counts.iteritems():
            for token, count in tokens.iteritems():
                app.buffer.incr(SearchToken, {
                    'times_seen': count,
                }, {
                    'document': document,
                    'token': token,
                    'field': field,
                })

        return document

    def query(self, project, query, sort_by='score', offset=0, limit=100):
        from sentry.search.django.models import SearchDocument

        tokens = self._tokenize(query)

        if sort_by == 'score':
            order_by = 'SUM(st.times_seen) / sd.total_events DESC'
        elif sort_by == 'new':
            order_by = 'sd.date_added DESC'
        elif sort_by == 'date':
            order_by = 'sd.date_changed DESC'
        else:
            raise ValueError('sort_by: %r' % sort_by)

        if tokens:
            token_sql = ' st.token IN (%s) AND ' % \
                ', '.join('%s' for i in range(len(tokens)))
        else:
            token_sql = ' '

        sql = """
            SELECT sd.*,
                   SUM(st.times_seen) / sd.total_events as score
            FROM sentry_searchdocument as sd
            INNER JOIN sentry_searchtoken as st
                ON st.document_id = sd.id
            WHERE %s
                sd.project_id = %s
            GROUP BY sd.id, sd.group_id, sd.total_events, sd.date_changed, sd.date_added, sd.project_id, sd.status
            ORDER BY %s
            LIMIT %d OFFSET %d
        """ % (
            token_sql,
            project.id,
            order_by,
            limit,
            offset,
        )
        params = tokens

        return list(SearchDocument.objects.raw(sql, params))
<commit_msg>Remove logger/server_name dupe from search<commit_after>"""
sentry.search.django.backend
~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:copyright: (c) 2010-2014 by the Sentry Team, see AUTHORS for more details.
:license: BSD, see LICENSE for more details.
"""

from __future__ import absolute_import

import re
import itertools

from collections import defaultdict

from django.utils.encoding import force_unicode

from sentry.search.base import SearchBackend

# Words which should not be indexed
STOP_WORDS = set(['the', 'of', 'to', 'and', 'a', 'in', 'is', 'it', 'you', 'that'])

# Do not index any words shorter than this
MIN_WORD_LENGTH = 3

# Consider these characters to be punctuation (they will be replaced with spaces prior to word extraction)
PUNCTUATION_CHARS = re.compile('[%s]' % re.escape(".,;:!?@$%^&*()-<>[]{}\\|/`~'\""))

class DjangoSearchBackend(SearchBackend):
    def _tokenize(self, text):
        """
        Given a string, returns a list of tokens.
        """
        if not text:
            return []

        text = PUNCTUATION_CHARS.sub(' ', text)

        words = [
            t[:128].lower() for t in text.split()
            if len(t) >= MIN_WORD_LENGTH and t.lower() not in STOP_WORDS
        ]

        return words

    def index(self, group, event):
        from sentry import app
        from sentry.search.django.models import SearchDocument, SearchToken

        document, created = SearchDocument.objects.get_or_create(
            project=event.project,
            group=group,
            defaults={
                'status': group.status,
                'total_events': 1,
                'date_added': group.first_seen,
                'date_changed': group.last_seen,
            }
        )
        if not created:
            app.buffer.incr(SearchDocument, {
                'total_events': 1,
            }, {
                'id': document.id,
            }, {
                'date_changed': group.last_seen,
                'status': group.status,
            })

            document.total_events += 1
            document.date_changed = group.last_seen
            document.status = group.status

        context = defaultdict(list)
        for interface in event.interfaces.itervalues():
            for k, v in interface.get_search_context(event).iteritems():
                context[k].extend(v)

        context['text'].extend([
            event.message,
            event.culprit,
        ])

        token_counts = defaultdict(lambda: defaultdict(int))
        for field, values in context.iteritems():
            field = field.lower()
            if field == 'text':
                # we only tokenize the base text field
                values = itertools.chain(*[self._tokenize(force_unicode(v)) for v in values])
            else:
                values = [v.lower() for v in values]
            for value in values:
                if not value:
                    continue
                token_counts[field][value] += 1

        for field, tokens in token_counts.iteritems():
            for token, count in tokens.iteritems():
                app.buffer.incr(SearchToken, {
                    'times_seen': count,
                }, {
                    'document': document,
                    'token': token,
                    'field': field,
                })

        return document

    def query(self, project, query, sort_by='score', offset=0, limit=100):
        from sentry.search.django.models import SearchDocument

        tokens = self._tokenize(query)

        if sort_by == 'score':
            order_by = 'SUM(st.times_seen) / sd.total_events DESC'
        elif sort_by == 'new':
            order_by = 'sd.date_added DESC'
        elif sort_by == 'date':
            order_by = 'sd.date_changed DESC'
        else:
            raise ValueError('sort_by: %r' % sort_by)

        if tokens:
            token_sql = ' st.token IN (%s) AND ' % \
                ', '.join('%s' for i in range(len(tokens)))
        else:
            token_sql = ' '

        sql = """
            SELECT sd.*,
                   SUM(st.times_seen) / sd.total_events as score
            FROM sentry_searchdocument as sd
            INNER JOIN sentry_searchtoken as st
                ON st.document_id = sd.id
            WHERE %s
                sd.project_id = %s
            GROUP BY sd.id, sd.group_id, sd.total_events, sd.date_changed, sd.date_added, sd.project_id, sd.status
            ORDER BY %s
            LIMIT %d OFFSET %d
        """ % (
            token_sql,
            project.id,
            order_by,
            limit,
            offset,
        )
        params = tokens

        return list(SearchDocument.objects.raw(sql, params))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
requests.models
~~~~~~~~~~~~~~~

"""

import urllib
import zlib
from Cookie import SimpleCookie
from urlparse import urlparse, urlunparse, urljoin

from .packages import urllib3
from .packages.urllib3.filepost import encode_multipart_formdata

from ._config import get_config
from .structures import CaseInsensitiveDict
from .utils import *
from .status_codes import codes
from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects
from .packages.urllib3.poolmanager import PoolManager

REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)

class Request(object):
    """The :class:`Request <Request>` object. It carries out all functionality of
    Requests. Recommended interface is with the Requests functions.
    """

    def __init__(self,
        url=None, headers=dict(), files=None, method=None, data=dict(),
        params=dict(), auth=None, cookies=None, timeout=None, redirect=False,
        allow_redirects=False, proxies=None, config=None, hooks=None,
        _pools=None):

        if cookies is None:
            cookies = {}

        #: Float describ the timeout of the request.
        #  (Use socket.setdefaulttimeout() as fallback)
        self.timeout = timeout

        #: Request URL.
        self.url = url

        #: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
        self.headers = headers or {}

        #: Dictionary of files to multipart upload (``{filename: content}``).
        self.files = files or {}

        #: HTTP Method to use.
        self.method = method

        #: Dictionary or byte of request body data to attach to the
        #: :class:`Request <Request>`.
        self.data = None

        #: Dictionary or byte of querystring data to attach to the
        #: :class:`Request <Request>`.
        self.params = None

        #: True if :class:`Request <Request>` is part of a redirect chain (disables history
        #: and HTTPError storage).
        self.redirect = redirect

        #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``)
        self.allow_redirects = allow_redirects

        # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
        self.proxies = proxies

        self.config = get_config(config)

        self.data = data
        self._enc_data = encode_params(data)
        self.params = params
        self._enc_params = encode_params(params)

        #: :class:`Response <Response>` instance, containing
        #: content and metadata of HTTP Response, once :attr:`sent <send>`.
        self.response = Response()

        #: :class:`AuthObject` to attach to :class:`Request <Request>`.
        self.auth = auth

        #: CookieJar to attach to :class:`Request <Request>`.
        self.cookies = cookies

        #: True if Request has been sent.
        self.sent = False

        # Header manipulation and defaults.

        if self.config.get('accept_gzip'):
            self.headers.update({'Accept-Encoding': 'gzip'})

        if headers:
            headers = CaseInsensitiveDict(self.headers)
        else:
            headers = CaseInsensitiveDict()

        for (k, v) in self.config.get('base_headers').items():
            if k not in headers:
                headers[k] = v

        self.headers = headers

        self.hooks = hooks
        self._pools = _pools

    def __repr__(self):
        return '<Request [%s]>' % (self.method)

    def _checks(self):
        """Deterministic checks for consistency."""

        if not self.url:
            raise URLRequired

    def _build_response(self, resp, is_error=False):
        """Build internal :class:`Response <Response>` object
        from given response.
        """

        def build(resp):

            response = Response()

            # Pass settings over.
            response.config = self.config

            # Fallback to None if there's no staus_code, for whatever reason.
            response.status_code = getattr(resp, 'status', None)

            # Make headers case-insensitive.
            response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))

            # Start off with our local cookies.
            cookies = self.cookies or dict()

            # Add new cookies from the server.
            if 'set-cookie' in response.headers:
                cookie_header = response.headers['set-cookie']

                c = SimpleCookie()
                c.load(cookie_header)

                for k,v in c.items():
                    cookies.update({k: v.value})

            # Save cookies in Response.
            response.cookies = cookies

            # Save original resopnse for later.
            response.raw = resp

            # TODO: ?
            if is_error:
                response.error = resp

            return response

        # Request collector.
        history = []

        # Create the lone response object.
        r = build(resp)
        self.cookies.update(r.cookies)

        # Store the HTTP response, just in case.
        r._response = resp

        # It's a redirect, and we're not already in a redirect loop.
        if r.status_code in REDIRECT_STATI and not self.redirect:

            while (
                # There's a `Location` header.
                ('location' in r.headers) and

                # See other response.
                ((r.status_code is codes.see_other) or

                # Opt-in to redirects for non- idempotent methods.
                (self.allow_redirects))
            ):

                # We already redirected. Don't keep it alive.
                # r.raw.close()

                # Woah, this is getting crazy.
                if len(history) >= self.config.get('max_redirects'):
                    raise TooManyRedirects()

                # Add the old request to the history collector.
                history.append(r)

                url = cleanup_url(r.headers['location'], parent_url=self.url)

                # If 303, convert to idempotent GET.
                # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
                if r.status_code is codes.see_other:
                    method = 'GET'
                else:
                    method = self.method

                # Create the new Request.
                request = Request(
                    url=url,
                    headers=self.headers,
                    files=self.files,
                    method=method,
                    data=self.data,
                    # params=self.params,
                    params=None,
                    auth=self.auth,
                    cookies=self.cookies,
                    _pools=self._pools,
                    config=self.config,

                    # Flag as part of a redirect loop.
                    redirect=True
                )

                # Send her away!
                request.send()
                r = request.response

                self.cookies.update(r.cookies or {})

            # Insert collected history.
            r.history = history

        # Attach Response to Request.
        self.response = r

        # Give Response some context.
        self.response.request = self

    def send(self, anyway=False):
        """Sends the shit."""

        # Safety check.
        self._checks()

        # Build the final URL.
        url = build_url(self.url, self.params)

        # Nottin' on you.
        body = None
        content_type = None

        # Multi-part file uploads.
        if self.files:
            if not isinstance(self.data, basestring):
                fields = self.data.copy()
                for (k, v) in self.files.items():
                    fields.update({k: (k, v.read())})
                (body, content_type) = encode_multipart_formdata(fields)

        # Setup form data.
        if self.data and (not body):
            if isinstance(self.data, basestring):
                body = self.data
            else:
                body = encode_params(self.data)
                content_type = 'application/x-www-form-urlencoded'

        # Setup cookies.

        # Add content-type if it wasn't explicitly provided.
        if (content_type) and (not 'content-type' in self.headers):
            self.headers['Content-Type'] = content_type

        # Only send the Request if new or forced.
        if (anyway) or (not self.sent):

            try:
                # Create a new HTTP connection, since one wasn't passed in.
                if not self._pools:

                    # Create a pool manager for this one connection.
                    pools = PoolManager(
                        num_pools=self.config.get('max_connections'),
                        maxsize=1,
                        timeout=self.timeout
                    )

                    # Create a connection.
                    connection = pools.connection_from_url(url)

                    # One-off request. Delay fetching the content until needed.
                    do_block = False
                else:
                    # Create a connection.
                    connection = self._pools.connection_from_url(url)

                    # Syntax sugar.
                    pools = self._pools

                    # Part of a connection pool, so no fancy stuff. Sorry!
                    do_block = True

                if self.cookies:
                    # Skip if 'cookie' header is explicitly set.
                    if 'cookie' not in self.headers:

                        # Simple cookie with our dict.
                        c = SimpleCookie()
                        c.load(self.cookies)

                        # Turn it into a header.
                        cookie_header = c.output(header='').strip()

                        # Attach Cookie header to request.
                        self.headers['Cookie'] = cookie_header

                # Create the connection.
                r = connection.urlopen(
                    method=self.method,
                    url=url,
                    body=body,
                    headers=self.headers,
                    redirect=False,
                    assert_same_host=False,
                    preload_content=do_block,
                    decode_content=False
                )

                # Set the pools manager for redirections, if allowed.
                if self.config.get('keep_alive') and pools:
                    self._pools = pools

            # except (urllib2.HTTPError, urllib2.URLError), why:
            except Exception, why:
                print why.__dict__
                # if hasattr(why, 'reason'):
                #     if isinstance(why.reason, socket.timeout):
                #         why = Timeout(why)

                # self._build_response(why, is_error=True)
                print 'FUCK'
                print why

            else:
                # self.response = Response.from_urllib3()
                self._build_response(r)
                self.response.ok = True

        self.sent = self.response.ok

        return self.sent

class Response(object):
    """The core :class:`Response <Response>` object.

    All :class:`Request <Request>` objects contain a :class:`response
    <Response>` attribute, which is an instance of this class.
    """

    def __init__(self):

        self._content = None
        self._content_consumed = False

        #: Integer Code of responded HTTP Status.
        self.status_code = None

        #: Case-insensitive Dictionary of Response Headers.
        #: For example, ``headers['content-encoding']`` will return the
        #: value of a ``'Content-Encoding'`` response header.
        self.headers = CaseInsensitiveDict()

        #: File-like object representation of response (for advanced usage).
        self.raw = None

        #: True if no :attr:`error` occured.
        self.ok = False

        #: Resulting :class:`HTTPError` of request, if one occured.
        self.error = None

        #: A list of :class:`Response <Response>` objects from
        #: the history of the Request. Any redirect responses will end
        #: up here.
        self.history = []

        #: The :class:`Request <Request>` that created the Response.
        self.request = None

        #: A dictionary of Cookies the server sent back.
        self.cookies = None

        #: A dictionary of configuration.
        self.config = None

    def __repr__(self):
        return '<Response [%s]>' % (self.status_code)

    def __nonzero__(self):
        """Returns true if :attr:`status_code` is 'OK'."""

        return not self.error

    def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
        """Iterates over the response data.  This avoids reading the content
        at once into memory for large responses.  The chunk size is the number
        of bytes it should read into memory.  This is not necessarily the
        length of each item returned as decoding can take place.
        """
        if self._content_consumed:
            raise RuntimeError('The content for this response was '
                               'already consumed')

        def generate():
            while 1:
                chunk = self.raw.read(chunk_size)
                if not chunk:
                    break
                yield chunk
            self._content_consumed = True
        gen = generate

        if 'gzip' in self.headers.get('content-encoding', ''):
            gen = stream_decode_gzip(gen)

        if decode_unicode is None:
            decode_unicode = settings.decode_unicode

        if decode_unicode:
            gen = stream_decode_response_unicode(gen, self)

        return gen

    @property
    def content(self):
        """Content of the response, in bytes or unicode
        (if available).
        """

        if self._content is not None:
            return self._content

        if self._content_consumed:
            raise RuntimeError(
                'The content for this response was already consumed')

        # Read the contents.
        self._content = self.raw.read() or self.raw.data

        # Decode GZip'd content.
        if 'gzip' in self.headers.get('content-encoding', ''):
            try:
                self._content = decode_gzip(self._content)
            except zlib.error:
                pass

        # Decode unicode content.
        if self.config.get('decode_unicode'):
            self._content = get_unicode_from_response(self)

        self._content_consumed = True
        return self._content

    def raise_for_status(self):
        """Raises stored :class:`HTTPError` or :class:`URLError`,
        if one occured.
        """

        if self.error:
            raise self.error

        if (self.status_code >= 300) and (self.status_code < 400):
            raise Exception('300 yo')

        elif (self.status_code >= 400) and (self.status_code < 500):
            raise Exception('400 yo')

        elif (self.status_code >= 500) and (self.status_code < 600):
            raise Exception('500 yo')
<commit_msg>Fix accidental-merge docstrings<commit_after># -*- coding: utf-8 -*-

"""
requests.models
~~~~~~~~~~~~~~~

This module contains the primary classes that power Requests.
"""

import urllib
import zlib
from Cookie import SimpleCookie
from urlparse import urlparse, urlunparse, urljoin

from .packages import urllib3
from .packages.urllib3.filepost import encode_multipart_formdata

from ._config import get_config
from .structures import CaseInsensitiveDict
from .utils import *
from .status_codes import codes
from .exceptions import RequestException, Timeout, URLRequired, TooManyRedirects
from .packages.urllib3.poolmanager import PoolManager

REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved)

class Request(object):
    """The :class:`Request <Request>` object. It carries out all functionality of
    Requests. Recommended interface is with the Requests functions.
    """

    def __init__(self,
        url=None, headers=dict(), files=None, method=None, data=dict(),
        params=dict(), auth=None, cookies=None, timeout=None, redirect=False,
        allow_redirects=False, proxies=None, config=None, hooks=None,
        _pools=None):

        if cookies is None:
            cookies = {}

        #: Float describ the timeout of the request.
        #  (Use socket.setdefaulttimeout() as fallback)
        self.timeout = timeout

        #: Request URL.
        self.url = url

        #: Dictonary of HTTP Headers to attach to the :class:`Request <Request>`.
        self.headers = headers or {}

        #: Dictionary of files to multipart upload (``{filename: content}``).
        self.files = files or {}

        #: HTTP Method to use.
        self.method = method

        #: Dictionary or byte of request body data to attach to the
        #: :class:`Request <Request>`.
        self.data = None

        #: Dictionary or byte of querystring data to attach to the
        #: :class:`Request <Request>`.
        self.params = None

        #: True if :class:`Request <Request>` is part of a redirect chain (disables history
        #: and HTTPError storage).
        self.redirect = redirect

        #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``)
        self.allow_redirects = allow_redirects

        # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'})
        self.proxies = proxies

        self.config = get_config(config)

        self.data = data
        self._enc_data = encode_params(data)
        self.params = params
        self._enc_params = encode_params(params)

        #: :class:`Response <Response>` instance, containing
        #: content and metadata of HTTP Response, once :attr:`sent <send>`.
        self.response = Response()

        #: :class:`AuthObject` to attach to :class:`Request <Request>`.
        self.auth = auth

        #: CookieJar to attach to :class:`Request <Request>`.
        self.cookies = cookies

        #: True if Request has been sent.
        self.sent = False

        # Header manipulation and defaults.

        if self.config.get('accept_gzip'):
            self.headers.update({'Accept-Encoding': 'gzip'})

        if headers:
            headers = CaseInsensitiveDict(self.headers)
        else:
            headers = CaseInsensitiveDict()

        for (k, v) in self.config.get('base_headers').items():
            if k not in headers:
                headers[k] = v

        self.headers = headers

        self.hooks = hooks
        self._pools = _pools

    def __repr__(self):
        return '<Request [%s]>' % (self.method)

    def _checks(self):
        """Deterministic checks for consistency."""

        if not self.url:
            raise URLRequired

    def _build_response(self, resp, is_error=False):
        """Build internal :class:`Response <Response>` object
        from given response.
        """

        def build(resp):

            response = Response()

            # Pass settings over.
            response.config = self.config

            # Fallback to None if there's no staus_code, for whatever reason.
            response.status_code = getattr(resp, 'status', None)

            # Make headers case-insensitive.
            response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None))

            # Start off with our local cookies.
            cookies = self.cookies or dict()

            # Add new cookies from the server.
            if 'set-cookie' in response.headers:
                cookie_header = response.headers['set-cookie']

                c = SimpleCookie()
                c.load(cookie_header)

                for k,v in c.items():
                    cookies.update({k: v.value})

            # Save cookies in Response.
            response.cookies = cookies

            # Save original resopnse for later.
            response.raw = resp

            # TODO: ?
            if is_error:
                response.error = resp

            return response

        # Request collector.
        history = []

        # Create the lone response object.
        r = build(resp)
        self.cookies.update(r.cookies)

        # Store the HTTP response, just in case.
        r._response = resp

        # It's a redirect, and we're not already in a redirect loop.
        if r.status_code in REDIRECT_STATI and not self.redirect:

            while (
                # There's a `Location` header.
                ('location' in r.headers) and

                # See other response.
                ((r.status_code is codes.see_other) or

                # Opt-in to redirects for non- idempotent methods.
                (self.allow_redirects))
            ):

                # We already redirected. Don't keep it alive.
                # r.raw.close()

                # Woah, this is getting crazy.
                if len(history) >= self.config.get('max_redirects'):
                    raise TooManyRedirects()

                # Add the old request to the history collector.
                history.append(r)

                url = cleanup_url(r.headers['location'], parent_url=self.url)

                # If 303, convert to idempotent GET.
                # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
                if r.status_code is codes.see_other:
                    method = 'GET'
                else:
                    method = self.method

                # Create the new Request.
                request = Request(
                    url=url,
                    headers=self.headers,
                    files=self.files,
                    method=method,
                    data=self.data,
                    # params=self.params,
                    params=None,
                    auth=self.auth,
                    cookies=self.cookies,
                    _pools=self._pools,
                    config=self.config,

                    # Flag as part of a redirect loop.
                    redirect=True
                )

                # Send her away!
                request.send()
                r = request.response

                self.cookies.update(r.cookies or {})

            # Insert collected history.
            r.history = history

        # Attach Response to Request.
        self.response = r

        # Give Response some context.
        self.response.request = self

    def send(self, anyway=False):
        """Sends the HTTP Request. Populates `Request.response`.

        Returns True if everything went according to plan.
        """

        # Safety check.
        self._checks()

        # Build the final URL.
        url = build_url(self.url, self.params)

        # Nottin' on you.
        body = None
        content_type = None

        # Multi-part file uploads.
        if self.files:
            if not isinstance(self.data, basestring):
                fields = self.data.copy()
                for (k, v) in self.files.items():
                    fields.update({k: (k, v.read())})
                (body, content_type) = encode_multipart_formdata(fields)

        # Setup form data.
        if self.data and (not body):
            if isinstance(self.data, basestring):
                body = self.data
            else:
                body = encode_params(self.data)
                content_type = 'application/x-www-form-urlencoded'

        # Setup cookies.

        # Add content-type if it wasn't explicitly provided.
        if (content_type) and (not 'content-type' in self.headers):
            self.headers['Content-Type'] = content_type

        # Only send the Request if new or forced.
        if (anyway) or (not self.sent):

            try:
                # Create a new HTTP connection, since one wasn't passed in.
                if not self._pools:

                    # Create a pool manager for this one connection.
                    pools = PoolManager(
                        num_pools=self.config.get('max_connections'),
                        maxsize=1,
                        timeout=self.timeout
                    )

                    # Create a connection.
                    connection = pools.connection_from_url(url)

                    # One-off request. Delay fetching the content until needed.
                    do_block = False
                else:
                    # Create a connection.
                    connection = self._pools.connection_from_url(url)

                    # Syntax sugar.
                    pools = self._pools

                    # Part of a connection pool, so no fancy stuff. Sorry!
                    do_block = True

                if self.cookies:
                    # Skip if 'cookie' header is explicitly set.
                    if 'cookie' not in self.headers:

                        # Simple cookie with our dict.
                        c = SimpleCookie()
                        c.load(self.cookies)

                        # Turn it into a header.
                        cookie_header = c.output(header='').strip()

                        # Attach Cookie header to request.
                        self.headers['Cookie'] = cookie_header

                # Create the connection.
                r = connection.urlopen(
                    method=self.method,
                    url=url,
                    body=body,
                    headers=self.headers,
                    redirect=False,
                    assert_same_host=False,
                    preload_content=do_block,
                    decode_content=False
                )

                # Set the pools manager for redirections, if allowed.
                if self.config.get('keep_alive') and pools:
                    self._pools = pools

            # except (urllib2.HTTPError, urllib2.URLError), why:
            except Exception, why:
                print why.__dict__
                # if hasattr(why, 'reason'):
                #     if isinstance(why.reason, socket.timeout):
                #         why = Timeout(why)

                # self._build_response(why, is_error=True)
                print 'FUCK'
                print why

            else:
                # self.response = Response.from_urllib3()
                self._build_response(r)
                self.response.ok = True

        self.sent = self.response.ok

        return self.sent

class Response(object):
    """The core :class:`Response <Response>` object.

    All :class:`Request <Request>` objects contain a :class:`response
    <Response>` attribute, which is an instance of this class.
    """

    def __init__(self):

        self._content = None
        self._content_consumed = False

        #: Integer Code of responded HTTP Status.
        self.status_code = None

        #: Case-insensitive Dictionary of Response Headers.
        #: For example, ``headers['content-encoding']`` will return the
        #: value of a ``'Content-Encoding'`` response header.
        self.headers = CaseInsensitiveDict()

        #: File-like object representation of response (for advanced usage).
        self.raw = None

        #: True if no :attr:`error` occured.
        self.ok = False

        #: Resulting :class:`HTTPError` of request, if one occured.
        self.error = None

        #: A list of :class:`Response <Response>` objects from
        #: the history of the Request. Any redirect responses will end
        #: up here.
        self.history = []

        #: The :class:`Request <Request>` that created the Response.
        self.request = None

        #: A dictionary of Cookies the server sent back.
        self.cookies = None

        #: A dictionary of configuration.
        self.config = None

    def __repr__(self):
        return '<Response [%s]>' % (self.status_code)

    def __nonzero__(self):
        """Returns true if :attr:`status_code` is 'OK'."""

        return not self.error

    def iter_content(self, chunk_size=10 * 1024, decode_unicode=None):
        """Iterates over the response data.  This avoids reading the content
        at once into memory for large responses.  The chunk size is the number
        of bytes it should read into memory.  This is not necessarily the
        length of each item returned as decoding can take place.
        """
        if self._content_consumed:
            raise RuntimeError('The content for this response was '
                               'already consumed')

        def generate():
            while 1:
                chunk = self.raw.read(chunk_size)
                if not chunk:
                    break
                yield chunk
            self._content_consumed = True
        gen = generate

        if 'gzip' in self.headers.get('content-encoding', ''):
            gen = stream_decode_gzip(gen)

        if decode_unicode is None:
            decode_unicode = settings.decode_unicode

        if decode_unicode:
            gen = stream_decode_response_unicode(gen, self)

        return gen

    @property
    def content(self):
        """Content of the response, in bytes or unicode
        (if available).
        """

        if self._content is not None:
            return self._content

        if self._content_consumed:
            raise RuntimeError(
                'The content for this response was already consumed')

        # Read the contents.
        self._content = self.raw.read() or self.raw.data

        # Decode GZip'd content.
        if 'gzip' in self.headers.get('content-encoding', ''):
            try:
                self._content = decode_gzip(self._content)
            except zlib.error:
                pass

        # Decode unicode content.
        if self.config.get('decode_unicode'):
            self._content = get_unicode_from_response(self)

        self._content_consumed = True
        return self._content

    def raise_for_status(self):
        """Raises stored :class:`HTTPError` or :class:`URLError`,
        if one occured.
        """

        if self.error:
            raise self.error

        if (self.status_code >= 300) and (self.status_code < 400):
            raise Exception('300 yo')

        elif (self.status_code >= 400) and (self.status_code < 500):
            raise Exception('400 yo')

        elif (self.status_code >= 500) and (self.status_code < 600):
            raise Exception('500 yo')
<|endoftext|>


<commit_before>import os

from django.conf import settings
try:
    from django.utils.module_loading import import_by_path
except:  # Added in Django 1.6
    from simpleimages.django_compat import import_by_path

def get_caller():
    caller_text = getattr(
        settings,
        'SIMPLEIMAGES_TRANSFORM_CALLER',
        'simpleimages.callers.default'
    )
    return import_by_path(caller_text)

def perform_transformation(instance, field_names_to_transform=None):
    '''
    Transforms a model based on the fields specified in the
    ``transformed_fields`` attribute. This should map source image
    field names to dictionaries mapping destination field name to
    their transformations. For instance::

        {
            'image': {
                'thumbnail': scale(width=10),
            }
        }

    If ``field_names_to_transform`` is None, then it will transform
    all fields. Otherwise it will only transform
    from those fields specified in ``field_names_to_transform``.

    :param instance: model instance to perform transformations on
    :type instance: instance of :py:class:`django.db.models.Model`

    :param field_names_to_transform: field names on model to perform transformations on
    :type field_names_to_transform: iterable of strings or None
    '''
    for source_field_name, destination_dict in instance.transformed_fields.items():
        if field_names_to_transform is None or source_field_name in field_names_to_transform:
            for destination_field_name, transformation in destination_dict.items():
                arguments = [instance, source_field_name, destination_field_name, transformation]
                get_caller()(transform_field, *arguments)

def transform_field(instance, source_field_name, destination_field_name, transformation):
    '''
    Does an image transformation on a instance. It will get the image
    from the source field attribute of the instnace, then call
    the transformation function with that instance, and finally
    save that transformed image into the destination field attribute
    of the instance.

    .. note::

    If either the source field is blank, or the transformation returns
    a false value then the destination field will be deleted, if it
    exists.

    :param instance: model instance to perform transformations on
    :type instance: instance of :py:class:`django.db.models.Model`

    :param source_field_name: field name on model to find source image
    :type source_field_name: string

    :param destination_field_name: field name on model save transformed image to
    :type destination_field_name: string

    :param transformation: function, such as :py:func:`~.transforms.scale`, that takes an image files and returns a transformed image
    :type transformation: function
    '''

    source_field = getattr(instance, source_field_name)
    destination_field = getattr(instance, destination_field_name)

    transformed_image = get_transformed_image(source_field, transformation)
    if transformed_image:
        destination_name = os.path.basename(source_field.name)

        destination_field.save(
            destination_name,
            transformed_image,
            save=False
        )
    elif destination_field:
        destination_field.delete()
    else:
        return
    instance.save(update_fields=[destination_field_name])

def get_transformed_image(source, transformation):
    if source:
        source.open()
        return transformation(source)
<commit_msg>Save height and width field (fixes #14)<commit_after>import os

from django.conf import settings
try:
    from django.utils.module_loading import import_by_path
except:  # Added in Django 1.6
    from simpleimages.django_compat import import_by_path

def get_caller():
    caller_text = getattr(
        settings,
        'SIMPLEIMAGES_TRANSFORM_CALLER',
        'simpleimages.callers.default'
    )
    return import_by_path(caller_text)

def perform_transformation(instance, field_names_to_transform=None):
    '''
    Transforms a model based on the fields specified in the
    ``transformed_fields`` attribute. This should map source image
    field names to dictionaries mapping destination field name to
    their transformations. For instance::

        {
            'image': {
                'thumbnail': scale(width=10),
            }
        }

    If ``field_names_to_transform`` is None, then it will transform
    all fields. Otherwise it will only transform
    from those fields specified in ``field_names_to_transform``.

    :param instance: model instance to perform transformations on
    :type instance: instance of :py:class:`django.db.models.Model`

    :param field_names_to_transform: field names on model to perform transformations on
    :type field_names_to_transform: iterable of strings or None
    '''
    for source_field_name, destination_dict in instance.transformed_fields.items():
        if field_names_to_transform is None or source_field_name in field_names_to_transform:
            for destination_field_name, transformation in destination_dict.items():
                arguments = [instance, source_field_name, destination_field_name, transformation]
                get_caller()(transform_field, *arguments)

def transform_field(instance, source_field_name, destination_field_name, transformation):
    '''
    Does an image transformation on a instance. It will get the image
    from the source field attribute of the instnace, then call
    the transformation function with that instance, and finally
    save that transformed image into the destination field attribute
    of the instance.

    .. note::

    If either the source field is blank, or the transformation returns
    a false value then the destination field will be deleted, if it
    exists.

    :param instance: model instance to perform transformations on
    :type instance: instance of :py:class:`django.db.models.Model`

    :param source_field_name: field name on model to find source image
    :type source_field_name: string

    :param destination_field_name: field name on model save transformed image to
    :type destination_field_name: string

    :param transformation: function, such as :py:func:`~.transforms.scale`, that takes an image files and returns a transformed image
    :type transformation: function
    '''

    source_field = getattr(instance, source_field_name)
    destination_field = getattr(instance, destination_field_name)
    update_fields = [destination_field_name]
    transformed_image = get_transformed_image(source_field, transformation)
    if transformed_image:
        destination_name = os.path.basename(source_field.name)
        dimension_field_names = [
            destination_field.field.height_field,
            destination_field.field.width_field]
        update_fields += filter(None, dimension_field_names)
        destination_field.save(
            destination_name,
            transformed_image,
            save=False
        )
    elif destination_field:
        destination_field.delete()
    else:
        return
    instance.save(update_fields=update_fields)

def get_transformed_image(source, transformation):
    if source:
        source.open()
        return transformation(source)
<|endoftext|>