<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Turbinia task."""

from __future__ import unicode_literals

from datetime import datetime
import errno
import json
import logging
import os
import pickle
import platform
import subprocess
import time
import traceback
import uuid

from turbinia import config
from turbinia import output_manager
from turbinia import TurbiniaException

log = logging.getLogger('turbinia')

class TurbiniaTaskResult(object):
  """Object to store task results to be returned by a TurbiniaTask.

  Attributes:
      base_output_dir: Base path for local output
      output_dir: Full path for local output
      error: Dict of error data ('error' and 'traceback' are some valid keys)
      evidence: List of newly created Evidence objects.
      input_evidence: The evidence this task processed.
      request_id: The id of the initial request to process this evidence.
      run_time: Length of time the task ran for.
      saved_paths: Paths where output has been saved.
      start_time: Datetime object of when the task was started
      status: A one line descriptive task status.
      successful: Bool indicating success status.
      task_id: Task ID of the parent task.
      task_name: Name of parent task.
      worker_name: Name of worker task executed on.
      _log: A list of log messages
      _output_manager: An output manager object
  """

  # The list of attributes that we will persist into storage
  STORED_ATTRIBUTES = ['worker_name', 'status', 'saved_paths', 'successful']

  def __init__(
      self,
      evidence=None,
      input_evidence=None,
      task_id=None,
      task_name=None,
      base_output_dir=None,
      request_id=None):
    """Initialize the TurbiniaTaskResult object."""

    self.evidence = evidence if evidence else []
    self.input_evidence = input_evidence if input_evidence else []
    self.task_id = task_id
    self.task_name = task_name
    self.base_output_dir = base_output_dir
    self.request_id = request_id

    self.start_time = datetime.now()
    self.run_time = None
    self.saved_paths = []
    self.successful = None
    self.status = None
    self.error = {}
    self.worker_name = platform.node()
    # TODO(aarontp): Create mechanism to grab actual python logging data.
    self._log = []
    self.output_manager = output_manager.OutputManager(self)
    self.output_dir = self.output_manager.get_local_output_dir()

  def close(self, success, status=None):
    """Handles closing of this result and writing logs.

    Normally this should be called by the Run method to make sure that the
    status, etc are set correctly, but if there is an exception thrown when the
    task executes, then run_wrapper will call this with default arguments
    indicating a failure.

    Args:
      success: Bool indicating task success
      status: One line descriptive task status.
    """
    self.successful = success
    self.run_time = datetime.now() - self.start_time
    if not status:
      status = 'Completed successfully in {0:s} on {1:s}'.format(
          str(self.run_time), self.worker_name)
    self.log(status)

    for evidence in self.evidence:
      if evidence.local_path:
        self.saved_paths.append(evidence.local_path)
        self.output_manager.save_evidence(evidence, self)
      if not evidence.request_id:
        evidence.request_id = self.request_id

    try:
      self.input_evidence.postprocess()
    # Adding a broad exception here because we want to try post-processing
    # to clean things up even after other failures in the task, so this could
    # also fail.
    # pylint: disable=broad-except
    except Exception as e:
      msg = 'Evidence post-processing for {0:s} failed: {1!s}'.format(
          self.input_evidence.name, e)
      log.error(msg)
      self.log(msg)

    # Write result log info to file
    logfile = os.path.join(self.output_dir, 'worker-log.txt')
    if self.output_dir and os.path.exists(self.output_dir):
      with open(logfile, 'w') as f:
        f.write('\n'.join(self._log))
        f.write('\n')
      self.output_manager.save_local_file(logfile, self)

    # Unset the output manager during the close because it won't serialize
    self.output_manager = None
    self.status = status

  def log(self, log_msg):
    """Add a log message to the result object.

    Args:
      log_msg: A log message string.
    """
    log.info(log_msg)
    self._log.append(log_msg)

  def add_evidence(self, evidence):
    """Populate the results list.

    Args:
        evidence: Evidence object
    """
    self.evidence.append(evidence)

  def set_error(self, error, traceback_):
    """Add error and traceback.

    Args:
        error: Short string describing the error.
        traceback: Traceback of the error.
    """
    self.error['error'] = error
    self.error['traceback'] = traceback_

class TurbiniaTask(object):
  """Base class for Turbinia tasks.

  Attributes:
      base_output_dir: The base directory that output will go into.  Per-task
                       directories will be created under this.
      id: Unique Id of task (string of hex)
      last_update: A datetime object with the last time the task was updated.
      name: Name of task
      output_dir: The directory output will go into (including per-task folder).
      result: A TurbiniaTaskResult object.
      request_id: The id of the initial request to process this evidence.
      state_key: A key used to manage task state
      stub: The task manager implementation specific task stub that exists
            server side to keep a reference to the remote task objects.  For PSQ
            this is a task result object, but other implementations have their
            own stub objects.
  """

  # The list of attributes that we will persist into storage
  STORED_ATTRIBUTES = ['id', 'last_update', 'name', 'request_id']

  def __init__(self, name=None, base_output_dir=None, request_id=None):
    """Initialization for TurbiniaTask."""
    self.base_output_dir = base_output_dir
    self.id = uuid.uuid4().hex
    self.last_update = datetime.now()
    self.name = name if name else self.__class__.__name__
    self.output_dir = None
    self.result = None
    self.request_id = request_id
    self.state_key = None
    self.stub = None

  def execute(self, cmd, result, save_files=None, new_evidence=None,
              close=False):
    """Executes a given binary and saves output.

    Args:
      cmd (list): Command arguments to run
      result (TurbiniaTaskResult): The result object to put data into.
      save_files (list): A list of files to save (files referenced by Evidence
          objects are automatically saved, so no need to include them).
      new_evidence (list): These are new evidence objects created by the task.
          If the task is successful, they will be added to the result.
      close (bool): Whether to close out the result.

    Returns:
      Tuple of the return code, and the TurbiniaTaskResult object
    """
    save_files = save_files if save_files else []
    new_evidence = new_evidence if new_evidence else []
    proc = subprocess.Popen(cmd)
    stdout, stderr = proc.communicate()
    result.error['stdout'] = stdout
    result.error['stderr'] = stderr
    ret = proc.returncode

    if ret:
      msg = 'Execution failed with status {0:d}'.format(ret)
      result.log(msg)
      if close:
        result.close(success=False, status=msg)
    else:
      for file_ in save_files:
        result.log('Output file at {0:s}'.format(file_))
        result.output_manager.save_local_file(file_, result)
      for evidence in new_evidence:
        result.add_evidence(evidence)

      if close:
        result.close(success=True)

    return (ret, result)

  def setup(self, evidence):
    """Perform common setup operations and runtime environment.

    Even though TurbiniaTasks are initially instantiated by the Jobs under the
    Task Manager, this setup method needs to be run from the task on the worker
    because it handles setting up the task runtime environment.

    Args:
      evidence: An Evidence object to process.

    Returns:
      A TurbiniaTaskResult object.

    Raises:
      TurbiniaException: If the evidence can not be found.
    """
    if not self.result:
      self.result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          input_evidence=evidence,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
    self.output_dir = self.result.output_dir

    if evidence.copyable and not config.SHARED_FILESYSTEM:
      self.result.output_manager.retrieve_evidence(evidence)

    if evidence.local_path and not os.path.exists(evidence.local_path):
      raise TurbiniaException(
          'Evidence local path {0:s} does not exist'.format(
              evidence.local_path))
    evidence.preprocess()
    return self.result

  def touch(self):
    """Updates the last_update time of the task."""
    self.last_update = datetime.now()

  def result_check(self, result):
    """Checks to make sure that the result is serializeable.

    We occasionally get something added into a TurbiniaTaskResult that makes
    it unpickleable.  We don't necessarily know what caused it to be in that
    state, so we need to create a new, mostly empty result so that the client
    is able to get the error message (otherwise the task will stay pending
    indefinitely).

    Args:
      result (TurbiniaTaskResult): Result object to check

    Returns:
      The original result object if it is OK, otherwise an empty result object
      indicating a failure.
    """
    try:
      pickle.dumps(result)
    except (TypeError, pickle.PicklingError) as e:
      msg = ('Error pickling TurbiniaTaskResult object. Returning a new result '
             'with the pickling error, and all previous result data will be '
             'lost. Pickle Error: {0!s}'.format(e))
      log.error(msg)
      log.error('Pickle error traceback: {0:s}'.format(traceback.format_exc()))
      result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
      result.set_error(e.message, traceback.format_exc())
      result.close(False, status=msg)

    return result

  def run_wrapper(self, evidence):
    """Wrapper to manage TurbiniaTaskResults and exception handling.

    This wrapper should be called to invoke the run() methods so it can handle
    the management of TurbiniaTaskResults and the exception handling.  Otherwise
    details from exceptions in the worker cannot be propogated back to the
    Turbinia TaskManager.

    Args:
      evidence: Evidence object

    Returns:
      A TurbiniaTaskResult object
    """
    log.info('Starting Task {0:s} {1:s}'.format(self.name, self.id))
    if not self.result:
      self.result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          input_evidence=evidence,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
    try:
      self.result = self.setup(evidence)
      self.result = self.run(evidence, self.result)
    # pylint: disable=broad-except
    except Exception as e:
      msg = 'Task failed with exception: [{0!s}]'.format(e)
      log.error(msg)
      self.result.close(success=False, status=msg)
      self.result.set_error(e.message, traceback.format_exc())

    result = self.result_check(self.result)

    return result

  def run(self, evidence, result):
    """Entry point to execute the task.

    Args:
      evidence: Evidence object.
      result: A TurbiniaTaskResult object to place task results into.

    Returns:
        TurbiniaTaskResult object.
    """
    raise NotImplementedError
<commit_msg>Auto close result objects after failures (#154)<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Turbinia task."""

from __future__ import unicode_literals

from datetime import datetime
import errno
import json
import logging
import os
import pickle
import platform
import subprocess
import time
import traceback
import uuid

from turbinia import config
from turbinia import output_manager
from turbinia import TurbiniaException

log = logging.getLogger('turbinia')

class TurbiniaTaskResult(object):
  """Object to store task results to be returned by a TurbiniaTask.

  Attributes:
      base_output_dir: Base path for local output
      closed: Boolean indicating whether this result is closed
      output_dir: Full path for local output
      error: Dict of error data ('error' and 'traceback' are some valid keys)
      evidence: List of newly created Evidence objects.
      id: Unique Id of result (string of hex)
      input_evidence: The evidence this task processed.
      request_id: The id of the initial request to process this evidence.
      run_time: Length of time the task ran for.
      saved_paths: Paths where output has been saved.
      start_time: Datetime object of when the task was started
      status: A one line descriptive task status.
      successful: Bool indicating success status.
      task_id: Task ID of the parent task.
      task_name: Name of parent task.
      worker_name: Name of worker task executed on.
      _log: A list of log messages
      _output_manager: An output manager object
  """

  # The list of attributes that we will persist into storage
  STORED_ATTRIBUTES = ['worker_name', 'status', 'saved_paths', 'successful']

  def __init__(
      self,
      evidence=None,
      input_evidence=None,
      task_id=None,
      task_name=None,
      base_output_dir=None,
      request_id=None):
    """Initialize the TurbiniaTaskResult object."""

    self.closed = False
    self.evidence = evidence if evidence else []
    self.input_evidence = input_evidence if input_evidence else []
    self.id = uuid.uuid4().hex
    self.task_id = task_id
    self.task_name = task_name
    self.base_output_dir = base_output_dir
    self.request_id = request_id

    self.start_time = datetime.now()
    self.run_time = None
    self.saved_paths = []
    self.successful = None
    self.status = None
    self.error = {}
    self.worker_name = platform.node()
    # TODO(aarontp): Create mechanism to grab actual python logging data.
    self._log = []
    self.output_manager = output_manager.OutputManager(self)
    self.output_dir = self.output_manager.get_local_output_dir()

  def close(self, success, status=None):
    """Handles closing of this result and writing logs.

    Normally this should be called by the Run method to make sure that the
    status, etc are set correctly, but if there is an exception thrown when the
    task executes, then run_wrapper will call this with default arguments
    indicating a failure.

    Args:
      success: Bool indicating task success
      status: One line descriptive task status.
    """
    self.successful = success
    self.run_time = datetime.now() - self.start_time
    if not status:
      status = 'Completed successfully in {0:s} on {1:s}'.format(
          str(self.run_time), self.worker_name)
    self.log(status)

    for evidence in self.evidence:
      if evidence.local_path:
        self.saved_paths.append(evidence.local_path)
        self.output_manager.save_evidence(evidence, self)
      if not evidence.request_id:
        evidence.request_id = self.request_id

    try:
      self.input_evidence.postprocess()
    # Adding a broad exception here because we want to try post-processing
    # to clean things up even after other failures in the task, so this could
    # also fail.
    # pylint: disable=broad-except
    except Exception as e:
      msg = 'Evidence post-processing for {0:s} failed: {1!s}'.format(
          self.input_evidence.name, e)
      log.error(msg)
      self.log(msg)

    # Write result log info to file
    logfile = os.path.join(self.output_dir, 'worker-log.txt')
    if self.output_dir and os.path.exists(self.output_dir):
      with open(logfile, 'w') as f:
        f.write('\n'.join(self._log))
        f.write('\n')
      self.output_manager.save_local_file(logfile, self)

    # Unset the writers during the close because they don't serialize
    self._output_writers = None
    self.closed = True
    self.status = status
    log.debug('Result close successful. Status is [{0:s}]'.format(self.status))

  def log(self, log_msg):
    """Add a log message to the result object.

    Args:
      log_msg: A log message string.
    """
    log.info(log_msg)
    self._log.append(log_msg)

  def add_evidence(self, evidence):
    """Populate the results list.

    Args:
        evidence: Evidence object
    """
    self.evidence.append(evidence)

  def set_error(self, error, traceback_):
    """Add error and traceback.

    Args:
        error: Short string describing the error.
        traceback: Traceback of the error.
    """
    self.error['error'] = error
    self.error['traceback'] = traceback_

class TurbiniaTask(object):
  """Base class for Turbinia tasks.

  Attributes:
      base_output_dir: The base directory that output will go into.  Per-task
                       directories will be created under this.
      id: Unique Id of task (string of hex)
      last_update: A datetime object with the last time the task was updated.
      name: Name of task
      output_dir: The directory output will go into (including per-task folder).
      result: A TurbiniaTaskResult object.
      request_id: The id of the initial request to process this evidence.
      state_key: A key used to manage task state
      stub: The task manager implementation specific task stub that exists
            server side to keep a reference to the remote task objects.  For PSQ
            this is a task result object, but other implementations have their
            own stub objects.
  """

  # The list of attributes that we will persist into storage
  STORED_ATTRIBUTES = ['id', 'last_update', 'name', 'request_id']

  def __init__(self, name=None, base_output_dir=None, request_id=None):
    """Initialization for TurbiniaTask."""
    self.base_output_dir = base_output_dir
    self.id = uuid.uuid4().hex
    self.last_update = datetime.now()
    self.name = name if name else self.__class__.__name__
    self.output_dir = None
    self.result = None
    self.request_id = request_id
    self.state_key = None
    self.stub = None

  def execute(self, cmd, result, save_files=None, new_evidence=None,
              close=False):
    """Executes a given binary and saves output.

    Args:
      cmd (list): Command arguments to run
      result (TurbiniaTaskResult): The result object to put data into.
      save_files (list): A list of files to save (files referenced by Evidence
          objects are automatically saved, so no need to include them).
      new_evidence (list): These are new evidence objects created by the task.
          If the task is successful, they will be added to the result.
      close (bool): Whether to close out the result.

    Returns:
      Tuple of the return code, and the TurbiniaTaskResult object
    """
    save_files = save_files if save_files else []
    new_evidence = new_evidence if new_evidence else []
    proc = subprocess.Popen(cmd)
    stdout, stderr = proc.communicate()
    result.error['stdout'] = stdout
    result.error['stderr'] = stderr
    ret = proc.returncode

    if ret:
      msg = 'Execution failed with status {0:d}'.format(ret)
      result.log(msg)
      if close:
        result.close(success=False, status=msg)
    else:
      for file_ in save_files:
        result.log('Output file at {0:s}'.format(file_))
        result.output_manager.save_local_file(file_, result)
      for evidence in new_evidence:
        result.add_evidence(evidence)

      if close:
        result.close(success=True)

    return (ret, result)

  def setup(self, evidence):
    """Perform common setup operations and runtime environment.

    Even though TurbiniaTasks are initially instantiated by the Jobs under the
    Task Manager, this setup method needs to be run from the task on the worker
    because it handles setting up the task runtime environment.

    Args:
      evidence: An Evidence object to process.

    Returns:
      A TurbiniaTaskResult object.

    Raises:
      TurbiniaException: If the evidence can not be found.
    """
    if not self.result:
      self.result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          input_evidence=evidence,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
    self.output_dir = self.result.output_dir

    if evidence.copyable and not config.SHARED_FILESYSTEM:
      self.result.output_manager.retrieve_evidence(evidence)

    if evidence.local_path and not os.path.exists(evidence.local_path):
      raise TurbiniaException(
          'Evidence local path {0:s} does not exist'.format(
              evidence.local_path))
    evidence.preprocess()
    return self.result

  def touch(self):
    """Updates the last_update time of the task."""
    self.last_update = datetime.now()

  def result_check(self, result):
    """Checks to make sure that the result is serializeable.

    We occasionally get something added into a TurbiniaTaskResult that makes
    it unpickleable.  We don't necessarily know what caused it to be in that
    state, so we need to create a new, mostly empty result so that the client
    is able to get the error message (otherwise the task will stay pending
    indefinitely).

    Args:
      result (TurbiniaTaskResult): Result object to check

    Returns:
      The original result object if it is OK, otherwise an empty result object
      indicating a failure.
    """
    try:
      log.debug('Checking TurbiniaTaskResult for serializability')
      pickle.dumps(result)
      dump_status = 'Successful'
    except (TypeError, pickle.PicklingError) as e:
      msg = ('Error pickling TurbiniaTaskResult object. Returning a new result '
             'with the pickling error, and all previous result data will be '
             'lost. Pickle Error: {0!s}'.format(e))
      log.error(msg)
      log.error('Pickle error traceback: {0:s}'.format(traceback.format_exc()))
      result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
      result.set_error(e.message, traceback.format_exc())
      result.close(False, status=msg)
      dump_status = 'Failed, but replaced with new result object'

    log.info('Result check: {0:s}'.format(dump_status))
    return result

  def run_wrapper(self, evidence):
    """Wrapper to manage TurbiniaTaskResults and exception handling.

    This wrapper should be called to invoke the run() methods so it can handle
    the management of TurbiniaTaskResults and the exception handling.  Otherwise
    details from exceptions in the worker cannot be propogated back to the
    Turbinia TaskManager.

    Args:
      evidence: Evidence object

    Returns:
      A TurbiniaTaskResult object
    """
    log.info('Starting Task {0:s} {1:s}'.format(self.name, self.id))
    if not self.result:
      self.result = TurbiniaTaskResult(
          task_id=self.id,
          task_name=self.name,
          input_evidence=evidence,
          base_output_dir=self.base_output_dir,
          request_id=self.request_id)
    original_result_id = self.result.id
    try:
      self.result = self.setup(evidence)
      self.result = self.run(evidence, self.result)
    # pylint: disable=broad-except
    except Exception as e:
      msg = 'Task failed with exception: [{0!s}]'.format(e)
      log.error(msg)
      log.error(traceback.format_exc())
      if self.result:
        self.result.log(msg)
        self.result.log(traceback.format_exc())
        self.result.set_error(e.message, traceback.format_exc())
      else:
        log.error(
            'No TurbiniaTaskResult object found after task execution.')

    # Trying to close the result if possible so that we clean up what we can.
    # This has a higher liklihood of failing because something must have gone
    # wrong as the Task should have already closed this.
    if self.result and not self.result.closed:
      msg = 'Trying last ditch attempt to close result'
      log.warning(msg)
      self.result.log(msg)

      if self.result.status:
        status = self.result.status
      else:
        status = 'No previous status'
      msg = ('Task Result was auto-closed from task executor on {0:s}.'
             ' {1:s}'.format(self.worker_name, status))
      self.result.log(msg)
      try:
        self.result.close(False, msg)
      # Using broad except here because lots can go wrong due to the reasons
      # listed above.
      # pylint: disable=broad-except
      except Exception as e:
        log.error('TurbiniaTaskResult close failed: {0!s}'.format(e))

    result = self.result_check(self.result)
    if original_result_id != self.result.id:
      log.debug(
          'Result object {0:s} is different from original {1:s} after task '
          'execution which indicates errors during execution'.format(
              self.result.id, original_result_id))
    else:
      log.debug(
          'Returning original result object {0:s} after task execution'.format(
              self.result.id))
    return result

  def run(self, evidence, result):
    """Entry point to execute the task.

    Args:
      evidence: Evidence object.
      result: A TurbiniaTaskResult object to place task results into.

    Returns:
        TurbiniaTaskResult object.
    """
    raise NotImplementedError
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##
## This file is part of Indico.
## Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
##
## Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 3 of the
## License, or (at your option) any later version.
##
## Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Indico; if not, see <http://www.gnu.org/licenses/>.

import argparse
import itertools

from MaKaC.plugins.RoomBooking.default.dalManager import DALManager
from MaKaC.plugins.RoomBooking.default.room import Room
from indico.core.db import DBMgr
from indico.util.string import natural_sort_key

def _main(args):
    if not args.locations:
        rooms = Room.find_all()
    else:
        rooms = itertools.chain.from_iterable(Room.find_all(location=loc) for loc in args.locations)

    print 'Month\tYear\tRoom'
    for room in rooms:
        print '{1:.3f}\t{2:.3f}\t{0}'.format(room.getFullName(),
                                             room.getMyAverageOccupation('pastmonth') * 100,
                                             room.getMyAverageOccupation('pastyear') * 100)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--location', '-l', action='append', dest='locations')
    args = parser.parse_args()
    with DBMgr.getInstance().global_connection():
        DALManager.connect()
        try:
            _main(args)
        finally:
            DALManager.disconnect()

if __name__ == '__main__':
    main()
<commit_msg>Fix room_occupation script<commit_after># -*- coding: utf-8 -*-
##
##
## This file is part of Indico.
## Copyright (C) 2002 - 2014 European Organization for Nuclear Research (CERN).
##
## Indico is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 3 of the
## License, or (at your option) any later version.
##
## Indico is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Indico; if not, see <http://www.gnu.org/licenses/>.

import argparse
from datetime import date

from dateutil.relativedelta import relativedelta

from indico.modules.rb.models.rooms import Room
from indico.modules.rb.models.locations import Location
from indico.modules.rb.statistics import calculate_rooms_occupancy
from indico.web.flask.app import make_app

def _main(args):
    today = date.today()
    past_month = today - relativedelta(months=1)
    past_year = today - relativedelta(years=1)

    if not args.locations:
        rooms = Room.find_all()
    else:
        rooms = Room.find_all(Location.name.in_(loc for loc in args.locations), _join=Location)

    print 'Month\tYear\tRoom'
    for room in rooms:
        print '{1:.3f}\t{2:.3f}\t{0}'.format(room.full_name,
                                             calculate_rooms_occupancy([room], past_month, today) * 100,
                                             calculate_rooms_occupancy([room], past_year, today) * 100)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--location', '-l', action='append', dest='locations')
    args = parser.parse_args()
    with make_app().app_context():
        _main(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging  # noqa @TODO:
import hmac
import base64
import datetime
import calendar
import json
import urllib.parse
from requests import request
from hashlib import sha1 as sha
from flask import current_app

import superdesk

from .exceptions import APIGracefulException

# @TODO: for debug purpose
from pprint import pprint  # noqa

def get_incandescent_results(href):
    with superdesk.app.app_context():
        uid = current_app.config['INCANDESCENT_UID']
        apiKey = current_app.config['INCANDESCENT_APIKEY']
    expires = datetime.datetime.now() - datetime.timedelta(minutes=110)
    utc_expires = calendar.timegm(expires.timetuple())
    to_string = str(uid) + "\n" + str(utc_expires)
    binary_signature = hmac.new(apiKey.encode(), to_string.encode(), sha)
    signature = urllib.parse.quote_plus(
        base64.b64encode(binary_signature.digest()))
    images = [href]

    add_data = {
        'uid': uid,
        'expires': utc_expires,
        'signature': signature,
        'images': images
    }
    add_headers = {'Content-type': 'application/json'}
    add_response = request(
        'POST', 'https://incandescent.xyz/api/add/', data=json.dumps(add_data), headers=add_headers)
    if add_response.status_code != 200:
        raise APIGracefulException(add_response)

    add_result = add_response.json()
    if 'project_id' not in add_result:
        raise APIGracefulException(add_result)
    get_data = {
        'uid': uid,
        'expires': utc_expires,
        'signature': signature,
        'project_id': add_result['project_id']
    }
    return get_data

def get_incandescent_results_callback(get_data):
    get_headers = {'Content-type': 'application/json'}
    get_response = request(
        'POST', 'https://incandescent.xyz/api/get/', data=json.dumps(get_data), headers=get_headers
    )
    raw_results = get_response.json()
    if raw_results.get('status', None) in [710, 755]:
        raise(APIGracefulException(raw_results))

    verification_results = {}
    verification_stats = {}
    try:
        for url, data in raw_results.items():
            for page_n, page in data['pages'].items():
                source = page['source']
                key = url.replace('.', '_')
                if source not in verification_results:
                    verification_results[source] = {}
                    verification_stats["total_{}".format(source)] = 0
                verification_stats["total_{}".format(source)] += 1
                if key not in verification_results[source]:
                    verification_results[source][key] = {}
                verification_results[source][key][page_n] = page
    except (TypeError, KeyError) as e:
        raise(APIGracefulException((e, raw_results, )))
    return {
        'stats': verification_stats,
        'results': verification_results
    }
<commit_msg>quickfix(server: vpp: incandescent): handle code 755 correctly<commit_after>import logging  # noqa @TODO:
import hmac
import base64
import datetime
import calendar
import json
import urllib.parse
from requests import request
from hashlib import sha1 as sha
from flask import current_app

import superdesk

from .exceptions import APIGracefulException

# @TODO: for debug purpose
from pprint import pprint  # noqa

def get_incandescent_results(href):
    with superdesk.app.app_context():
        uid = current_app.config['INCANDESCENT_UID']
        apiKey = current_app.config['INCANDESCENT_APIKEY']
    expires = datetime.datetime.now() - datetime.timedelta(minutes=110)
    utc_expires = calendar.timegm(expires.timetuple())
    to_string = str(uid) + "\n" + str(utc_expires)
    binary_signature = hmac.new(apiKey.encode(), to_string.encode(), sha)
    signature = urllib.parse.quote_plus(
        base64.b64encode(binary_signature.digest()))
    images = [href]

    add_data = {
        'uid': uid,
        'expires': utc_expires,
        'signature': signature,
        'images': images
    }
    add_headers = {'Content-type': 'application/json'}
    add_response = request(
        'POST', 'https://incandescent.xyz/api/add/', data=json.dumps(add_data), headers=add_headers)
    if add_response.status_code != 200:
        raise APIGracefulException(add_response)

    add_result = add_response.json()
    if 'project_id' not in add_result:
        raise APIGracefulException(add_result)
    get_data = {
        'uid': uid,
        'expires': utc_expires,
        'signature': signature,
        'project_id': add_result['project_id']
    }
    return get_data

def get_incandescent_results_callback(get_data):
    get_headers = {'Content-type': 'application/json'}
    get_response = request(
        'POST', 'https://incandescent.xyz/api/get/', data=json.dumps(get_data), headers=get_headers
    )
    raw_results = get_response.json()
    if raw_results.get('status', None) == 710:
        raise(APIGracefulException(raw_results))
    if raw_results.get('status', None) == 755:
        apis = ['google', 'bing', 'baidu', 'yandex', 'other']
        return {
            'stats': {
                "total_{api}".format(api=api): 0
                for api in apis
            },
            'results': {}
        }

    verification_results = {}
    verification_stats = {}
    try:
        for url, data in raw_results.items():
            for page_n, page in data['pages'].items():
                source = page['source']
                key = url.replace('.', '_')
                if source not in verification_results:
                    verification_results[source] = {}
                    verification_stats["total_{}".format(source)] = 0
                verification_stats["total_{}".format(source)] += 1
                if key not in verification_results[source]:
                    verification_results[source][key] = {}
                verification_results[source][key][page_n] = page
    except (TypeError, KeyError) as e:
        raise(APIGracefulException((e, raw_results, )))
    return {
        'stats': verification_stats,
        'results': verification_results
    }
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from django.utils.version import get_version

VERSION = (0, 3, 0, 'alpha', 1)

__version__ = get_version(VERSION)

logger = logging.getLogger('mutant')

default_app_config = 'mutant.apps.MutantConfig'
<commit_msg>Bumped version number to 0.3.0a2.<commit_after>from __future__ import unicode_literals

import logging

from django.utils.version import get_version

VERSION = (0, 3, 0, 'alpha', 2)

__version__ = get_version(VERSION)

logger = logging.getLogger('mutant')

default_app_config = 'mutant.apps.MutantConfig'
<|endoftext|>


<commit_before>from .reader import Reader
<commit_msg>pulling errors into main package namespace<commit_after>from .reader import Reader
from .errors import BioRadInvalidFileError, BioRadParsingError
<|endoftext|>


<commit_before>"""
This is the main module for PyBlosxom functionality.  PyBlosxom's setup
and default handlers are defined here.
"""
from __future__ import nested_scopes
import os, time, re, sys
import tools
from entries.fileentry import FileEntry

class PyBlosxom:
    """
    This is the main class for PyBlosxom functionality.  It handles
    initialization, defines default behavior, and also pushes the
    request through all the steps until the output is rendered and
    we're complete.
    """
    def __init__(self, request):
        """
        Initializes the PyBlosxom class

        @param request: A L{Pyblosxom.Request.Request} object
        @type request: L{Pyblosxom.Request.Request} object
        """
        self._request = request

    def startup(self):
        """
        The startup step further initializes the Request by setting
        additional information in the _data dict.
        """
        data = self._request.getData()
        pyhttp = self._request.getHttp()
        config = self._request.getConfiguration()

        data['pi_bl'] = ''

        # Get our URL and configure the base_url param
        if pyhttp.has_key('SCRIPT_NAME'):
            if not config.has_key('base_url'):
                config['base_url'] = 'http://%s%s' % (pyhttp['HTTP_HOST'], pyhttp['SCRIPT_NAME'])
        else:
            config['base_url'] = config.get('base_url', '')

        if len(config['datadir']) > 0 and config['datadir'][-1] == os.sep:
            config['datadir'] = config['datadir'][:-1]

    def common_start(self, start_callbacks=1, render=1):
        """
        Common pyblosxom initialization used by other CGI scripts.

        @param start_callbacks: whether (1) or not (0) we add Pyblosxom
            startup method to the startup callback.
        @type  start_callbacks: boolean

        @param render: whether (1) or not (0) we instantiate a renderer
        @type  render: boolean

        @returns: a tuple containing the config and data dicts
        @rtype: (config dict, data dict)
        """
        config = self._request.getConfiguration()
        data = self._request.getData()

        # import and initialize plugins
        import plugin_utils
        plugin_utils.initialize_plugins(config.get("plugin_dirs", []), config.get("load_plugins", None))

        # inject our own startup into the callback thing
        if start_callbacks:
            plugin_utils.callbacks.setdefault("startup", []).append(self.startup)

        if render:
            # get the renderer we want to use
            r = config.get("renderer", "blosxom")

            # import the renderer
            r = tools.importName("Pyblosxom.renderers", r)

            # get the renderer object
            r = r.Renderer(self._request, config.get("stdoutput", sys.stdout))

            # go through the renderer callback to see if anyone else
            # wants to render.  the default is the renderer object we
            # figured out from above.  this renderer gets stored in
            # the data dict for downstream processing.
            data['renderer'] = tools.run_callback('renderer',
                                                  {'request': self._request},
                                                  donefunc = lambda x: x != None,
                                                  defaultfunc = lambda x: r)

        # do start callback
        tools.run_callback("start", {'request': self._request})

        # entryparser callback is runned first here to allow other plugins
        # register what file extensions can be used
        data['extensions'] = tools.run_callback("entryparser",
                                        {'txt': blosxom_entry_parser},
                                        mappingfunc=lambda x,y:y,
                                        defaultfunc=lambda x:x)
        return (config, data)

    def run(self):
        """
        Main loop for pyblosxom.
        """
        config, data = self.common_start()

        # allow anyone else to handle the request at this point
        handled = tools.run_callback("handle",
                        {'request': self._request},
                        mappingfunc=lambda x,y:x,
                        donefunc=lambda x:x)

        if not handled == 1:
            self.defaultHandler(config, data)

        # do end callback
        tools.run_callback("end", {'request':self._request})

    def defaultHandler(self, config, data):
        import cgi

        self._request.addHttp( {"form": cgi.FieldStorage() } )
        # process the path info to determine what kind of blog entry(ies)
        # this is
        tools.run_callback("pathinfo",
                               {"request": self._request},
                               donefunc=lambda x:x != None,
                               defaultfunc=blosxom_process_path_info)

        # call the filelist callback to generate a list of entries
        data["entry_list"] = tools.run_callback("filelist",
                                   {"request": self._request},
                                   donefunc=lambda x:x != None,
                                   defaultfunc=blosxom_file_list_handler)

        # we pass the request with the entry_list through the prepare callback
        # giving everyone a chance to transform the data.  the request is
        # modified in place.
        tools.run_callback("prepare", {"request": self._request})

        # now we pass the entry_list through the renderer
        entry_list = data["entry_list"]
        renderer = data['renderer']

        if renderer and not renderer.rendered:
            if entry_list:
                renderer.setContent(entry_list)
                # Log it as success
                tools.run_callback("logrequest",
                        {'filename':config.get('logfile',''),
                         'return_code': '200',
                         'request': self._request})
            else:
                renderer.addHeader('Status', '404 Not Found')
                renderer.setContent(
                    {'title': 'The page you are looking for is not available',
                     'body': 'Somehow I cannot find the page you want. ' +
                     'Go Back to %s?'
                     % (config["base_url"], config["blog_title"])})
                # Log it as failure
                tools.run_callback("logrequest",
                        {'filename':config.get('logfile',''),
                         'return_code': '404',
                         'request': self._request})
            renderer.render()

            # do end callback
            tools.run_callback("end", {'request':self._request})

        elif not renderer:
            output = config.get('stdoutput', sys.stdout)
            output.write("Content-Type: text/plain\n\nThere is something wrong with your setup.\n  Check your config files and verify that your configuration is correct.\n")

     def runCallback(self, callback="help"):
         """
         Generic method to run the engine for a specific callback
         """
         # treated as a non-rendering startup
         config, data = self.common_start(render=0)

         # invoke all callbacks for the 'callback'
         handled = tools.run_callback(callback,
                         {'request': self._request},
                         mappingfunc=lambda x,y:x,
                         donefunc=lambda x:x)

def blosxom_entry_parser(filename, request):
    """
    Open up a *.txt file and read its contents

    @param filename: A filename to extra data and meta data from
    @type filename: string

    @param request: A standard request object
    @type request: L{Pyblosxom.Request.Request} object

    @returns: A dict containing parsed data and meta data with the
            particular file (and plugin)
    @rtype: dict
    """
    config = request.getConfiguration()

    entryData = {}

    try:
        story = open(filename).readlines()
    except IOError:
        raise IOError

    if len(story) > 0:
        entryData['title'] = story.pop(0).strip()

    while len(story) > 0:
        match = re.match(r'^#(\w+)\s+(.*)', story[0])
        if match:
            story.pop(0)
            entryData[match.groups()[0]] = match.groups()[1].strip()
        else:
            break

    # Call the preformat function
    entryData['body'] = tools.run_callback('preformat',
            {'parser': (entryData.get('parser', '')
                    or config.get('parser', 'plain')),
             'story': story,
             'request': request},
            donefunc = lambda x:x != None,
            defaultfunc = lambda x: ''.join(x['story']))

    # Call the postformat callbacks
    tools.run_callback('postformat',
            {'request': request,
             'entry_data': entryData})

    return entryData

def blosxom_file_list_handler(args):
    """
    This is the default handler for getting entries.  It takes the
    request object in and figures out which entries based on the
    default behavior that we want to show and generates a list of
    EntryBase subclass objects which it returns.

    @param args: dict containing the incoming Request object
    @type args: L{Pyblosxom.Request.Request}

    @returns: the content we want to render
    @rtype: list of EntryBase objects
    """
    request = args["request"]

    data = request.getData()
    config = request.getConfiguration()

    if data['bl_type'] == 'dir':
        filelist = tools.Walk(request, data['root_datadir'], int(config['depth']))
    elif data['bl_type'] == 'file':
        filelist = [data['root_datadir']]
    else:
        filelist = []

    entrylist = []
    for ourfile in filelist:
        entry = FileEntry(request, ourfile, data['root_datadir'])
        entrylist.append((entry._mtime, entry))

    # this sorts entries by mtime in reverse order.  entries that have
    # no mtime get sorted to the top.
    entrylist.sort()
    entrylist.reverse()
    entrylist = [x[1] for x in entrylist]

    # Match dates with files if applicable
    if data['pi_yr']:
        month = (data['pi_mo'] in tools.month2num.keys() and tools.month2num[data['pi_mo']] or data['pi_mo'])
        matchstr = "^" + data["pi_yr"] + month + data["pi_da"]
        valid_list = [x for x in entrylist if re.match(matchstr, x['fulltime'])]
    else:
        valid_list = entrylist

    return valid_list

def blosxom_process_path_info(args):
    """
    Process HTTP PATH_INFO for URI according to path specifications, fill in
    data dict accordingly

    The paths specification looks like this:
        - C{/cat} - category
        - C{/2002} - year
        - C{/2002/Feb} (or 02) - Year and Month
        - C{/cat/2002/Feb/31} - year and month day in category.
        - C{/foo.html} and C{/cat/foo.html} - file foo.* in / and /cat
    To simplify checking, four digits directory name is not allowed.

    @param args: dict containing the incoming Request object
    @type args: L{Pyblosxom.Request.Request}
    """
    request = args['request']
    config = request.getConfiguration()
    data = request.getData()
    pyhttp = request.getHttp()

    form = pyhttp["form"]
    data['flavour'] = (form.has_key('flav') and
            form['flav'].value or
            config.get('defaultFlavour', 'html'))

    path_info = []
    data['pi_yr'] = ''
    data['pi_mo'] = ''
    data['pi_da'] = ''

    if pyhttp.get('PATH_INFO', ''):
        path_info = pyhttp['PATH_INFO'].split('/')
    path_info = [x for x in path_info if x != '']

    data['path_info'] = list(path_info)
    data['root_datadir'] = config['datadir']

    got_date = 0
    for path_data in path_info:
        if not path_data:
            continue
        elif len(path_data) == 4 and path_data.isdigit():
            # We got a hot date here guys :)
            got_date = 1
            break
        else:
            data['pi_bl'] = os.path.join(data['pi_bl'], path_data)

    if got_date:
        # Get messy with dates
        while not (len(path_info[0]) == 4 and path_info[0].isdigit()):
            path_info.pop(0)
        # Year
        data['pi_yr'] = path_info.pop(0)
        # Month
        if path_info and path_info[0] in tools.MONTHS:
            data['pi_mo'] = path_info.pop(0)
            # Day
            if path_info and re.match("^([0-2][0-9]|3[0-1])", path_info[0]):
                # Potential day here
                data['pi_da'] = path_info.pop(0)

        if path_info and path_info[0]:
            # Potential flavour after date
            filename, ext = os.path.splitext(path_info[0])
            if filename == 'index':
                data['flavour'] = ext[1:]

    blog_result = os.path.join(config['datadir'], data['pi_bl'])

    data['bl_type'] = ''

    # If all we've got is a directory, things are simple
    if os.path.isdir(blog_result):
        if data['pi_bl'] != '':
            config['blog_title'] += ' : %s' % data['pi_bl']
        data['root_datadir'] = blog_result
        data['bl_type'] = 'dir'

    # Else we may have a file
    if not data['bl_type']:
        # Try for file

        ext = tools.what_ext(data["extensions"].keys(), blog_result)
        if ext:
            config['blog_title'] += ' : %s' % data['pi_bl']
            data['bl_type'] = 'file'
            data['root_datadir'] = blog_result + '.' + ext
        else:
            # We may have flavour embedded here
            filename, ext = os.path.splitext(blog_result)
            fileext = tools.what_ext(data["extensions"].keys(), filename)
            dirname = os.path.dirname(filename)

            if fileext:
                data['flavour'] = ext[1:]
                data['root_datadir'] = filename + '.' + fileext
                config['blog_title'] += ' : %s' % data['pi_bl']
                data['bl_type'] = 'file'
            elif (os.path.basename(filename) == 'index' and
                    os.path.isdir(dirname)):
                # blanket flavours?
                data['flavour'] = ext[1:]
                if data['pi_bl'] != '':
                    config['blog_title'] += ' : %s' % os.path.dirname(data['pi_bl'])
                data['root_datadir'] = dirname
                data['bl_type'] = 'dir'

    # Construct our final URL
    data['url'] = '%s/%s' % (config['base_url'], data['pi_bl'])

# vim: shiftwidth=4 tabstop=4 expandtab
<commit_msg>Oops--missed a space.  That'll teach me not to test things before I check them in.  (naughty Will!)<commit_after>"""
This is the main module for PyBlosxom functionality.  PyBlosxom's setup
and default handlers are defined here.
"""
from __future__ import nested_scopes
import os, time, re, sys
import tools
from entries.fileentry import FileEntry

class PyBlosxom:
    """
    This is the main class for PyBlosxom functionality.  It handles
    initialization, defines default behavior, and also pushes the
    request through all the steps until the output is rendered and
    we're complete.
    """
    def __init__(self, request):
        """
        Initializes the PyBlosxom class

        @param request: A L{Pyblosxom.Request.Request} object
        @type request: L{Pyblosxom.Request.Request} object
        """
        self._request = request

    def startup(self):
        """
        The startup step further initializes the Request by setting
        additional information in the _data dict.
        """
        data = self._request.getData()
        pyhttp = self._request.getHttp()
        config = self._request.getConfiguration()

        data['pi_bl'] = ''

        # Get our URL and configure the base_url param
        if pyhttp.has_key('SCRIPT_NAME'):
            if not config.has_key('base_url'):
                config['base_url'] = 'http://%s%s' % (pyhttp['HTTP_HOST'], pyhttp['SCRIPT_NAME'])
        else:
            config['base_url'] = config.get('base_url', '')

        if len(config['datadir']) > 0 and config['datadir'][-1] == os.sep:
            config['datadir'] = config['datadir'][:-1]

    def common_start(self, start_callbacks=1, render=1):
        """
        Common pyblosxom initialization used by other CGI scripts.

        @param start_callbacks: whether (1) or not (0) we add Pyblosxom
            startup method to the startup callback.
        @type  start_callbacks: boolean

        @param render: whether (1) or not (0) we instantiate a renderer
        @type  render: boolean

        @returns: a tuple containing the config and data dicts
        @rtype: (config dict, data dict)
        """
        config = self._request.getConfiguration()
        data = self._request.getData()

        # import and initialize plugins
        import plugin_utils
        plugin_utils.initialize_plugins(config.get("plugin_dirs", []), config.get("load_plugins", None))

        # inject our own startup into the callback thing
        if start_callbacks:
            plugin_utils.callbacks.setdefault("startup", []).append(self.startup)

        if render:
            # get the renderer we want to use
            r = config.get("renderer", "blosxom")

            # import the renderer
            r = tools.importName("Pyblosxom.renderers", r)

            # get the renderer object
            r = r.Renderer(self._request, config.get("stdoutput", sys.stdout))

            # go through the renderer callback to see if anyone else
            # wants to render.  the default is the renderer object we
            # figured out from above.  this renderer gets stored in
            # the data dict for downstream processing.
            data['renderer'] = tools.run_callback('renderer',
                                                  {'request': self._request},
                                                  donefunc = lambda x: x != None,
                                                  defaultfunc = lambda x: r)

        # do start callback
        tools.run_callback("start", {'request': self._request})

        # entryparser callback is runned first here to allow other plugins
        # register what file extensions can be used
        data['extensions'] = tools.run_callback("entryparser",
                                        {'txt': blosxom_entry_parser},
                                        mappingfunc=lambda x,y:y,
                                        defaultfunc=lambda x:x)
        return (config, data)

    def run(self):
        """
        Main loop for pyblosxom.
        """
        config, data = self.common_start()

        # allow anyone else to handle the request at this point
        handled = tools.run_callback("handle",
                        {'request': self._request},
                        mappingfunc=lambda x,y:x,
                        donefunc=lambda x:x)

        if not handled == 1:
            self.defaultHandler(config, data)

        # do end callback
        tools.run_callback("end", {'request':self._request})

    def defaultHandler(self, config, data):
        import cgi

        self._request.addHttp( {"form": cgi.FieldStorage() } )
        # process the path info to determine what kind of blog entry(ies)
        # this is
        tools.run_callback("pathinfo",
                               {"request": self._request},
                               donefunc=lambda x:x != None,
                               defaultfunc=blosxom_process_path_info)

        # call the filelist callback to generate a list of entries
        data["entry_list"] = tools.run_callback("filelist",
                                   {"request": self._request},
                                   donefunc=lambda x:x != None,
                                   defaultfunc=blosxom_file_list_handler)

        # we pass the request with the entry_list through the prepare callback
        # giving everyone a chance to transform the data.  the request is
        # modified in place.
        tools.run_callback("prepare", {"request": self._request})

        # now we pass the entry_list through the renderer
        entry_list = data["entry_list"]
        renderer = data['renderer']

        if renderer and not renderer.rendered:
            if entry_list:
                renderer.setContent(entry_list)
                # Log it as success
                tools.run_callback("logrequest",
                        {'filename':config.get('logfile',''),
                         'return_code': '200',
                         'request': self._request})
            else:
                renderer.addHeader('Status', '404 Not Found')
                renderer.setContent(
                    {'title': 'The page you are looking for is not available',
                     'body': 'Somehow I cannot find the page you want. ' +
                     'Go Back to %s?'
                     % (config["base_url"], config["blog_title"])})
                # Log it as failure
                tools.run_callback("logrequest",
                        {'filename':config.get('logfile',''),
                         'return_code': '404',
                         'request': self._request})
            renderer.render()

            # do end callback
            tools.run_callback("end", {'request':self._request})

        elif not renderer:
            output = config.get('stdoutput', sys.stdout)
            output.write("Content-Type: text/plain\n\nThere is something wrong with your setup.\n  Check your config files and verify that your configuration is correct.\n")

    def runCallback(self, callback="help"):
        """
        Generic method to run the engine for a specific callback
        """
        # treated as a non-rendering startup
        config, data = self.common_start(render=0)

        # invoke all callbacks for the 'callback'
        handled = tools.run_callback(callback,
                        {'request': self._request},
                        mappingfunc=lambda x,y:x,
                        donefunc=lambda x:x)

def blosxom_entry_parser(filename, request):
    """
    Open up a *.txt file and read its contents

    @param filename: A filename to extra data and meta data from
    @type filename: string

    @param request: A standard request object
    @type request: L{Pyblosxom.Request.Request} object

    @returns: A dict containing parsed data and meta data with the
            particular file (and plugin)
    @rtype: dict
    """
    config = request.getConfiguration()

    entryData = {}

    try:
        story = open(filename).readlines()
    except IOError:
        raise IOError

    if len(story) > 0:
        entryData['title'] = story.pop(0).strip()

    while len(story) > 0:
        match = re.match(r'^#(\w+)\s+(.*)', story[0])
        if match:
            story.pop(0)
            entryData[match.groups()[0]] = match.groups()[1].strip()
        else:
            break

    # Call the preformat function
    entryData['body'] = tools.run_callback('preformat',
            {'parser': (entryData.get('parser', '')
                    or config.get('parser', 'plain')),
             'story': story,
             'request': request},
            donefunc = lambda x:x != None,
            defaultfunc = lambda x: ''.join(x['story']))

    # Call the postformat callbacks
    tools.run_callback('postformat',
            {'request': request,
             'entry_data': entryData})

    return entryData

def blosxom_file_list_handler(args):
    """
    This is the default handler for getting entries.  It takes the
    request object in and figures out which entries based on the
    default behavior that we want to show and generates a list of
    EntryBase subclass objects which it returns.

    @param args: dict containing the incoming Request object
    @type args: L{Pyblosxom.Request.Request}

    @returns: the content we want to render
    @rtype: list of EntryBase objects
    """
    request = args["request"]

    data = request.getData()
    config = request.getConfiguration()

    if data['bl_type'] == 'dir':
        filelist = tools.Walk(request, data['root_datadir'], int(config['depth']))
    elif data['bl_type'] == 'file':
        filelist = [data['root_datadir']]
    else:
        filelist = []

    entrylist = []
    for ourfile in filelist:
        entry = FileEntry(request, ourfile, data['root_datadir'])
        entrylist.append((entry._mtime, entry))

    # this sorts entries by mtime in reverse order.  entries that have
    # no mtime get sorted to the top.
    entrylist.sort()
    entrylist.reverse()
    entrylist = [x[1] for x in entrylist]

    # Match dates with files if applicable
    if data['pi_yr']:
        month = (data['pi_mo'] in tools.month2num.keys() and tools.month2num[data['pi_mo']] or data['pi_mo'])
        matchstr = "^" + data["pi_yr"] + month + data["pi_da"]
        valid_list = [x for x in entrylist if re.match(matchstr, x['fulltime'])]
    else:
        valid_list = entrylist

    return valid_list

def blosxom_process_path_info(args):
    """
    Process HTTP PATH_INFO for URI according to path specifications, fill in
    data dict accordingly

    The paths specification looks like this:
        - C{/cat} - category
        - C{/2002} - year
        - C{/2002/Feb} (or 02) - Year and Month
        - C{/cat/2002/Feb/31} - year and month day in category.
        - C{/foo.html} and C{/cat/foo.html} - file foo.* in / and /cat
    To simplify checking, four digits directory name is not allowed.

    @param args: dict containing the incoming Request object
    @type args: L{Pyblosxom.Request.Request}
    """
    request = args['request']
    config = request.getConfiguration()
    data = request.getData()
    pyhttp = request.getHttp()

    form = pyhttp["form"]
    data['flavour'] = (form.has_key('flav') and
            form['flav'].value or
            config.get('defaultFlavour', 'html'))

    path_info = []
    data['pi_yr'] = ''
    data['pi_mo'] = ''
    data['pi_da'] = ''

    if pyhttp.get('PATH_INFO', ''):
        path_info = pyhttp['PATH_INFO'].split('/')
    path_info = [x for x in path_info if x != '']

    data['path_info'] = list(path_info)
    data['root_datadir'] = config['datadir']

    got_date = 0
    for path_data in path_info:
        if not path_data:
            continue
        elif len(path_data) == 4 and path_data.isdigit():
            # We got a hot date here guys :)
            got_date = 1
            break
        else:
            data['pi_bl'] = os.path.join(data['pi_bl'], path_data)

    if got_date:
        # Get messy with dates
        while not (len(path_info[0]) == 4 and path_info[0].isdigit()):
            path_info.pop(0)
        # Year
        data['pi_yr'] = path_info.pop(0)
        # Month
        if path_info and path_info[0] in tools.MONTHS:
            data['pi_mo'] = path_info.pop(0)
            # Day
            if path_info and re.match("^([0-2][0-9]|3[0-1])", path_info[0]):
                # Potential day here
                data['pi_da'] = path_info.pop(0)

        if path_info and path_info[0]:
            # Potential flavour after date
            filename, ext = os.path.splitext(path_info[0])
            if filename == 'index':
                data['flavour'] = ext[1:]

    blog_result = os.path.join(config['datadir'], data['pi_bl'])

    data['bl_type'] = ''

    # If all we've got is a directory, things are simple
    if os.path.isdir(blog_result):
        if data['pi_bl'] != '':
            config['blog_title'] += ' : %s' % data['pi_bl']
        data['root_datadir'] = blog_result
        data['bl_type'] = 'dir'

    # Else we may have a file
    if not data['bl_type']:
        # Try for file

        ext = tools.what_ext(data["extensions"].keys(), blog_result)
        if ext:
            config['blog_title'] += ' : %s' % data['pi_bl']
            data['bl_type'] = 'file'
            data['root_datadir'] = blog_result + '.' + ext
        else:
            # We may have flavour embedded here
            filename, ext = os.path.splitext(blog_result)
            fileext = tools.what_ext(data["extensions"].keys(), filename)
            dirname = os.path.dirname(filename)

            if fileext:
                data['flavour'] = ext[1:]
                data['root_datadir'] = filename + '.' + fileext
                config['blog_title'] += ' : %s' % data['pi_bl']
                data['bl_type'] = 'file'
            elif (os.path.basename(filename) == 'index' and
                    os.path.isdir(dirname)):
                # blanket flavours?
                data['flavour'] = ext[1:]
                if data['pi_bl'] != '':
                    config['blog_title'] += ' : %s' % os.path.dirname(data['pi_bl'])
                data['root_datadir'] = dirname
                data['bl_type'] = 'dir'

    # Construct our final URL
    data['url'] = '%s/%s' % (config['base_url'], data['pi_bl'])

# vim: shiftwidth=4 tabstop=4 expandtab
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *
from views import *

urlpatterns = patterns('',
    #(r'multifeatureshapefile/splitbyfield/$',upload_intersection_feature),

    (r'^login', 'django.contrib.auth.views.login'),
    (r'^intersect/testdrawing/$', test_drawing_intersect ),
    (r'^intersect/testpolygon/$', test_poly_intersect),
    (r'^info/organization_scheme/(\d+)$', org_scheme_info ),
    (r'^info/organization_schemes/$', all_org_scheme_info ),
    (r'^(\w+)/(\w+)/(.*)$', organized_intersection_by_name ),
    (r'(\w+)/(.*)$', default_intersection ),
#    (r'^intersect/testpolygon/csv$', test_poly_intersect_csv),
)
<commit_msg>login url in the intersection app was competing with nc_mlpa's login url. Disabling.<commit_after>from django.conf.urls.defaults import *
from views import *

urlpatterns = patterns('',
    #(r'multifeatureshapefile/splitbyfield/$',upload_intersection_feature),

    # (r'^login', 'django.contrib.auth.views.login'),
    (r'^intersect/testdrawing/$', test_drawing_intersect ),
    (r'^intersect/testpolygon/$', test_poly_intersect),
    (r'^info/organization_scheme/(\d+)$', org_scheme_info ),
    (r'^info/organization_schemes/$', all_org_scheme_info ),
    (r'^(\w+)/(\w+)/(.*)$', organized_intersection_by_name ),
    (r'(\w+)/(.*)$', default_intersection ),
#    (r'^intersect/testpolygon/csv$', test_poly_intersect_csv),
)
<|endoftext|>


<commit_before>import platform
import textwrap
import os

import pytest
try:
    from unittest.mock import MagicMock
except:
    from mock import MagicMock

from conan.tools.files import replace_in_file
from conans.test.utils.tools import TestClient

toolchain_props = """
  <ImportGroup Label="PropertySheets">
      <Import Project="conan\\conantoolchain_release_x64.props" />
"""

@pytest.mark.skipif(platform.system() not in ["Windows"], reason="Requires Windows")
def test_msbuildtoolchain_props_with_extra_flags():
    """
    Real test which is injecting some compiler/linker options and other dummy defines and
    checking that they are being processed somehow.

    Expected result: everything was built successfully.
    """
    profile = textwrap.dedent("""\
    include(default)

    [conf]
    tools.build:cxxflags=["/analyze:quiet"]
    tools.build:cflags+=["/doc"]
    tools.build:sharedlinkflags+=["/VERBOSE:UNUSEDLIBS"]
    tools.build:exelinkflags+=["/PDB:mypdbfile"]
    tools.build:defines+=["DEF1", "DEF2"]
    """)
    client = TestClient(path_with_spaces=False)
    client.run("new hello/0.1 --template=msbuild_exe")
    client.save({
        "myprofile": profile
    })
    # Let's import manually the created conantoolchain_release_x64.props
    replace_in_file(MagicMock(), os.path.join(client.current_folder, "hello.vcxproj"),
                    r'  <ImportGroup Label="PropertySheets">', toolchain_props)
    client.run("create . -pr myprofile -tf None")
    assert "/analyze:quiet /doc src/hello.cpp" in client.out
    assert r"/VERBOSE:UNUSEDLIBS /PDB:mypdbfile x64\Release\hello.obj" in client.out
    assert "/D DEF1 /D DEF2" in client.out
    assert "Build succeeded." in client.out
<commit_msg>skip py2 test<commit_after>import platform
import sys
import textwrap
import os

import pytest
try:
    from unittest.mock import MagicMock
except:
    from mock import MagicMock

from conan.tools.files import replace_in_file
from conans.test.utils.tools import TestClient

toolchain_props = """
  <ImportGroup Label="PropertySheets">
      <Import Project="conan\\conantoolchain_release_x64.props" />
"""

@pytest.mark.skipif(sys.version_info.major == 2, reason="Meson not supported in Py2")
@pytest.mark.skipif(platform.system() not in ["Windows"], reason="Requires Windows")
def test_msbuildtoolchain_props_with_extra_flags():
    """
    Real test which is injecting some compiler/linker options and other dummy defines and
    checking that they are being processed somehow.

    Expected result: everything was built successfully.
    """
    profile = textwrap.dedent("""\
    include(default)

    [conf]
    tools.build:cxxflags=["/analyze:quiet"]
    tools.build:cflags+=["/doc"]
    tools.build:sharedlinkflags+=["/VERBOSE:UNUSEDLIBS"]
    tools.build:exelinkflags+=["/PDB:mypdbfile"]
    tools.build:defines+=["DEF1", "DEF2"]
    """)
    client = TestClient(path_with_spaces=False)
    client.run("new hello/0.1 --template=msbuild_exe")
    client.save({
        "myprofile": profile
    })
    # Let's import manually the created conantoolchain_release_x64.props
    replace_in_file(MagicMock(), os.path.join(client.current_folder, "hello.vcxproj"),
                    r'  <ImportGroup Label="PropertySheets">', toolchain_props)
    client.run("create . -pr myprofile -tf None")
    assert "/analyze:quiet /doc src/hello.cpp" in client.out
    assert r"/VERBOSE:UNUSEDLIBS /PDB:mypdbfile x64\Release\hello.obj" in client.out
    assert "/D DEF1 /D DEF2" in client.out
    assert "Build succeeded." in client.out
<|endoftext|>


<commit_before>
import pandas as pd

def series(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    data = df.values
    assert(isinstance(idx, pd.core.index.Index))

    series = []
    for k, c in enumerate(col):
        if df[c].dtype.kind in 'fib':
            v = data[:, k]
            sec = c in kwargs.get('secondary_y', [])
            d = {
                'name': c if not sec else c + ' (right)',
                'yAxis': int(sec),
                'data': [[idx[q], v[q]] for q in range(len(v))],
            }
            if c in kwargs.get('color', []):
                d['color'] = kwargs['color'].get(c)
            if kwargs.get('dashStyle', []):
                d['dashStyle'] = kwargs['dashStyle'].get(c, 'Solid')
            series.append(d)
    return series

def series_range(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    data = df.values
    assert(isinstance(idx, pd.core.index.Index))
    assert(len(col) == 2)
    assert(df[col[0]].dtype.kind in 'if')
    assert(df[col[1]].dtype.kind in 'if')

    series = [{
        'data': [[data[q, 0], data[q, 1]] for q in range(data.shape[0])],
    }]
    if 'color' in kwargs:
        series['color'] = kwargs['color']
    axis_categories = list(idx)

    return axis_categories, series

def series_drilldown(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 2)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'if')

    levone = list(idx.levels[0])
    data = []
    drilldownSeries = []
    for c in levone:
        dfs = df.xs(c)
        ii = dfs.index.values.flatten()
        vv = dfs.values.flatten()

        d1 = {
            'name': c,
            'y': dfs.sum().values[0],
            'drilldown': c if len(dfs) > 1 else None,
        }
        data.append(d1)

        if len(dfs) > 1:
            d2 = {
                'name': c,
                'data': [[str(ii[q]), vv[q]] for q in range(len(ii))],
                'id': c,
            }
            drilldownSeries.append(d2)

    series = [{'name': col[0], 'data': data, 'colorByPoint': True}]
    return series, drilldownSeries

def series_scatter(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 2)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'iO')

    data = df.values.flatten()
    elmt = list(set(data))
    color = kwargs.get('color', {})
    series = []

    for e in elmt:
        dfs = df[df.iloc[:, 0] == e]
        idx = dfs.index
        series.append({'animation': False,
                       'name': e,
                       'color': color.get(e, None),
                       'data': [[idx[k][0], idx[k][1]] for k in range(len(dfs))],
                    })
    return series

def series_bubble(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 3)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'fib')

    names = list(idx.levels[0])
    series = []
    for s in names:
        dfs = df.xs(s)
        v = dfs.values.flatten()
        idxs = dfs.index
        d = {
            'name': s,
            'data': [[idxs[q][0], idxs[q][1], v[q]] for q in range(len(v))],
        }
        if s in kwargs.get('color', []):
            d['color'] = kwargs['color'].get(s)

        series.append(d)

    return series

def series_treemap(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(col) == 1)

    data = df.values
    elmt = list(set(data))
    color = kwargs.get('color', {})
    series = []

    for e in elmt:
        dfs = df[df.iloc[:, 0] == e]
        idx = dfs.index
        series.append({
            'animation': False,
            'name': e,
            'color': color.get(e, None),
            'data': [[idx[k][0], idx[k][1]] for k in range(len(dfs))],
        })
    return series
    pass

def series_heatmap(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.Index))
    for c in col:
        assert(df[c].dtype.kind in 'if')

    dft = df.copy()
    dft.index = range(len(df.index))
    dft.columns = range(len(df.columns))
    res_data = list(dft.stack().swaplevel(0, 1).reset_index().values)
    res_idx = list(df.columns)
    res_col = list(df.index)

    return res_idx, res_col, res_data
<commit_msg>Adding fillColor and LineColor options to series function<commit_after>
import pandas as pd

def series(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    data = df.values
    assert(isinstance(idx, pd.core.index.Index))

    series = []
    for k, c in enumerate(col):
        if df[c].dtype.kind in 'fib':
            v = data[:, k]
            sec = c in kwargs.get('secondary_y', [])
            d = {
                'name': c if not sec else c + ' (right)',
                'yAxis': int(sec),
                'data': [[idx[q], v[q]] for q in range(len(v))],
            }
            if c in kwargs.get('color', []):
                d['color'] = kwargs['color'].get(c)
            if c in kwargs.get('fillColor', []):
                d['type'] = 'area'
                d['fillColor'] = kwargs['fillColor'].get(c)
            if c in kwargs.get('lineColor', []):
                d['lineColor'] = kwargs['lineColor'].get(c)
            if kwargs.get('dashStyle', []):
                d['dashStyle'] = kwargs['dashStyle'].get(c, 'Solid')
            series.append(d)
    return series

def series_range(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    data = df.values
    assert(isinstance(idx, pd.core.index.Index))
    assert(len(col) == 2)
    assert(df[col[0]].dtype.kind in 'if')
    assert(df[col[1]].dtype.kind in 'if')

    series = [{
        'data': [[data[q, 0], data[q, 1]] for q in range(data.shape[0])],
    }]
    if 'color' in kwargs:
        series['color'] = kwargs['color']
    axis_categories = list(idx)

    return axis_categories, series

def series_drilldown(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 2)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'if')

    levone = list(idx.levels[0])
    data = []
    drilldownSeries = []
    for c in levone:
        dfs = df.xs(c)
        ii = dfs.index.values.flatten()
        vv = dfs.values.flatten()

        d1 = {
            'name': c,
            'y': dfs.sum().values[0],
            'drilldown': c if len(dfs) > 1 else None,
        }
        data.append(d1)

        if len(dfs) > 1:
            d2 = {
                'name': c,
                'data': [[str(ii[q]), vv[q]] for q in range(len(ii))],
                'id': c,
            }
            drilldownSeries.append(d2)

    series = [{'name': col[0], 'data': data, 'colorByPoint': True}]
    return series, drilldownSeries

def series_scatter(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 2)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'iO')

    data = df.values.flatten()
    elmt = list(set(data))
    color = kwargs.get('color', {})
    series = []

    for e in elmt:
        dfs = df[df.iloc[:, 0] == e]
        idx = dfs.index
        series.append({'animation': False,
                       'name': e,
                       'color': color.get(e, None),
                       'data': [[idx[k][0], idx[k][1]] for k in range(len(dfs))],
                    })
    return series

def series_bubble(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(idx.levshape) == 3)
    assert(len(col) == 1)
    assert(df[col[0]].dtype.kind in 'fib')

    names = list(idx.levels[0])
    series = []
    for s in names:
        dfs = df.xs(s)
        v = dfs.values.flatten()
        idxs = dfs.index
        d = {
            'name': s,
            'data': [[idxs[q][0], idxs[q][1], v[q]] for q in range(len(v))],
        }
        if s in kwargs.get('color', []):
            d['color'] = kwargs['color'].get(s)

        series.append(d)

    return series

def series_treemap(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.MultiIndex))
    assert(len(col) == 1)

    data = df.values
    elmt = list(set(data))
    color = kwargs.get('color', {})
    series = []

    for e in elmt:
        dfs = df[df.iloc[:, 0] == e]
        idx = dfs.index
        series.append({
            'animation': False,
            'name': e,
            'color': color.get(e, None),
            'data': [[idx[k][0], idx[k][1]] for k in range(len(dfs))],
        })
    return series
    pass

def series_heatmap(df, *args, **kwargs):
    idx = df.index
    col = df.columns
    assert(isinstance(idx, pd.core.index.Index))
    for c in col:
        assert(df[c].dtype.kind in 'if')

    dft = df.copy()
    dft.index = range(len(df.index))
    dft.columns = range(len(df.columns))
    res_data = list(dft.stack().swaplevel(0, 1).reset_index().values)
    res_idx = list(df.columns)
    res_col = list(df.index)

    return res_idx, res_col, res_data
<|endoftext|>


<commit_before>import requests
import sys
from .models import Message

class Telegram:
    # TODO ? : Convert this into a simple array
    # and get value by doing "on_"+"value"
    handlerTypeCallback = {
        "update": "on_update",
        "forward_from": "on_forward",
        "reply_to_message": "on_reply",
        "text": "on_text",
        "audio": "on_audio",
        "document": "on_document",
        "photo": "on_photo",
        "sticker": "on_sticker",
        "video": "on_video",
        "contact": "on_contact",
        "location": "on_location",
        "new_chat_participant": "on_new_chat_carticipant",
        "left_chat_participant": "on_left_chat_participant",
        "new_chat_title": "on_new_chat_title",
        "new_chat_photo": "on_new_chat_photo",
        "delete_chat_Photo": "on_delete_chat_photo",
        "group_chat_created": "on_group_chat_created",
    }

    def __init__(self, api_url, token):
        self.api_url = api_url
        self.access_token = token
        self.loopingUpdateHandler = False
        self.lastID = 0
        self.handlers = []

    def send_request(self, action, params):
        url = "{}{}/{}".format(self.api_url, self.access_token, action)
        r = requests.get(url, params=params)
        try:
            return r.json()
        except ValueError:
            print("There has been a parsing error on this message : {}"
                  .format(r.text))
            return {"ok": False,
                    "why": "Parsing Error",
                    "message": r.text}

    def get_updates(self, offset=0, limit=100, timeout=0):
        return self.send_request("getUpdates", {"offset": offset,
                                                "limit": limit,
                                                "timeout": timeout})

    def send_message(self, chat_id, text,
                     reply_to_message=None,
                     reply_markup=None):
        params = {"chat_id": chat_id, "text": text}
        if reply_to_message is not None:
            params["reply_to_message"] = reply_to_message

        if reply_markup is not None:
            params["reply_markup"] = reply_markup

        return self.send_request("sendMessage", params)

    def forward_message(self, chat_id, from_chat_id, message_id):
        return self.send_request("forwardMessage",
                                 {"chat_id": chat_id,
                                  "from_chat_id": from_chat_id,
                                  "message_id": message_id})

    def get_me(self):
        return self.send_request("getMe")

    def add_handler(self, handler):
        if "callback" not in self.handlers:
            self.handlers.append(handler)

    def remove_handler(self, callback, **kwargs):
        if callback in self.handlers:
            self.handlers.remove(callback)

    def call_handlers(self, message):
        for handler in self.handlers:
            for k, v in self.handlerTypeCallback.items():
                if (k == "update" or hasattr(message, k))\
                   and hasattr(handler, v):
                    try:
                        getattr(handler, v)(self, message)
                    except:
                        print("""Oops, there has been a problem
                        with this handler : {}""".format(handler))
                        print(sys.exc_info())

    def process_updates(self):
        self.loopingUpdateHandler = True
        while self.loopingUpdateHandler:
            notifications = self.get_updates(self.lastID)
            if notifications["ok"] is True:
                for notification in notifications['result']:
                    self.lastID = max(self.lastID, notification["update_id"])+1
                    message = Message(notification["message"])
                    self.call_handlers(message)
            else:
                print("Oops, something went bad : {}".format(notifications))
<commit_msg>Added the ability to not pass parameters to the request.<commit_after>import requests
import sys
from .models import Message

class Telegram:
    # TODO ? : Convert this into a simple array
    # and get value by doing "on_"+"value"
    handlerTypeCallback = {
        "update": "on_update",
        "forward_from": "on_forward",
        "reply_to_message": "on_reply",
        "text": "on_text",
        "audio": "on_audio",
        "document": "on_document",
        "photo": "on_photo",
        "sticker": "on_sticker",
        "video": "on_video",
        "contact": "on_contact",
        "location": "on_location",
        "new_chat_participant": "on_new_chat_carticipant",
        "left_chat_participant": "on_left_chat_participant",
        "new_chat_title": "on_new_chat_title",
        "new_chat_photo": "on_new_chat_photo",
        "delete_chat_Photo": "on_delete_chat_photo",
        "group_chat_created": "on_group_chat_created",
    }

    def __init__(self, api_url, token):
        self.api_url = api_url
        self.access_token = token
        self.loopingUpdateHandler = False
        self.lastID = 0
        self.handlers = []

    def send_request(self, action, params={}):
        url = "{}{}/{}".format(self.api_url, self.access_token, action)
        r = requests.get(url, params=params)
        try:
            return r.json()
        except ValueError:
            print("There has been a parsing error on this message : {}"
                  .format(r.text))
            return {"ok": False,
                    "why": "Parsing Error",
                    "message": r.text}

    def get_updates(self, offset=0, limit=100, timeout=0):
        return self.send_request("getUpdates", {"offset": offset,
                                                "limit": limit,
                                                "timeout": timeout})

    def send_message(self, chat_id, text,
                     reply_to_message=None,
                     reply_markup=None):
        params = {"chat_id": chat_id, "text": text}
        if reply_to_message is not None:
            params["reply_to_message"] = reply_to_message

        if reply_markup is not None:
            params["reply_markup"] = reply_markup

        return self.send_request("sendMessage", params)

    def forward_message(self, chat_id, from_chat_id, message_id):
        return self.send_request("forwardMessage",
                                 {"chat_id": chat_id,
                                  "from_chat_id": from_chat_id,
                                  "message_id": message_id})

    def get_me(self):
        return self.send_request("getMe")

    def add_handler(self, handler):
        if "callback" not in self.handlers:
            self.handlers.append(handler)

    def remove_handler(self, callback, **kwargs):
        if callback in self.handlers:
            self.handlers.remove(callback)

    def call_handlers(self, message):
        for handler in self.handlers:
            for k, v in self.handlerTypeCallback.items():
                if (k == "update" or hasattr(message, k))\
                   and hasattr(handler, v):
                    try:
                        getattr(handler, v)(self, message)
                    except:
                        print("""Oops, there has been a problem
                        with this handler : {}""".format(handler))
                        print(sys.exc_info())

    def process_updates(self):
        self.loopingUpdateHandler = True
        while self.loopingUpdateHandler:
            notifications = self.get_updates(self.lastID)
            if notifications["ok"] is True:
                for notification in notifications['result']:
                    self.lastID = max(self.lastID, notification["update_id"])+1
                    message = Message(notification["message"])
                    self.call_handlers(message)
            else:
                print("Oops, something went bad : {}".format(notifications))
<|endoftext|>


<commit_before>from functools import lru_cache

from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect
from django.template.response import TemplateResponse
from django.utils.text import capfirst
from django.utils.translation import gettext as _

from wagtail.admin import messages
from wagtail.admin.panels import (
    ObjectList,
    TabbedInterface,
    extract_panel_definitions_from_model_class,
)
from wagtail.log_actions import log
from wagtail.models import Site

from .forms import SiteSwitchForm
from .permissions import user_can_edit_setting_type
from .registry import registry

def get_model_from_url_params(app_name, model_name):
    """
    retrieve a content type from an app_name / model_name combo.
    Throw Http404 if not a valid setting type
    """
    model = registry.get_by_natural_key(app_name, model_name)
    if model is None:
        raise Http404
    return model

@lru_cache()
def get_setting_edit_handler(model):
    if hasattr(model, "edit_handler"):
        edit_handler = model.edit_handler
    else:
        panels = extract_panel_definitions_from_model_class(model, ["site"])
        edit_handler = ObjectList(panels)
    return edit_handler.bind_to(model=model)

def edit_current_site(request, app_name, model_name):
    # Redirect the user to the edit page for the current site
    # (or the current request does not correspond to a site, the first site in the list)
    site_request = Site.find_for_request(request)
    site = site_request or Site.objects.first()
    if not site:
        messages.error(
            request,
            _("This setting could not be opened because there is no site defined."),
        )
        return redirect("wagtailadmin_home")
    return redirect("wagtailsettings:edit", app_name, model_name, site.pk)

def edit(request, app_name, model_name, site_pk):
    model = get_model_from_url_params(app_name, model_name)
    if not user_can_edit_setting_type(request.user, model):
        raise PermissionDenied
    site = get_object_or_404(Site, pk=site_pk)

    setting_type_name = model._meta.verbose_name

    instance = model.for_site(site)
    edit_handler = get_setting_edit_handler(model)
    edit_handler = edit_handler.bind_to(instance=instance, request=request)
    form_class = edit_handler.get_form_class()

    if request.method == "POST":
        form = form_class(
            request.POST, request.FILES, instance=instance, for_user=request.user
        )

        if form.is_valid():
            with transaction.atomic():
                form.save()
                log(instance, "wagtail.edit")

            messages.success(
                request,
                _("%(setting_type)s updated.")
                % {"setting_type": capfirst(setting_type_name), "instance": instance},
            )
            return redirect("wagtailsettings:edit", app_name, model_name, site.pk)
        else:
            messages.validation_error(
                request, _("The setting could not be saved due to errors."), form
            )
    else:
        form = form_class(instance=instance, for_user=request.user)

    edit_handler = edit_handler.bind_to(form=form)

    # Show a site switcher form if there are multiple sites
    site_switcher = None
    if Site.objects.count() > 1:
        site_switcher = SiteSwitchForm(site, model)

    return TemplateResponse(
        request,
        "wagtailsettings/edit.html",
        {
            "opts": model._meta,
            "setting_type_name": setting_type_name,
            "instance": instance,
            "edit_handler": edit_handler,
            "form": form,
            "site": site,
            "site_switcher": site_switcher,
            "tabbed": isinstance(edit_handler, TabbedInterface),
        },
    )
<commit_msg>Update settings views to bind instance, request and form at the same time<commit_after>from functools import lru_cache

from django.core.exceptions import PermissionDenied
from django.db import transaction
from django.http import Http404
from django.shortcuts import get_object_or_404, redirect
from django.template.response import TemplateResponse
from django.utils.text import capfirst
from django.utils.translation import gettext as _

from wagtail.admin import messages
from wagtail.admin.panels import (
    ObjectList,
    TabbedInterface,
    extract_panel_definitions_from_model_class,
)
from wagtail.log_actions import log
from wagtail.models import Site

from .forms import SiteSwitchForm
from .permissions import user_can_edit_setting_type
from .registry import registry

def get_model_from_url_params(app_name, model_name):
    """
    retrieve a content type from an app_name / model_name combo.
    Throw Http404 if not a valid setting type
    """
    model = registry.get_by_natural_key(app_name, model_name)
    if model is None:
        raise Http404
    return model

@lru_cache()
def get_setting_edit_handler(model):
    if hasattr(model, "edit_handler"):
        edit_handler = model.edit_handler
    else:
        panels = extract_panel_definitions_from_model_class(model, ["site"])
        edit_handler = ObjectList(panels)
    return edit_handler.bind_to(model=model)

def edit_current_site(request, app_name, model_name):
    # Redirect the user to the edit page for the current site
    # (or the current request does not correspond to a site, the first site in the list)
    site_request = Site.find_for_request(request)
    site = site_request or Site.objects.first()
    if not site:
        messages.error(
            request,
            _("This setting could not be opened because there is no site defined."),
        )
        return redirect("wagtailadmin_home")
    return redirect("wagtailsettings:edit", app_name, model_name, site.pk)

def edit(request, app_name, model_name, site_pk):
    model = get_model_from_url_params(app_name, model_name)
    if not user_can_edit_setting_type(request.user, model):
        raise PermissionDenied
    site = get_object_or_404(Site, pk=site_pk)

    setting_type_name = model._meta.verbose_name

    instance = model.for_site(site)
    edit_handler = get_setting_edit_handler(model)
    form_class = edit_handler.get_form_class()

    if request.method == "POST":
        form = form_class(
            request.POST, request.FILES, instance=instance, for_user=request.user
        )

        if form.is_valid():
            with transaction.atomic():
                form.save()
                log(instance, "wagtail.edit")

            messages.success(
                request,
                _("%(setting_type)s updated.")
                % {"setting_type": capfirst(setting_type_name), "instance": instance},
            )
            return redirect("wagtailsettings:edit", app_name, model_name, site.pk)
        else:
            messages.validation_error(
                request, _("The setting could not be saved due to errors."), form
            )
    else:
        form = form_class(instance=instance, for_user=request.user)

    edit_handler = edit_handler.bind_to(instance=instance, request=request, form=form)

    # Show a site switcher form if there are multiple sites
    site_switcher = None
    if Site.objects.count() > 1:
        site_switcher = SiteSwitchForm(site, model)

    return TemplateResponse(
        request,
        "wagtailsettings/edit.html",
        {
            "opts": model._meta,
            "setting_type_name": setting_type_name,
            "instance": instance,
            "edit_handler": edit_handler,
            "form": form,
            "site": site,
            "site_switcher": site_switcher,
            "tabbed": isinstance(edit_handler, TabbedInterface),
        },
    )
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 23 18:53:50 2017

Unit test for linguistic_analysis.py

@author: duc
"""

import unittest
import linguistic_analysis as la

tweets = ["Is this not a question?!", "Sure! Why not?"]
norm = [[["is", "this", "not", "a", "question"]],[["sure", "why", "not"]]]

class LinguisticAnalysisTests(unittest.TestCase):

    def test_average_question_marks(self):
        self.assertEqual(la.get_average_question_marks(tweets), 1)

    def test_average_exclamation_marks(self):
        self.assertEqual(la.get_average_exclamation_marks(tweets), 1)

    def test_average_word_characters(self):
        number = la.get_average_word_characters(norm)
        self.assertEqual(round(number), 4)

    def test_average_word_syllables(self):
        number = la.get_average_word_syllables(norm)
        self.assertEqual(round(number), 1)

    def test_average_sentence_length(self):
        number = la.get_average_sentence_length(norm)
        self.assertEqual(round(number), 4)

    def test_average_tweet_length(self):
        number = la.get_average_tweet_length(norm)
        self.assertEqual(round(number), 4)

    def test_most_frequent_words(self):
        most = la.get_most_frequent_keywords(norm)
        self.assertEqual(most["question"], 1)
        self.assertEqual(most["sure"], 1)

if __name__ == '__main__':
    unittest.main()

<commit_msg>feat(linguistic_analysis_tests): update unit tests<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Aug 23 18:53:50 2017

Unit test for linguistic_analysis.py

@author: duc
"""

import unittest
import linguistic_analysis as la

tweets = ["Is this not a question?! Interactive introduction reference information", "Sure! Why not?"]
norm = [
        [["is", "this", "not", "a", "question"], ["interactive", "introduction", "reference", "information"]],
        [["sure", "why", "not"]]
]

class LinguisticAnalysisTests(unittest.TestCase):

    def test_average_question_marks(self):
        self.assertEqual(la.get_average_question_marks(tweets), 1)

    def test_average_exclamation_marks(self):
        self.assertEqual(la.get_average_exclamation_marks(tweets), 1)

    def test_average_word_characters(self):
        number = la.get_average_word_characters(norm)
        self.assertEqual(round(number), 6)

    def test_average_word_syllables(self):
        number = la.get_average_word_syllables(norm)
        self.assertEqual(round(number), 2)

    def test_average_sentence_length(self):
        number = la.get_average_sentence_length(norm)
        self.assertEqual(round(number), 4)

    def test_average_tweet_length(self):
        number = la.get_average_tweet_length(norm)
        self.assertEqual(round(number), 6)

    def test_most_frequent_words(self):
        most = la.get_most_frequent_keywords(norm)
        self.assertEqual(most["question"], 1)
        self.assertEqual(most["sure"], 1)

if __name__ == '__main__':
    unittest.main()

<|endoftext|>