<commit_before># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Eager mode tests for the experimental `replicate` transformation."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized

from tensorflow.core.protobuf import cluster_pb2
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import tensorflow_server_pb2
from tensorflow.python import pywrap_tfe
from tensorflow.python.data.experimental.ops import distribute
from tensorflow.python.data.experimental.ops import distribute_options
from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.eager import context
from tensorflow.python.framework import combinations
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.platform import test
from tensorflow.python.training import server_lib

class LocalReplicateTest(test_base.DatasetTestBase, parameterized.TestCase):

  def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
    super(LocalReplicateTest, self).__init__(methodName)
    self._device0 = "/device:CPU:0"
    self._device1 = "/device:CPU:1"
    self._device2 = "/device:CPU:2"

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testBasic(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(100))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(100))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(100))

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testVariableInput(self):
    with ops.device(self._device0):
      counter_var = variable_scope.get_variable(
          "counter", (), dtypes.int32, use_resource=True)
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: counter_var.assign_add(1))
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    self.evaluate(counter_var.initializer)
    with ops.device(self._device0):
      self.assertDatasetProduces(
          dataset0, range(1, 101), requires_initialization=True)
    with ops.device(self._device1):
      self.assertDatasetProduces(
          dataset1, range(101, 201), requires_initialization=True)
    with ops.device(self._device2):
      self.assertDatasetProduces(
          dataset2, range(201, 301), requires_initialization=True)

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyIgnore(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.IGNORE)
      dataset0 = dataset0.with_options(opt)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      get_next0 = self.getNext(dataset0)
    with ops.device(self._device1):
      get_next1 = self.getNext(dataset1)
    with ops.device(self._device2):
      get_next2 = self.getNext(dataset2)

    for _ in range(100):
      self.evaluate(get_next0())
      self.evaluate(get_next1())
      self.evaluate(get_next2())

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyWarn(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.WARN)
      dataset0 = dataset0.with_options(opt)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      get_next0 = self.getNext(dataset0)
    with ops.device(self._device1):
      get_next1 = self.getNext(dataset1)
    with ops.device(self._device2):
      get_next2 = self.getNext(dataset2)

    for _ in range(100):
      self.evaluate(get_next0())
      self.evaluate(get_next1())
      self.evaluate(get_next2())

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyFail(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.FAIL)
      dataset0 = dataset0.with_options(opt)
    with self.assertRaises(errors.FailedPreconditionError):
      replicated_ds = distribute.replicate(dataset0,
                                           [self._device1, self._device2])
      dataset1 = replicated_ds[self._device1]
      dataset2 = replicated_ds[self._device2]

      with ops.device(self._device0):
        get_next0 = self.getNext(dataset0)
      with ops.device(self._device1):
        get_next1 = self.getNext(dataset1)
      with ops.device(self._device2):
        get_next2 = self.getNext(dataset2)

      for _ in range(100):
        self.evaluate(get_next0())
        self.evaluate(get_next1())
        self.evaluate(get_next2())

JOB_NAME = "remote_device"

def _get_server_def(job_name, local_server_port, remote_server_addresses,
                    task_index):
  """Returns a server def with a single job + multiple tasks."""
  cluster_def = cluster_pb2.ClusterDef()
  job_def = cluster_def.job.add()
  job_def.name = job_name
  job_def.tasks[0] = "localhost:%d" % local_server_port

  for i, remote_server_address in enumerate(remote_server_addresses, start=1):
    job_def.tasks[i] = remote_server_address

  server_def = tensorflow_server_pb2.ServerDef(
      cluster=cluster_def,
      job_name=job_name,
      task_index=task_index,
      protocol="grpc")

  return server_def

# Pure eager mode test that sets up a cluster of processes.
class RemoteReplicateTest(test_base.DatasetTestBase, parameterized.TestCase):

  def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
    super(RemoteReplicateTest, self).__init__(methodName)
    self._cached_server1 = server_lib.Server.create_local_server()
    self._cached_server2 = server_lib.Server.create_local_server()
    self._cached_server1_target = self._cached_server1.target[len("grpc://"):]
    self._cached_server2_target = self._cached_server2.target[len("grpc://"):]
    self._device0 = "/job:%s/replica:0/task:0/device:CPU:0" % JOB_NAME
    self._device1 = "/job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME
    self._device2 = "/job:%s/replica:0/task:2/device:CPU:0" % JOB_NAME

  def setUp(self):
    super(RemoteReplicateTest, self).setUp()
    # Start the local server.
    local_port = pywrap_tfe.TF_PickUnusedPortOrDie()
    context.set_server_def(
        server_def=_get_server_def(
            JOB_NAME,
            local_server_port=local_port,
            remote_server_addresses=[
                self._cached_server1_target, self._cached_server2_target
            ],
            task_index=0))

  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def testBasic(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(100))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(100))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(100))

  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def testMap(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(lambda x: x * 2)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(0, 200, 2))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(0, 200, 2))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(0, 200, 2))

  # TODO(b/150821179): Re-enable this test.
  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def _testVariableInput(self):
    with ops.device(self._device0):
      counter_var = variable_scope.get_variable(
          "counter", (), dtypes.int32, use_resource=True)
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: counter_var.assign_add(1))
    with self.assertRaises(errors.InvalidArgumentError):
      replicated_ds = distribute.replicate(dataset0,
                                           [self._device1, self._device2])
      dataset1 = replicated_ds[self._device1]
      dataset2 = replicated_ds[self._device2]
      with ops.device(self._device0):
        get_next0 = self.getNext(dataset0)
      with ops.device(self._device1):
        get_next1 = self.getNext(dataset1)
      with ops.device(self._device2):
        get_next2 = self.getNext(dataset2)
      for _ in range(100):
        self.evaluate(get_next0())
        self.evaluate(get_next1())
        self.evaluate(get_next2())

if __name__ == "__main__":
  ops.enable_eager_execution(
      config=config_pb2.ConfigProto(device_count={"CPU": 3}))
  test.main()
<commit_msg>Re-enable testVariableInput since the underlying issue has been fixed.<commit_after># Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Eager mode tests for the experimental `replicate` transformation."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl.testing import parameterized

from tensorflow.core.protobuf import cluster_pb2
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import tensorflow_server_pb2
from tensorflow.python import pywrap_tfe
from tensorflow.python.data.experimental.ops import distribute
from tensorflow.python.data.experimental.ops import distribute_options
from tensorflow.python.data.kernel_tests import test_base
from tensorflow.python.data.ops import dataset_ops
from tensorflow.python.eager import context
from tensorflow.python.framework import combinations
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import errors
from tensorflow.python.framework import ops
from tensorflow.python.ops import random_ops
from tensorflow.python.ops import variable_scope
from tensorflow.python.platform import test
from tensorflow.python.training import server_lib

class LocalReplicateTest(test_base.DatasetTestBase, parameterized.TestCase):

  def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
    super(LocalReplicateTest, self).__init__(methodName)
    self._device0 = "/device:CPU:0"
    self._device1 = "/device:CPU:1"
    self._device2 = "/device:CPU:2"

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testBasic(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(100))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(100))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(100))

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testVariableInput(self):
    with ops.device(self._device0):
      counter_var = variable_scope.get_variable(
          "counter", (), dtypes.int32, use_resource=True)
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: counter_var.assign_add(1))
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    self.evaluate(counter_var.initializer)
    with ops.device(self._device0):
      self.assertDatasetProduces(
          dataset0, range(1, 101), requires_initialization=True)
    with ops.device(self._device1):
      self.assertDatasetProduces(
          dataset1, range(101, 201), requires_initialization=True)
    with ops.device(self._device2):
      self.assertDatasetProduces(
          dataset2, range(201, 301), requires_initialization=True)

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyIgnore(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.IGNORE)
      dataset0 = dataset0.with_options(opt)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      get_next0 = self.getNext(dataset0)
    with ops.device(self._device1):
      get_next1 = self.getNext(dataset1)
    with ops.device(self._device2):
      get_next2 = self.getNext(dataset2)

    for _ in range(100):
      self.evaluate(get_next0())
      self.evaluate(get_next1())
      self.evaluate(get_next2())

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyWarn(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.WARN)
      dataset0 = dataset0.with_options(opt)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]

    with ops.device(self._device0):
      get_next0 = self.getNext(dataset0)
    with ops.device(self._device1):
      get_next1 = self.getNext(dataset1)
    with ops.device(self._device2):
      get_next2 = self.getNext(dataset2)

    for _ in range(100):
      self.evaluate(get_next0())
      self.evaluate(get_next1())
      self.evaluate(get_next2())

  @combinations.generate(
      combinations.combine(tf_api_version=[1], mode=["graph", "eager"]))
  def testExternalStatePolicyFail(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: random_ops.random_uniform(  # pylint:disable=g-long-lambda
              [],
              minval=1,
              maxval=10,
              dtype=dtypes.float32))
      opt = dataset_ops.Options()
      opt.experimental_external_state_policy = (
          distribute_options.ExternalStatePolicy.FAIL)
      dataset0 = dataset0.with_options(opt)
    with self.assertRaises(errors.FailedPreconditionError):
      replicated_ds = distribute.replicate(dataset0,
                                           [self._device1, self._device2])
      dataset1 = replicated_ds[self._device1]
      dataset2 = replicated_ds[self._device2]

      with ops.device(self._device0):
        get_next0 = self.getNext(dataset0)
      with ops.device(self._device1):
        get_next1 = self.getNext(dataset1)
      with ops.device(self._device2):
        get_next2 = self.getNext(dataset2)

      for _ in range(100):
        self.evaluate(get_next0())
        self.evaluate(get_next1())
        self.evaluate(get_next2())

JOB_NAME = "remote_device"

def _get_server_def(job_name, local_server_port, remote_server_addresses,
                    task_index):
  """Returns a server def with a single job + multiple tasks."""
  cluster_def = cluster_pb2.ClusterDef()
  job_def = cluster_def.job.add()
  job_def.name = job_name
  job_def.tasks[0] = "localhost:%d" % local_server_port

  for i, remote_server_address in enumerate(remote_server_addresses, start=1):
    job_def.tasks[i] = remote_server_address

  server_def = tensorflow_server_pb2.ServerDef(
      cluster=cluster_def,
      job_name=job_name,
      task_index=task_index,
      protocol="grpc")

  return server_def

# Pure eager mode test that sets up a cluster of processes.
class RemoteReplicateTest(test_base.DatasetTestBase, parameterized.TestCase):

  def __init__(self, methodName="runTest"):  # pylint: disable=invalid-name
    super(RemoteReplicateTest, self).__init__(methodName)
    self._cached_server1 = server_lib.Server.create_local_server()
    self._cached_server2 = server_lib.Server.create_local_server()
    self._cached_server1_target = self._cached_server1.target[len("grpc://"):]
    self._cached_server2_target = self._cached_server2.target[len("grpc://"):]
    self._device0 = "/job:%s/replica:0/task:0/device:CPU:0" % JOB_NAME
    self._device1 = "/job:%s/replica:0/task:1/device:CPU:0" % JOB_NAME
    self._device2 = "/job:%s/replica:0/task:2/device:CPU:0" % JOB_NAME

  def setUp(self):
    super(RemoteReplicateTest, self).setUp()
    # Start the local server.
    local_port = pywrap_tfe.TF_PickUnusedPortOrDie()
    context.set_server_def(
        server_def=_get_server_def(
            JOB_NAME,
            local_server_port=local_port,
            remote_server_addresses=[
                self._cached_server1_target, self._cached_server2_target
            ],
            task_index=0))

  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def testBasic(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(100))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(100))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(100))

  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def testMap(self):
    with ops.device(self._device0):
      dataset0 = dataset_ops.Dataset.range(100).map(lambda x: x * 2)
    replicated_ds = distribute.replicate(dataset0,
                                         [self._device1, self._device2])
    dataset1 = replicated_ds[self._device1]
    dataset2 = replicated_ds[self._device2]
    with ops.device(self._device0):
      self.assertDatasetProduces(dataset0, range(0, 200, 2))
    with ops.device(self._device1):
      self.assertDatasetProduces(dataset1, range(0, 200, 2))
    with ops.device(self._device2):
      self.assertDatasetProduces(dataset2, range(0, 200, 2))

  @combinations.generate(
      combinations.combine(tf_api_version=[2], mode=["eager"]))
  def testVariableInput(self):
    with ops.device(self._device0):
      counter_var = variable_scope.get_variable(
          "counter", (), dtypes.int32, use_resource=True)
      dataset0 = dataset_ops.Dataset.range(100).map(
          lambda _: counter_var.assign_add(1))
    with self.assertRaises(errors.InvalidArgumentError):
      replicated_ds = distribute.replicate(dataset0,
                                           [self._device1, self._device2])
      dataset1 = replicated_ds[self._device1]
      dataset2 = replicated_ds[self._device2]
      with ops.device(self._device0):
        get_next0 = self.getNext(dataset0)
      with ops.device(self._device1):
        get_next1 = self.getNext(dataset1)
      with ops.device(self._device2):
        get_next2 = self.getNext(dataset2)
      for _ in range(100):
        self.evaluate(get_next0())
        self.evaluate(get_next1())
        self.evaluate(get_next2())

if __name__ == "__main__":
  ops.enable_eager_execution(
      config=config_pb2.ConfigProto(device_count={"CPU": 3}))
  test.main()
<|endoftext|>


<commit_before>'''prosper_logging.py: extension/standadized logging utilities for prosper'''

from os import path, makedirs, access, W_OK#, R_OK
import logging
from logging.handlers import TimedRotatingFileHandler
import warnings

import requests

from prosper.common.prosper_config import get_config
import prosper.common.prosper_utilities as p_utils

HERE = path.abspath(path.dirname(__file__))
ME = __file__.replace('.py', '')
CONFIG_ABSPATH = path.join(HERE, 'common_config.cfg')

COMMON_CONFIG = get_config(CONFIG_ABSPATH)
DISCORD_MESSAGE_LIMIT = 2000
DISCORD_PAD_SIZE = 100

#DEFAULT_LOGGER = logging.getLogger('NULL')
#DEFAULT_LOGGER.addHandler(logging.NullHandler())

class ReportingFormats:
    '''a handy enum of reporting formats for every occasion'''
    DEFAULT = '[%(asctime)s;%(levelname)s;%(filename)s;%(funcName)s;%(lineno)s] %(message)s'
    PRETTY_PRINT = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s]\n%(message).1400s'
    STDOUT = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s] %(message)s'

class ProsperLogger(object):
    '''container for building a logger for prosper tools'''
    _debug_mode = False
    def __init__(
            self,
            log_name,
            log_path,
            config_obj=COMMON_CONFIG,
            debug_mode=_debug_mode
    ):
        self.logger = logging.getLogger(log_name)
        self.log_options = p_utils.parse_options(config_obj, 'LOGGING')
        self._debug_mode = debug_mode
        self.log_name = log_name
        self.log_path = test_logpath(log_path, debug_mode)

        self.log_info = []
        self.log_handlers = []
        self.configure_default_logger(debug_mode=debug_mode)

    def get_logger(self):
        '''yield logging object for script to work with'''
        return self.logger

    def __str__(self):
        '''for debug purposes, list attached log handlers'''
        return ','.join(self.log_info)

    def configure_default_logger(
            self,
            log_freq=None,
            log_total=None,
            log_level=None,
            log_format=ReportingFormats().DEFAULT,
            debug_mode=_debug_mode
    ):
        '''build default logger object and handlers'''
        try:
            if not log_freq:
                log_freq = self.log_options['log_freq']
            if not log_total:
                log_total = self.log_options['log_total']
            if not log_level:
                log_level = self.log_options['log_level']
        except KeyError as error_msg:
            raise error_msg

        log_filename = self.log_name + '.log'
        log_abspath = path.join(self.log_path, log_filename)
        general_handler = TimedRotatingFileHandler(
            log_abspath,
            when=log_freq,
            interval=1,
            backupCount=int(log_total) #FIXME: defaults
        )

        formatter = logging.Formatter(log_format)
        general_handler.setFormatter(formatter)

        self.logger.setLevel(log_level)
        self.logger.addHandler(general_handler)

        self.log_info.append('default @ ' + str(log_level))
        self.log_handlers.append(general_handler)

    def configure_debug_logger(
            self,
            log_level='DEBUG',
            log_format=ReportingFormats().STDOUT,
            debug_mode=_debug_mode
    ):
        '''attach debug logging info for debug'''
        formatter = logging.Formatter(log_format)
        debug_handler = logging.StreamHandler()
        debug_handler.setFormatter(formatter)
        debug_handler.setLevel(log_level)

        self.logger.addHandler(debug_handler)
        self.logger.setLevel(log_level)     #override log_level minimum

        self.log_info.append('debug @ ' + str(log_level))
        self.log_handlers.append(debug_handler)

    def configure_discord_logger(
            self,
            log_level='ERROR',
            log_format=ReportingFormats().PRETTY_PRINT,
            discord_webhook=None,
            discord_recipient=None,
            debug_mode=_debug_mode
    ):
        '''attach discord options'''

        ## Make sure we can set discord handlers ##
        if not discord_webhook:
            try:
                discord_webhook = self.log_options['discord_webhook']
            except KeyError:
                warnings.warn(
                    'Lacking discord_webhook defintion, unable to attach webhook',
                    ResourceWarning
                )
                return None
                #raise KeyError('Lacking discord_webhook definition')
            finally:
                if not discord_webhook:
                    #raise KeyError('Lacking discord_webhook definition')
                    warnings.warn(
                        'Lacking discord_webhook defintion, unable to attach webhook',
                        ResourceWarning
                    )
                    return None
        ## Actually build discord logging handler ##
        discord_obj = DiscordWebhook()
        discord_obj.webhook(discord_webhook)
        if discord_obj.can_query:
            try:
                discord_handler = HackyDiscordHandler(
                    discord_obj,
                    discord_recipient
                )
                discord_format = logging.Formatter(log_format)
                discord_handler.setFormatter(discord_format)
                discord_handler.setLevel(log_level)
                self.logger.addHandler(discord_handler)
            except Exception as error_msg:
                raise error_msg

            self.log_info.append('discord @ ' + str(log_level))
            self.log_handlers.append(discord_handler)

def test_logpath(log_path, debug_mode=False):
    '''test if logger has access to given path, and set up directories
    RETURNS: valid log_path, after setting up pathing
        '.' or log_path
    '''
    if debug_mode:
        return '.' #if debug, do not deploy to production paths

    ## Try to create path to log ##
    if not path.exists(log_path):
        try:
            makedirs(log_path, exist_ok=True)
        except PermissionError as err_permission:
            #UNABLE TO CREATE LOG PATH
            warning_msg = \
                'Unable to create logging path.  Defaulting to \'.\'' + \
                'log_path={0}'.format(log_path) + \
                'exception={0}'.format(err_permission)
            warnings.warn(
                warning_msg,
                ResourceWarning
            )
            return '.'
        except Exception as err_msg:
            raise err_msg

    ## Make sure logger can write to path ##
    if not access(log_path, W_OK):
        #UNABLE TO WRITE TO LOG PATH
        warning_msg = \
            'Lacking write permissions to path.  Defaulting to \'.\'' + \
            'log_path={0}'.format(log_path) + \
            'exception={0}'.format(err_permission)
        warnings.warn(
            warning_msg,
            ResourceWarning
        )
        return '.'
        #TODO: windows behavior requires abspath to existing file

    return log_path

def create_logger(
        log_name,
        log_path,
        config_obj = None,
        log_level_override = ''
):
    '''creates logging handle for programs'''
    warnings.warn(
        'create_logger replaced with ProsperLogger object',
        DeprecationWarning
    )
    if not config_obj:
        config_obj = COMMON_CONFIG

    if not path.exists(log_path):
        makedirs(log_path)

    Logger = logging.getLogger(log_name)

    log_level = config_obj.get('LOGGING', 'log_level')
    log_freq  = config_obj.get('LOGGING', 'log_freq')
    log_total = config_obj.get('LOGGING', 'log_total')

    log_name = log_name + '.log'
    log_format = '[%(asctime)s;%(levelname)s;%(filename)s;%(funcName)s;%(lineno)s] %(message)s'
    #print(logName + ':' + logLevel)
    if log_level_override:
        log_level = log_level_override

    log_full_path = path.join(log_path, log_name)

    Logger.setLevel(log_level)
    generalHandler = TimedRotatingFileHandler(
        log_full_path,
        when=log_freq,
        interval=1,
        backupCount=log_total
    )
    formatter = logging.Formatter(log_format)
    generalHandler.setFormatter(formatter)
    Logger.addHandler(generalHandler)

    if log_level_override == 'DEBUG':
        #print('LOGGER: adding stdout handler')
        short_format = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s] %(message)s'
        short_formatter = logging.Formatter(short_format)
        stdout = logging.StreamHandler()
        stdout.setFormatter(short_formatter)
        Logger.addHandler(stdout)

    if all([
            config_obj.has_option('LOGGING', 'discord_webhook'),
            config_obj.has_option('LOGGING', 'discord_level'),
            #config_obj.has_option('LOGGING', 'discord_alert_recipient')
    ]):
        #message_maxlength = DISCORD_MESSAGE_LIMIT - DISCORD_PAD_SIZE
        #print('LOGGER: adding discord handler')
        discord_format = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s]\n%(message).1400s'
        alert_recipient = None
        if config_obj.has_option('LOGGING', 'discord_alert_recipient'):
            alert_recipient = config_obj.get('LOGGING', 'discord_alert_recipient')

        discord_obj = DiscordWebhook()
        discord_obj.webhook(config_obj.get('LOGGING', 'discord_webhook'))
        discord_level = config_obj.get('LOGGING', 'discord_level')
        do_discord = all([discord_obj.can_query, discord_level])
        if do_discord:
            try:
                dh = HackyDiscordHandler(
                    discord_obj,
                    alert_recipient
                )
                dh_format = logging.Formatter(discord_format)
                dh.setFormatter(dh_format)
                dh.setLevel(discord_level)
                Logger.addHandler(dh)
            except Exception as error_msg:
                print('UNABLE TO ATTATCH DISCORD HANDLER - ' + str(error_msg))

    return Logger

class DiscordWebhook(object):
    '''parser for webhooks for easier api calling'''
    __base_url = 'https://discordapp.com/api/webhooks/'
    def __init__(self):
        self.webhook_url = ''
        self.serverid = 0
        self.api_key = ''
        self.can_query = False
        self.webhook_response = None

    def webhook(self, webhook_url):
        '''parse pieces of webhook credentials from url'''
        if webhook_url:
            self.can_query = True
        self.webhook_url = webhook_url
        #FIXME vv this is hacky as fuck
        trunc_url = self.webhook_url.replace(self.__base_url, '')
        id_list = trunc_url.split('/')
        self.serverid = int(id_list[0])
        self.api_key = id_list[1]

    def api_keys(self, serverid, api_key):
        '''with a id/api pair, assemble the webhook_url'''
        if serverid and api_key:
            self.can_query = True
        self.serverid = serverid
        self.api_key = api_key
        self.webhook_url = self.__base_url + self.serverid + '/' + self.api_key

    def get_webhook_info(self):
        '''fetch api profile from discord servers'''
        if not self.can_query:
            raise RuntimeError('webhook information not loaded, cannot query')

        raise NotImplementedError('requests call to discord server for API info: TODO')

    def __bool__(self):
        return self.can_query

    def __str__(self):
        return self.webhook_url

class HackyDiscordHandler(logging.Handler):
    '''hacky in-house discord/REST handler'''
    def __init__(self, webhook_obj, alert_recipient=None):
        logging.Handler.__init__(self)
        self.webhook_obj = webhook_obj
        self.api_url = self.webhook_obj.webhook_url
        self.alert_recipient = alert_recipient
        self.alert_length = 0
        if self.alert_recipient:
            self.alert_length = len(self.alert_recipient)

    def emit(self, record):
        '''logging logic goes here'''
        log_msg = self.format(record)
        if len(log_msg) + self.alert_length > DISCORD_MESSAGE_LIMIT:
            log_msg = log_msg[:(DISCORD_MESSAGE_LIMIT - DISCORD_PAD_SIZE)]

        if self.alert_recipient and record.levelno == logging.CRITICAL:
            log_msg = log_msg + '\n' + str(self.alert_recipient)

        self.send_msg_to_webhook(log_msg)

    def send_msg_to_webhook(self, message):
        '''requests framework for sending log message to webhook'''
        payload = {
            'content':message
        }

        header = {
            'Content-Type':'application/json'
        }

        try:
            request = requests.post(
                self.api_url,
                headers=header,
                json=payload
            )
        except Exception as error_msg:
            print(
                'EXCEPTION: UNABLE TO COMMIT LOG MESSAGE' + \
                '\r\texception={0}'.format(error_msg) + \
                '\r\tmessage={0}'.format(message)
            )
    def test(self, message):
        '''hook for testing webhook logic'''
        try:
            self.send_msg_to_webhook(message)
        except Exception as error_msg:
            raise error_msg
class LoggerLevels:
    '''enums for logger'''
    #FIXME vvv import actual logger enum?
    #https://docs.python.org/3/library/logging.html#logging-levels
    CRITICAL = 50
    ERROR = 40
    WARNING = 30
    INFO = 20
    DEBUG = 10
    NOTSET = 0

    def str_to_level(self, log_level):
        '''converts str to int enum'''
        log_level = log_level.upper()
        if  log_level == 'CRITICAL':
            return self.CRITICAL
        elif log_level == 'ERROR':
            return self.ERROR
        elif log_level == 'WARNING':
            return self.WARNING
        elif log_level == 'INFO':
            return self.INFO
        elif log_level == 'DEBUG':
            return self.DEBUG
        else:
            return self.NOTSET

if __name__ == '__main__':
    TEST_LOGGER = create_logger(
        'common_logger_debug',
        '.',
        COMMON_CONFIG,
        'DEBUG'
    )

    TEST_LOGGER.debug('prosper.common.prosper_logging TEST --DEBUG--')
    TEST_LOGGER.info('prosper.common.prosper_logging TEST --INFO--')
    TEST_LOGGER.warning('prosper.common.prosper_logging TEST --WARNING--')
    TEST_LOGGER.error('prosper.common.prosper_logging TEST --ERROR--')
    TEST_LOGGER.critical('prosper.common.prosper_logging TEST --CRITICAL--')
<commit_msg>fixing TimedRotatingFileHandler issue in (deprecated) create_logger() call<commit_after>'''prosper_logging.py: extension/standadized logging utilities for prosper'''

from os import path, makedirs, access, W_OK#, R_OK
import logging
from logging.handlers import TimedRotatingFileHandler
import warnings

import requests

from prosper.common.prosper_config import get_config
import prosper.common.prosper_utilities as p_utils

HERE = path.abspath(path.dirname(__file__))
ME = __file__.replace('.py', '')
CONFIG_ABSPATH = path.join(HERE, 'common_config.cfg')

COMMON_CONFIG = get_config(CONFIG_ABSPATH)
DISCORD_MESSAGE_LIMIT = 2000
DISCORD_PAD_SIZE = 100

#DEFAULT_LOGGER = logging.getLogger('NULL')
#DEFAULT_LOGGER.addHandler(logging.NullHandler())

class ReportingFormats:
    '''a handy enum of reporting formats for every occasion'''
    DEFAULT = '[%(asctime)s;%(levelname)s;%(filename)s;%(funcName)s;%(lineno)s] %(message)s'
    PRETTY_PRINT = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s]\n%(message).1400s'
    STDOUT = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s] %(message)s'

class ProsperLogger(object):
    '''container for building a logger for prosper tools'''
    _debug_mode = False
    def __init__(
            self,
            log_name,
            log_path,
            config_obj=COMMON_CONFIG,
            debug_mode=_debug_mode
    ):
        self.logger = logging.getLogger(log_name)
        self.log_options = p_utils.parse_options(config_obj, 'LOGGING')
        self._debug_mode = debug_mode
        self.log_name = log_name
        self.log_path = test_logpath(log_path, debug_mode)

        self.log_info = []
        self.log_handlers = []
        self.configure_default_logger(debug_mode=debug_mode)

    def get_logger(self):
        '''yield logging object for script to work with'''
        return self.logger

    def __str__(self):
        '''for debug purposes, list attached log handlers'''
        return ','.join(self.log_info)

    def configure_default_logger(
            self,
            log_freq=None,
            log_total=None,
            log_level=None,
            log_format=ReportingFormats().DEFAULT,
            debug_mode=_debug_mode
    ):
        '''build default logger object and handlers'''
        try:
            if not log_freq:
                log_freq = self.log_options['log_freq']
            if not log_total:
                log_total = self.log_options['log_total']
            if not log_level:
                log_level = self.log_options['log_level']
        except KeyError as error_msg:
            raise error_msg

        log_filename = self.log_name + '.log'
        log_abspath = path.join(self.log_path, log_filename)
        general_handler = TimedRotatingFileHandler(
            log_abspath,
            when=log_freq,
            interval=1,
            backupCount=int(log_total) #FIXME: defaults
        )

        formatter = logging.Formatter(log_format)
        general_handler.setFormatter(formatter)

        self.logger.setLevel(log_level)
        self.logger.addHandler(general_handler)

        self.log_info.append('default @ ' + str(log_level))
        self.log_handlers.append(general_handler)

    def configure_debug_logger(
            self,
            log_level='DEBUG',
            log_format=ReportingFormats().STDOUT,
            debug_mode=_debug_mode
    ):
        '''attach debug logging info for debug'''
        formatter = logging.Formatter(log_format)
        debug_handler = logging.StreamHandler()
        debug_handler.setFormatter(formatter)
        debug_handler.setLevel(log_level)

        self.logger.addHandler(debug_handler)
        self.logger.setLevel(log_level)     #override log_level minimum

        self.log_info.append('debug @ ' + str(log_level))
        self.log_handlers.append(debug_handler)

    def configure_discord_logger(
            self,
            log_level='ERROR',
            log_format=ReportingFormats().PRETTY_PRINT,
            discord_webhook=None,
            discord_recipient=None,
            debug_mode=_debug_mode
    ):
        '''attach discord options'''

        ## Make sure we can set discord handlers ##
        if not discord_webhook:
            try:
                discord_webhook = self.log_options['discord_webhook']
            except KeyError:
                warnings.warn(
                    'Lacking discord_webhook defintion, unable to attach webhook',
                    ResourceWarning
                )
                return None
                #raise KeyError('Lacking discord_webhook definition')
            finally:
                if not discord_webhook:
                    #raise KeyError('Lacking discord_webhook definition')
                    warnings.warn(
                        'Lacking discord_webhook defintion, unable to attach webhook',
                        ResourceWarning
                    )
                    return None
        ## Actually build discord logging handler ##
        discord_obj = DiscordWebhook()
        discord_obj.webhook(discord_webhook)
        if discord_obj.can_query:
            try:
                discord_handler = HackyDiscordHandler(
                    discord_obj,
                    discord_recipient
                )
                discord_format = logging.Formatter(log_format)
                discord_handler.setFormatter(discord_format)
                discord_handler.setLevel(log_level)
                self.logger.addHandler(discord_handler)
            except Exception as error_msg:
                raise error_msg

            self.log_info.append('discord @ ' + str(log_level))
            self.log_handlers.append(discord_handler)

def test_logpath(log_path, debug_mode=False):
    '''test if logger has access to given path, and set up directories
    RETURNS: valid log_path, after setting up pathing
        '.' or log_path
    '''
    if debug_mode:
        return '.' #if debug, do not deploy to production paths

    ## Try to create path to log ##
    if not path.exists(log_path):
        try:
            makedirs(log_path, exist_ok=True)
        except PermissionError as err_permission:
            #UNABLE TO CREATE LOG PATH
            warning_msg = \
                'Unable to create logging path.  Defaulting to \'.\'' + \
                'log_path={0}'.format(log_path) + \
                'exception={0}'.format(err_permission)
            warnings.warn(
                warning_msg,
                ResourceWarning
            )
            return '.'
        except Exception as err_msg:
            raise err_msg

    ## Make sure logger can write to path ##
    if not access(log_path, W_OK):
        #UNABLE TO WRITE TO LOG PATH
        warning_msg = \
            'Lacking write permissions to path.  Defaulting to \'.\'' + \
            'log_path={0}'.format(log_path) + \
            'exception={0}'.format(err_permission)
        warnings.warn(
            warning_msg,
            ResourceWarning
        )
        return '.'
        #TODO: windows behavior requires abspath to existing file

    return log_path

def create_logger(
        log_name,
        log_path,
        config_obj = None,
        log_level_override = ''
):
    '''creates logging handle for programs'''
    warnings.warn(
        'create_logger replaced with ProsperLogger object',
        DeprecationWarning
    )
    if not config_obj:
        config_obj = COMMON_CONFIG

    if not path.exists(log_path):
        makedirs(log_path)

    Logger = logging.getLogger(log_name)

    log_level = config_obj.get('LOGGING', 'log_level')
    log_freq  = config_obj.get('LOGGING', 'log_freq')
    log_total = config_obj.get('LOGGING', 'log_total')

    log_name = log_name + '.log'
    log_format = '[%(asctime)s;%(levelname)s;%(filename)s;%(funcName)s;%(lineno)s] %(message)s'
    #print(logName + ':' + logLevel)
    if log_level_override:
        log_level = log_level_override

    log_full_path = path.join(log_path, log_name)

    Logger.setLevel(log_level)
    generalHandler = TimedRotatingFileHandler(
        log_full_path,
        when=log_freq,
        interval=1,
        backupCount=int(log_total)
    )
    formatter = logging.Formatter(log_format)
    generalHandler.setFormatter(formatter)
    Logger.addHandler(generalHandler)

    if log_level_override == 'DEBUG':
        #print('LOGGER: adding stdout handler')
        short_format = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s] %(message)s'
        short_formatter = logging.Formatter(short_format)
        stdout = logging.StreamHandler()
        stdout.setFormatter(short_formatter)
        Logger.addHandler(stdout)

    if all([
            config_obj.has_option('LOGGING', 'discord_webhook'),
            config_obj.has_option('LOGGING', 'discord_level'),
            #config_obj.has_option('LOGGING', 'discord_alert_recipient')
    ]):
        #message_maxlength = DISCORD_MESSAGE_LIMIT - DISCORD_PAD_SIZE
        #print('LOGGER: adding discord handler')
        discord_format = '[%(levelname)s:%(filename)s--%(funcName)s:%(lineno)s]\n%(message).1400s'
        alert_recipient = None
        if config_obj.has_option('LOGGING', 'discord_alert_recipient'):
            alert_recipient = config_obj.get('LOGGING', 'discord_alert_recipient')

        discord_obj = DiscordWebhook()
        discord_obj.webhook(config_obj.get('LOGGING', 'discord_webhook'))
        discord_level = config_obj.get('LOGGING', 'discord_level')
        do_discord = all([discord_obj.can_query, discord_level])
        if do_discord:
            try:
                dh = HackyDiscordHandler(
                    discord_obj,
                    alert_recipient
                )
                dh_format = logging.Formatter(discord_format)
                dh.setFormatter(dh_format)
                dh.setLevel(discord_level)
                Logger.addHandler(dh)
            except Exception as error_msg:
                print('UNABLE TO ATTATCH DISCORD HANDLER - ' + str(error_msg))

    return Logger

class DiscordWebhook(object):
    '''parser for webhooks for easier api calling'''
    __base_url = 'https://discordapp.com/api/webhooks/'
    def __init__(self):
        self.webhook_url = ''
        self.serverid = 0
        self.api_key = ''
        self.can_query = False
        self.webhook_response = None

    def webhook(self, webhook_url):
        '''parse pieces of webhook credentials from url'''
        if webhook_url:
            self.can_query = True
        self.webhook_url = webhook_url
        #FIXME vv this is hacky as fuck
        trunc_url = self.webhook_url.replace(self.__base_url, '')
        id_list = trunc_url.split('/')
        self.serverid = int(id_list[0])
        self.api_key = id_list[1]

    def api_keys(self, serverid, api_key):
        '''with a id/api pair, assemble the webhook_url'''
        if serverid and api_key:
            self.can_query = True
        self.serverid = serverid
        self.api_key = api_key
        self.webhook_url = self.__base_url + self.serverid + '/' + self.api_key

    def get_webhook_info(self):
        '''fetch api profile from discord servers'''
        if not self.can_query:
            raise RuntimeError('webhook information not loaded, cannot query')

        raise NotImplementedError('requests call to discord server for API info: TODO')

    def __bool__(self):
        return self.can_query

    def __str__(self):
        return self.webhook_url

class HackyDiscordHandler(logging.Handler):
    '''hacky in-house discord/REST handler'''
    def __init__(self, webhook_obj, alert_recipient=None):
        logging.Handler.__init__(self)
        self.webhook_obj = webhook_obj
        self.api_url = self.webhook_obj.webhook_url
        self.alert_recipient = alert_recipient
        self.alert_length = 0
        if self.alert_recipient:
            self.alert_length = len(self.alert_recipient)

    def emit(self, record):
        '''logging logic goes here'''
        log_msg = self.format(record)
        if len(log_msg) + self.alert_length > DISCORD_MESSAGE_LIMIT:
            log_msg = log_msg[:(DISCORD_MESSAGE_LIMIT - DISCORD_PAD_SIZE)]

        if self.alert_recipient and record.levelno == logging.CRITICAL:
            log_msg = log_msg + '\n' + str(self.alert_recipient)

        self.send_msg_to_webhook(log_msg)

    def send_msg_to_webhook(self, message):
        '''requests framework for sending log message to webhook'''
        payload = {
            'content':message
        }

        header = {
            'Content-Type':'application/json'
        }

        try:
            request = requests.post(
                self.api_url,
                headers=header,
                json=payload
            )
        except Exception as error_msg:
            print(
                'EXCEPTION: UNABLE TO COMMIT LOG MESSAGE' + \
                '\r\texception={0}'.format(error_msg) + \
                '\r\tmessage={0}'.format(message)
            )
    def test(self, message):
        '''hook for testing webhook logic'''
        try:
            self.send_msg_to_webhook(message)
        except Exception as error_msg:
            raise error_msg
class LoggerLevels:
    '''enums for logger'''
    #FIXME vvv import actual logger enum?
    #https://docs.python.org/3/library/logging.html#logging-levels
    CRITICAL = 50
    ERROR = 40
    WARNING = 30
    INFO = 20
    DEBUG = 10
    NOTSET = 0

    def str_to_level(self, log_level):
        '''converts str to int enum'''
        log_level = log_level.upper()
        if  log_level == 'CRITICAL':
            return self.CRITICAL
        elif log_level == 'ERROR':
            return self.ERROR
        elif log_level == 'WARNING':
            return self.WARNING
        elif log_level == 'INFO':
            return self.INFO
        elif log_level == 'DEBUG':
            return self.DEBUG
        else:
            return self.NOTSET

if __name__ == '__main__':
    TEST_LOGGER = create_logger(
        'common_logger_debug',
        '.',
        COMMON_CONFIG,
        'DEBUG'
    )

    TEST_LOGGER.debug('prosper.common.prosper_logging TEST --DEBUG--')
    TEST_LOGGER.info('prosper.common.prosper_logging TEST --INFO--')
    TEST_LOGGER.warning('prosper.common.prosper_logging TEST --WARNING--')
    TEST_LOGGER.error('prosper.common.prosper_logging TEST --ERROR--')
    TEST_LOGGER.critical('prosper.common.prosper_logging TEST --CRITICAL--')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This file is part of wger Workout Manager.
#
# wger Workout Manager is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# wger Workout Manager is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License

import logging
import csv
import json

from django.template import RequestContext
from django.shortcuts import render_to_response
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.forms import ModelForm
from django.forms import DateField
from django.forms import CharField
from django.forms import Textarea
from django.forms import Form
from django import forms
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.contrib.auth.decorators import login_required
from django.contrib.formtools.preview import FormPreview
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy
from django.db.models import Min
from django.db.models import Max

from django.views.generic import CreateView
from django.views.generic import UpdateView

from wger.weight.models import WeightEntry
from wger.weight import helpers

from wger.utils.generic_views import WgerFormMixin
from wger.utils.constants import DATE_FORMATS

logger = logging.getLogger('wger.custom')

class WeightForm(ModelForm):
    creation_date = DateField(input_formats=DATE_FORMATS)

    class Meta:
        model = WeightEntry

class WeightAddView(WgerFormMixin, CreateView):
    '''
    Generic view to add a new weight entry
    '''
    model = WeightEntry
    form_class = WeightForm
    custom_js = '''$(document).ready(function () {
        init_weight_datepicker();
    });'''
    title = ugettext_lazy('Add weight entry')
    form_action = reverse_lazy('weight-add')
    success_url = reverse_lazy('wger.weight.views.overview')

    def form_valid(self, form):
        '''
        Set the owner of the entry here
        '''

        form.instance.user = self.request.user
        return super(WeightAddView, self).form_valid(form)

class WeightUpdateView(WgerFormMixin, UpdateView):
    '''
    Generic view to edit an existing weight entry
    '''
    model = WeightEntry
    form_class = WeightForm
    custom_js = '''$(document).ready(function () {
        init_weight_datepicker();
    });'''
    success_url = reverse_lazy('wger.weight.views.overview')

    def get_context_data(self, **kwargs):
        context = super(WeightUpdateView, self).get_context_data(**kwargs)
        context['form_action'] = reverse('weight-edit', kwargs={'pk': self.object.id})
        context['title'] = _('Edit weight entry for the %s') % self.object.creation_date

        return context

@login_required
def export_csv(request):
    '''
    Exports the saved weight data as a CSV file
    '''

    # Prepare the response headers
    filename = "{0}-{1}".format(_('weightdata'), request.user.username)
    response = HttpResponse(mimetype='text/csv')
    response['Content-Disposition'] = 'attachment; filename=%s.csv' % filename

    # Convert all weight data to CSV
    writer = csv.writer(response)

    weights = WeightEntry.objects.filter(user=request.user)
    writer.writerow([_('Weight'), _('Date')])

    for entry in weights:
        writer.writerow([entry.weight, entry.creation_date])

    # Send the data to the browser
    return response

@login_required
def overview(request):
    '''
    Shows a plot with the weight data

    More info about the D3 library can be found here:
        * https://github.com/mbostock/d3
        * http://d3js.org/
    '''
    template_data = {}

    min_date = WeightEntry.objects.filter(user=request.user).\
        aggregate(Min('creation_date'))['creation_date__min']
    max_date = WeightEntry.objects.filter(user=request.user).\
        aggregate(Max('creation_date'))['creation_date__max']
    if min_date:
        template_data['min_date'] = 'new Date(%(year)s, %(month)s, %(day)s)' % \
                                    {'year': min_date.year,
                                     'month': min_date.month,
                                     'day': min_date.day}
    if max_date:
        template_data['max_date'] = 'new Date(%(year)s, %(month)s, %(day)s)' % \
                                    {'year': max_date.year,
                                     'month': max_date.month,
                                     'day': max_date.day}
    return render_to_response('weight_overview.html',
                              template_data,
                              context_instance=RequestContext(request))

@login_required
def get_weight_data(request):
    '''
    Process the data to pass it to the JS libraries to generate an SVG image
    '''

    date_min = request.GET.get('date_min', False)
    date_max = request.GET.get('date_max', True)

    if date_min and date_max:
        weights = WeightEntry.objects.filter(user=request.user,
                                             creation_date__range=(date_min, date_max))
    else:
        weights = WeightEntry.objects.filter(user=request.user)

    chart_data = []

    for i in weights:
        chart_data.append({'x': "%(month)s/%(day)s/%(year)s" % {
                           'year': i.creation_date.year,
                           'month': i.creation_date.month,
                           'day': i.creation_date.day},
                           'y': i.weight,
                           'id': i.id})

    # Return the results to the server
    mimetype = 'application/json'
    return HttpResponse(json.dumps(chart_data), mimetype)

CSV_DATE_FORMAT = (('%d.%m.%Y', 'DD.MM.YYYY (30.01.2012)'),
                  ('%d.%m.%y', 'DD.MM.YY (30.01.12)'),
                  ('%Y-%m-%d', 'YYYY-MM-DD (2012-01-30)'),
                  ('%y-%m-%d', 'YY-MM-DD (12-01-30)'),
                  ('%m/%d/%Y', 'MM/DD/YYYY (01/30/2012)'),
                  ('%m/%d/%y', 'MM/DD/YY (01/30/12)'),)

class WeightCsvImportForm(Form):
    '''
    A helper form with only a textarea
    '''
    csv_input = CharField(widget=Textarea, label=_('Input'))
    date_format = forms.ChoiceField(choices=CSV_DATE_FORMAT, label=_('Date format'))

class WeightCsvImportFormPreview(FormPreview):
    preview_template = 'import_csv_preview.html'
    form_template = 'import_csv_form.html'

    def get_context(self, request, form):
        '''
        Context for template rendering.
        '''

        return {'form': form,
                'stage_field': self.unused_name('stage'),
                'state': self.state,
                'form_action': reverse('weight-import-csv')}

    def process_preview(self, request, form, context):
        context['weight_list'], context['error_list'] = helpers.parse_weight_csv(request,
                                                                                form.cleaned_data)
        return context

    def done(self, request, cleaned_data):
        weight_list, error_list = helpers.parse_weight_csv(request, cleaned_data)
        WeightEntry.objects.bulk_create(weight_list)
        return HttpResponseRedirect(reverse('weight-overview'))
<commit_msg>Correctly handle non-ASCII headers for the weight CSV export<commit_after># -*- coding: utf-8 -*-

# This file is part of wger Workout Manager.
#
# wger Workout Manager is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# wger Workout Manager is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License

import logging
import csv
import json

from django.template import RequestContext
from django.shortcuts import render_to_response
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.forms import ModelForm
from django.forms import DateField
from django.forms import CharField
from django.forms import Textarea
from django.forms import Form
from django import forms
from django.core.urlresolvers import reverse
from django.core.urlresolvers import reverse_lazy
from django.contrib.auth.decorators import login_required
from django.contrib.formtools.preview import FormPreview
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy
from django.db.models import Min
from django.db.models import Max

from django.views.generic import CreateView
from django.views.generic import UpdateView

from wger.weight.models import WeightEntry
from wger.weight import helpers

from wger.utils.generic_views import WgerFormMixin
from wger.utils.constants import DATE_FORMATS

logger = logging.getLogger('wger.custom')

class WeightForm(ModelForm):
    creation_date = DateField(input_formats=DATE_FORMATS)

    class Meta:
        model = WeightEntry

class WeightAddView(WgerFormMixin, CreateView):
    '''
    Generic view to add a new weight entry
    '''
    model = WeightEntry
    form_class = WeightForm
    custom_js = '''$(document).ready(function () {
        init_weight_datepicker();
    });'''
    title = ugettext_lazy('Add weight entry')
    form_action = reverse_lazy('weight-add')
    success_url = reverse_lazy('wger.weight.views.overview')

    def form_valid(self, form):
        '''
        Set the owner of the entry here
        '''

        form.instance.user = self.request.user
        return super(WeightAddView, self).form_valid(form)

class WeightUpdateView(WgerFormMixin, UpdateView):
    '''
    Generic view to edit an existing weight entry
    '''
    model = WeightEntry
    form_class = WeightForm
    custom_js = '''$(document).ready(function () {
        init_weight_datepicker();
    });'''
    success_url = reverse_lazy('wger.weight.views.overview')

    def get_context_data(self, **kwargs):
        context = super(WeightUpdateView, self).get_context_data(**kwargs)
        context['form_action'] = reverse('weight-edit', kwargs={'pk': self.object.id})
        context['title'] = _('Edit weight entry for the %s') % self.object.creation_date

        return context

@login_required
def export_csv(request):
    '''
    Exports the saved weight data as a CSV file
    '''

    # Prepare the response headers
    filename = u"{0}-{1}".format('weightdata', request.user.username)
    response = HttpResponse(mimetype='text/csv')
    response['Content-Disposition'] = 'attachment; filename=%s.csv' % filename

    # Convert all weight data to CSV
    writer = csv.writer(response)

    weights = WeightEntry.objects.filter(user=request.user)
    writer.writerow([_('Weight').encode('utf8'), _('Date').encode('utf8')])

    for entry in weights:
        writer.writerow([entry.weight, entry.creation_date])

    # Send the data to the browser
    return response

@login_required
def overview(request):
    '''
    Shows a plot with the weight data

    More info about the D3 library can be found here:
        * https://github.com/mbostock/d3
        * http://d3js.org/
    '''
    template_data = {}

    min_date = WeightEntry.objects.filter(user=request.user).\
        aggregate(Min('creation_date'))['creation_date__min']
    max_date = WeightEntry.objects.filter(user=request.user).\
        aggregate(Max('creation_date'))['creation_date__max']
    if min_date:
        template_data['min_date'] = 'new Date(%(year)s, %(month)s, %(day)s)' % \
                                    {'year': min_date.year,
                                     'month': min_date.month,
                                     'day': min_date.day}
    if max_date:
        template_data['max_date'] = 'new Date(%(year)s, %(month)s, %(day)s)' % \
                                    {'year': max_date.year,
                                     'month': max_date.month,
                                     'day': max_date.day}
    return render_to_response('weight_overview.html',
                              template_data,
                              context_instance=RequestContext(request))

@login_required
def get_weight_data(request):
    '''
    Process the data to pass it to the JS libraries to generate an SVG image
    '''

    date_min = request.GET.get('date_min', False)
    date_max = request.GET.get('date_max', True)

    if date_min and date_max:
        weights = WeightEntry.objects.filter(user=request.user,
                                             creation_date__range=(date_min, date_max))
    else:
        weights = WeightEntry.objects.filter(user=request.user)

    chart_data = []

    for i in weights:
        chart_data.append({'x': "%(month)s/%(day)s/%(year)s" % {
                           'year': i.creation_date.year,
                           'month': i.creation_date.month,
                           'day': i.creation_date.day},
                           'y': i.weight,
                           'id': i.id})

    # Return the results to the server
    mimetype = 'application/json'
    return HttpResponse(json.dumps(chart_data), mimetype)

CSV_DATE_FORMAT = (('%d.%m.%Y', 'DD.MM.YYYY (30.01.2012)'),
                  ('%d.%m.%y', 'DD.MM.YY (30.01.12)'),
                  ('%Y-%m-%d', 'YYYY-MM-DD (2012-01-30)'),
                  ('%y-%m-%d', 'YY-MM-DD (12-01-30)'),
                  ('%m/%d/%Y', 'MM/DD/YYYY (01/30/2012)'),
                  ('%m/%d/%y', 'MM/DD/YY (01/30/12)'),)

class WeightCsvImportForm(Form):
    '''
    A helper form with only a textarea
    '''
    csv_input = CharField(widget=Textarea, label=_('Input'))
    date_format = forms.ChoiceField(choices=CSV_DATE_FORMAT, label=_('Date format'))

class WeightCsvImportFormPreview(FormPreview):
    preview_template = 'import_csv_preview.html'
    form_template = 'import_csv_form.html'

    def get_context(self, request, form):
        '''
        Context for template rendering.
        '''

        return {'form': form,
                'stage_field': self.unused_name('stage'),
                'state': self.state,
                'form_action': reverse('weight-import-csv')}

    def process_preview(self, request, form, context):
        context['weight_list'], context['error_list'] = helpers.parse_weight_csv(request,
                                                                                form.cleaned_data)
        return context

    def done(self, request, cleaned_data):
        weight_list, error_list = helpers.parse_weight_csv(request, cleaned_data)
        WeightEntry.objects.bulk_create(weight_list)
        return HttpResponseRedirect(reverse('weight-overview'))
<|endoftext|>


<commit_before>"""Local workflow executor"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import random
import shlex
import subprocess

from django.conf import settings

from .local import FlowExecutor as LocalFlowExecutor

class FlowExecutor(LocalFlowExecutor):

    def __init__(self, *args, **kwargs):
        super(FlowExecutor, self).__init__(*args, **kwargs)
        self.command = settings.FLOW_EXECUTOR.get('COMMAND', 'docker')

    def start(self):
        container_image = settings.FLOW_EXECUTOR['CONTAINER_IMAGE']

        if self.data_id != 'no_data_id':
            container_name = 'resolwe_{}'.format(self.data_id)
        else:
            # set random container name for tests
            rand_int = random.randint(1000, 9999)
            container_name = 'resolwe_test_{}'.format(rand_int)

        # render Docker mappings in FLOW_DOCKER_MAPPINGS setting
        mappings_template = getattr(settings, 'FLOW_DOCKER_MAPPINGS', [])
        context = {'data_id': self.data_id}
        mappings = [{key.format(**context): value.format(**context) for key, value in template.items()}
                    for template in mappings_template]

        # create mappings for tools
        # NOTE: To prevent processes tampering with tools, all tools are mounted read-only
        self.mappings_tools = [{'src': tool, 'dest': '/usr/local/bin/resolwe/{}'.format(i), 'mode': 'ro'}
                               for i, tool in enumerate(self.get_tools())]
        mappings += self.mappings_tools
        # create Docker --volume parameters from mappings
        volumes = ' '.join(['--volume="{src}":"{dest}":{mode}'.format(**map_) for map_ in mappings])

        # set working directory inside the container to the mapped directory of
        # the current Data's directory
        workdir = ''
        for template in mappings_template:
            if '{data_id}' in template['src']:
                workdir = '--workdir={}'.format(template['dest'])

        # create environment variables to pass certain information to the
        # process running in the container
        envs = ' '.join(['--env={name}={value}'.format(**env) for env in [
            {'name': 'HOST_UID', 'value': os.getuid()},
            {'name': 'HOST_GID', 'value': os.getgid()},
        ]])

        # a login Bash shell is needed to source ~/.bash_profile
        self.proc = subprocess.Popen(
            shlex.split(
                '{} run --rm --interactive --name={} {} {} {} {} /bin/bash --login'.format(
                    self.command, container_name, volumes, envs, workdir, container_image)),
            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)

        self.stdout = self.proc.stdout

    def run_script(self, script):
        mappings = getattr(settings, 'FLOW_DOCKER_MAPPINGS', {})
        for map_ in mappings:
            script = script.replace(map_['src'], map_['dest'])
        # create a Bash command to add all the tools to PATH
        tools_paths = ':'.join([map_["dest"] for map_ in self.mappings_tools])
        add_tools_path = 'export PATH=$PATH:{}'.format(tools_paths)
        self.proc.stdin.write(os.linesep.join(['set -x', 'set +B', add_tools_path, script]) + os.linesep)
        self.proc.stdin.close()

    def end(self):
        self.proc.wait()

        return self.proc.returncode

    def terminate(self, data_id):
        subprocess.call(shlex.split('{} rm -f {}'.format(self.command, str(data_id))))
<commit_msg>Refactor resolwe.flow.executors.docker.FlowExecutor.start()<commit_after>"""Local workflow executor"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import random
import shlex
import subprocess

from django.conf import settings

from .local import FlowExecutor as LocalFlowExecutor

class FlowExecutor(LocalFlowExecutor):

    def __init__(self, *args, **kwargs):
        super(FlowExecutor, self).__init__(*args, **kwargs)
        self.command = settings.FLOW_EXECUTOR.get('COMMAND', 'docker')

    def start(self):
        # arguments passed to the Docker command
        command_args = {
            'command': self.command,
            'container_image': settings.FLOW_EXECUTOR['CONTAINER_IMAGE'],
        }

        if self.data_id != 'no_data_id':
            data_id = self.data_id
        else:
            # set random container name for tests
            data_id = 'test_{}'.format(random.randint(1000, 9999))
        command_args['container_name'] = '--name=resolwe_{}'.format(data_id)

        # render Docker mappings in FLOW_DOCKER_MAPPINGS setting
        mappings_template = getattr(settings, 'FLOW_DOCKER_MAPPINGS', [])
        context = {'data_id': self.data_id}
        mappings = [{key.format(**context): value.format(**context) for key, value in template.items()}
                    for template in mappings_template]

        # create mappings for tools
        # NOTE: To prevent processes tampering with tools, all tools are mounted read-only
        self.mappings_tools = [{'src': tool, 'dest': '/usr/local/bin/resolwe/{}'.format(i), 'mode': 'ro'}
                               for i, tool in enumerate(self.get_tools())]
        mappings += self.mappings_tools
        # create Docker --volume parameters from mappings
        command_args['volumes'] = ' '.join(['--volume="{src}":"{dest}":{mode}'.format(**map_)
                                            for map_ in mappings])

        # set working directory inside the container to the mapped directory of
        # the current Data's directory
        command_args['workdir'] = ''
        for template in mappings_template:
            if '{data_id}' in template['src']:
                command_args['workdir'] = '--workdir={}'.format(template['dest'])

        # create environment variables to pass certain information to the
        # process running in the container
        command_args['envs'] = ' '.join(['--env={name}={value}'.format(**env) for env in [
            {'name': 'HOST_UID', 'value': os.getuid()},
            {'name': 'HOST_GID', 'value': os.getgid()},
        ]])

        # a login Bash shell is needed to source ~/.bash_profile
        command_args['shell'] = '/bin/bash --login'

        self.proc = subprocess.Popen(shlex.split(
            '{command} run --rm --interactive {container_name} {volumes} '
            '{envs} {workdir} {container_image} {shell}'.format(**command_args)
        ), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        universal_newlines=True)

        self.stdout = self.proc.stdout

    def run_script(self, script):
        mappings = getattr(settings, 'FLOW_DOCKER_MAPPINGS', {})
        for map_ in mappings:
            script = script.replace(map_['src'], map_['dest'])
        # create a Bash command to add all the tools to PATH
        tools_paths = ':'.join([map_["dest"] for map_ in self.mappings_tools])
        add_tools_path = 'export PATH=$PATH:{}'.format(tools_paths)
        self.proc.stdin.write(os.linesep.join(['set -x', 'set +B', add_tools_path, script]) + os.linesep)
        self.proc.stdin.close()

    def end(self):
        self.proc.wait()

        return self.proc.returncode

    def terminate(self, data_id):
        subprocess.call(shlex.split('{} rm -f {}'.format(self.command, str(data_id))))
<|endoftext|>


<commit_before>import time, subprocess, os, argparse
import math
path = os.path.dirname(os.path.realpath(__file__))

#Creates the name for the file
def name_generator():
    filename='osmaxx_excerpt_'+time.strftime("%Y-%m-%d")+'_'+time.strftime("%H%M%S")
    return filename

#Converts the OSM map projection to mercator projection
def to_mercator(mercator_x_lon, mercator_y_lat):
    if math.fabs(mercator_x_lon) > 180 or math.fabs(mercator_y_lat) > 90:
        return
    num = mercator_x_lon * 0.017453292519943295
    x_value = 6378137.0 * num
    additional = mercator_y_lat * 0.017453292519943295
    mercator_x_lon = x_value
    mercator_y_lat = 3189068.5 * \
                     math.log((1.0 + math.sin(additional)) /
                              (1.0 - math.sin(additional)))
    return [mercator_x_lon, mercator_y_lat]

#Extract Statistics
def get_statistics(data, name):
    statcmd = 'bash', './extract/extract_statistics', data[0], data[1], data[2], data[3], name
    subprocess.check_call(statcmd)

#Calls the shell script that exports siles of the specified format(file_format) from existing database
def export_format(data, name, file_format):
    get_statistics(data, name)
    dbcmd = 'sh', './extract/extract_format.sh', data[0], data[1], data[2], data[3], name, file_format
    subprocess.check_call(dbcmd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-f','--format',choices=['fgdb','shp','gpkg','spatialite'], default='fgdb', help= 'output formats')
    parser.add_argument('xmin', help='Min Longitude/Left/West', type=float)
    parser.add_argument('ymin', help='Min Latitude/Bottom/South', type=float)
    parser.add_argument('xmax', help='Max Longitude/Right/East', type=float)
    parser.add_argument('ymax', help='Max Latitude/Top/North', type=float)

    args = parser.parse_args()
    min_xy=to_mercator(args.xmin, args.ymin)
    max_xy=to_mercator(args.xmax, args.ymax)
    name=name_generator()

    export_format([str(min_xy[0]), str(min_xy[1]), str(max_xy[0]), str(max_xy[1])], name, args.format)
    print name+' have been completed in '+args.format+' format.'
<commit_msg>fix typo in comment<commit_after>import time, subprocess, os, argparse
import math
path = os.path.dirname(os.path.realpath(__file__))

#Creates the name for the file
def name_generator():
    filename='osmaxx_excerpt_'+time.strftime("%Y-%m-%d")+'_'+time.strftime("%H%M%S")
    return filename

#Converts the OSM map projection to mercator projection
def to_mercator(mercator_x_lon, mercator_y_lat):
    if math.fabs(mercator_x_lon) > 180 or math.fabs(mercator_y_lat) > 90:
        return
    num = mercator_x_lon * 0.017453292519943295
    x_value = 6378137.0 * num
    additional = mercator_y_lat * 0.017453292519943295
    mercator_x_lon = x_value
    mercator_y_lat = 3189068.5 * \
                     math.log((1.0 + math.sin(additional)) /
                              (1.0 - math.sin(additional)))
    return [mercator_x_lon, mercator_y_lat]

#Extract Statistics
def get_statistics(data, name):
    statcmd = 'bash', './extract/extract_statistics', data[0], data[1], data[2], data[3], name
    subprocess.check_call(statcmd)

#Calls the shell script that exports files of the specified format(file_format) from existing database
def export_format(data, name, file_format):
    get_statistics(data, name)
    dbcmd = 'sh', './extract/extract_format.sh', data[0], data[1], data[2], data[3], name, file_format
    subprocess.check_call(dbcmd)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-f','--format',choices=['fgdb','shp','gpkg','spatialite'], default='fgdb', help= 'output formats')
    parser.add_argument('xmin', help='Min Longitude/Left/West', type=float)
    parser.add_argument('ymin', help='Min Latitude/Bottom/South', type=float)
    parser.add_argument('xmax', help='Max Longitude/Right/East', type=float)
    parser.add_argument('ymax', help='Max Latitude/Top/North', type=float)

    args = parser.parse_args()
    min_xy=to_mercator(args.xmin, args.ymin)
    max_xy=to_mercator(args.xmax, args.ymax)
    name=name_generator()

    export_format([str(min_xy[0]), str(min_xy[1]), str(max_xy[0]), str(max_xy[1])], name, args.format)
    print name+' have been completed in '+args.format+' format.'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import numpy as np
import csv

###########
### I/O ###
###########

def find_path(file_name, directory="data", file_type=".csv"):
    import os
    cwd = os.getcwd()
    if not os.path.exists(directory):
        os.makedirs(directory)
    d = os.path.join(cwd, os.path.join(directory,file_name+file_type))
    return d

def create_file(file_name):
    import os
    d = find_path(file_name)
    if not os.path.isfile(d):
        open(d, 'w').close()
    return d

def file_exists(file_name):
    import os
    d = find_path(file_name)
    return os.path.isfile(d)

def load_csv(file_name, delimiter=';', comment=None):
    d = find_path(file_name)
    pass

def write_columns_csv(lst, file_name, header=[], index=None, start_char=None, delimiter=';', open_as='w'):
    """
    write_columns_csv outputs tree data to an NEW (not existing) csv file

    lst       : a list of a list containing data for a single tree
    file_name :
    headers   : names of the trees; these are put in the first row of the csv file.
    index     : index data (e.g., Year and Node)
                  - NB: Header should have the index names as the first element(s)

    """
    d = find_path(file_name)
    if file_name.find('tree') >0:
        print('***in write_column_csv, file_name =',file_name,' header =',header,'index=',index)
        print('***lst = ',lst)
    if index is not None:
        index.extend(lst)
        output_lst = list(zip(*index))
    else:
        output_lst = list(zip(*lst))
    if file_name.find('tree') >0:
        print()
        print('***in write_column_csv,output_lst=',output_lst)
        print()

    with open(d, open_as) as f:
        writer = csv.writer(f, delimiter=delimiter)
        if start_char is not None:
            writer.writerow([start_char])
        if header:
            writer.writerow(header)
        for row in output_lst:
            if file_name.find('tree') >0:
                print('    ***write_columns_csv -- type:',type(row),', row=', row)
            writer.writerow(row)
        if file_name.find('tree') >0:
            print('***DONE -- rite_columns_csv')
            print()
    if file_name.find('tree') >0:
        x = input('WCC-- Please halt the program here and examine the _trees file in the data folder')

def write_columns_to_existing(lst, file_name, header="", delimiter=';'):
    """
    writes the tree elements in lst to and EXISTING file with name file_name.
    """
    # d = find_path(file_name)
    # if file_name.find('tree') >0:
    #     print('WCTE -- input lst of length',len(lst),' is ', lst)
    # all_lst = []

    # print(d)

    # with open(d, 'r') as finput:
        # reader = csv.reader(finput, delimiter=delimiter)
        # row = next(reader)

    is_nested_list = lst and (isinstance(lst[0], list) or
                                isinstance(lst[0], np.ndarray))
    if is_nested_list:
        lst = list(zip(*lst))   # transpose columns -> rows

    file_path = find_path(file_name)
    output_rows = []

    # read and extend input
    with open(file_path, 'r') as finput:
        reader = csv.reader(finput, delimiter=delimiter)

        # extend header row
        row = next(reader)
        row.extend(header if is_nested_list else [header])
        output_rows.append(row)

        # extend rest of the rows
        for i,row in enumerate(reader):
            row.extend(lst[i] if is_nested_list else [lst[i]])
            output_rows.append(row)

    # emit output, overwriting original file
    with open(file_path, 'w') as foutput:
        writer = csv.writer(foutput, delimiter=delimiter)
        writer.writerows(output_rows)

        # n = len(lst)
        # i = 0
        # for row in reader:
            # rw = row.copy()
            # print('***In WCTE after read, rw[',i,'] = ', rw)
            # print('***In WCTE after read, rw[] = ', row)
            # if i==0:
            #     nested_list = isinstance(lst[0], list) or isinstance(lst[0], np.ndarray)
            #     if nested_list:
            #         lst = list(zip(*lst))
            #         rw.extend(header)
            #     else:
            #         rw.append(header)
            # else:
            #     if nested_list:
            #         rw.extend(lst[i-1])
            #     else:
            #         try:
            #             item = lst[i-1]
            #         except IndexError:
            #             print(' *** Index Error (1) in WCTE; i=',i-1)
            #             print(' *** Index Error (1) in WCTE; len(lst)=',len(lst))
            #             print(' *** Index Error (1) in WCTE; rw=', rw)
            #         try:
            #             rw.append(item)
            #         except IndexError:
            #             print(' *** Index Error (2) in WCTE; i=',i)

            # print('***In WCTE after write, rw[',i,'] = ', row)
            # print('***IN WCTE, len(lst),i,lst[i]',len(lst),i,lst[i])
            # all_lst.append(row)
            # i += 1

    # with open(d, 'w') as foutput:
    #     writer = csv.writer(foutput, delimiter=delimiter)
    #     writer.writerows(all_lst)
    # if file_name.find('tree') >0:
    #     x = input('WCTE - Please halt the program here and examine the _trees file in the data folder')

def append_to_existing(lst, file_name, header="", index=None, delimiter=';', start_char=None):
    write_columns_csv(lst, file_name, header, index, start_char=start_char, delimiter=delimiter, open_as='a')

def import_csv(file_name, delimiter=';', header=True, indices=None, start_at=0, break_at='\n', ignore=""):
    d = find_path(file_name)
    input_lst = []
    indices_lst = []
    with open(d, 'r') as f:
        reader = csv.reader(f, delimiter=delimiter)
        for _ in range(0, start_at):
            next(reader)
        if header:
            header_row = next(reader)
        for row in reader:
            if row[0] == break_at:
                break
            if row[0] == ignore:
                continue
            if indices:
                input_lst.append(row[indices:])
                indices_lst.append(row[:indices])
            else:
                input_lst.append(row)
    if header and not indices :
        return header_row, np.array(input_lst, dtype="float64")
    elif header and indices:
        return header_row[indices:], indices_lst, np.array(input_lst, dtype="float64")
    return np.array(input_lst, dtype="float64")

##########
### MP ###
##########

def _pickle_method(method):
    func_name = method.__func__.__name__
    obj = method.__self__
    cls = method.__self__.__class__
    if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names
        cls_name = cls.__name__.lstrip('_')
        func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)

def _unpickle_method(func_name, obj, cls):
    for cls in cls.__mro__:
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls)

<commit_msg>Insert/fixed clean_lines<commit_after># -*- coding: utf-8 -*-
import numpy as np
import csv
import io

###########
### I/O ###
###########

def find_path(file_name, directory="data", file_type=".csv"):
    import os
    cwd = os.getcwd()
    if not os.path.exists(directory):
        os.makedirs(directory)
    d = os.path.join(cwd, os.path.join(directory,file_name+file_type))
    return d

def create_file(file_name):
    import os
    d = find_path(file_name)
    if not os.path.isfile(d):
        open(d, 'w').close()
    return d

def file_exists(file_name):
    import os
    d = find_path(file_name)
    return os.path.isfile(d)

def load_csv(file_name, delimiter=';', comment=None):
    d = find_path(file_name)
    pass

import io

def clean_lines(f):
    """
    Filter out blank lines to avoid prior cross-platform line termination problems.
    """
    lines = f.read().splitlines()
    lines = [line for line in lines if line.strip()]
    content = '\n'.join(lines)
    sio = io.StringIO()
    sio.write(content)
    sio.seek(0)
    return sio

def write_columns_csv(lst, file_name, header=[], index=None, start_char=None, delimiter=';', open_as='w'):
    """
    write_columns_csv outputs tree data to an NEW (not existing) csv file

    lst       : a list of a list containing data for a single tree
    file_name :
    headers   : names of the trees; these are put in the first row of the csv file.
    index     : index data (e.g., Year and Node)
                  - NB: Header should have the index names as the first element(s)

    """
    d = find_path(file_name)
    if file_name.find('tree') >0:
        print('***in write_column_csv, file_name =',file_name,' header =',header,'index=',index)
        print('***lst = ',lst)
    if index is not None:
        index.extend(lst)
        output_lst = list(zip(*index))
    else:
        output_lst = list(zip(*lst))
    if file_name.find('tree') >0:
        print()
        print('***in write_column_csv,output_lst=',output_lst)
        print()

    with open(d, open_as) as f:
        writer = csv.writer(f, delimiter=delimiter)
        if start_char is not None:
            writer.writerow([start_char])
        if header:
            writer.writerow(header)
        for row in output_lst:
            if file_name.find('tree') >0:
                print('    ***write_columns_csv -- type:',type(row),', row=', row)
            writer.writerow(row)
        if file_name.find('tree') >0:
            print('***DONE -- rite_columns_csv')
            print()
    if file_name.find('tree') >0:
        x = input('WCC-- Please halt the program here and examine the _trees file in the data folder')

def clean_lines(f):
    """
    Filter out blank lines in the given file in order to avoid
    cross-platform line termination problems that
    previously led to data files with blank lines.
    """
    lines = f.read().splitlines()
    lines = [line for line in lines if line.strip()]
    content = '\n'.join(lines)
    sio = io.StringIO()
    sio.write(content)
    sio.seek(0)
    return sio

def write_columns_to_existing(lst, file_name, header="", delimiter=';'):
    """
    writes the tree elements in lst to and EXISTING file with name file_name.
    """
    # d = find_path(file_name)
    # if file_name.find('tree') >0:
    #     print('WCTE -- input lst of length',len(lst),' is ', lst)
    # all_lst = []

    # print(d)

    # with open(d, 'r') as finput:
        # reader = csv.reader(clean_lines(finput), delimiter=delimiter)
        # row = next(reader)

    is_nested_list = lst and (isinstance(lst[0], list) or
                                isinstance(lst[0], np.ndarray))
    if is_nested_list:
        lst = list(zip(*lst))   # transpose columns -> rows

    file_path = find_path(file_name)
    output_rows = []

    # read and extend input
    with open(file_path, 'r') as finput:
        reader = csv.reader(clean_lines(finput), delimiter=delimiter)

        # extend header row
        row = next(reader)
        row.extend(header if is_nested_list else [header])
        output_rows.append(row)

        # extend rest of the rows
        for i,row in enumerate(reader):
            row.extend(lst[i] if is_nested_list else [lst[i]])
            output_rows.append(row)

    # emit output, overwriting original file
    with open(file_path, 'w') as foutput:
        writer = csv.writer(foutput, delimiter=delimiter)
        writer.writerows(output_rows)

        # n = len(lst)
        # i = 0
        # for row in reader:
            # rw = row.copy()
            # print('***In WCTE after read, rw[',i,'] = ', rw)
            # print('***In WCTE after read, rw[] = ', row)
            # if i==0:
            #     nested_list = isinstance(lst[0], list) or isinstance(lst[0], np.ndarray)
            #     if nested_list:
            #         lst = list(zip(*lst))
            #         rw.extend(header)
            #     else:
            #         rw.append(header)
            # else:
            #     if nested_list:
            #         rw.extend(lst[i-1])
            #     else:
            #         try:
            #             item = lst[i-1]
            #         except IndexError:
            #             print(' *** Index Error (1) in WCTE; i=',i-1)
            #             print(' *** Index Error (1) in WCTE; len(lst)=',len(lst))
            #             print(' *** Index Error (1) in WCTE; rw=', rw)
            #         try:
            #             rw.append(item)
            #         except IndexError:
            #             print(' *** Index Error (2) in WCTE; i=',i)

            # print('***In WCTE after write, rw[',i,'] = ', row)
            # print('***IN WCTE, len(lst),i,lst[i]',len(lst),i,lst[i])
            # all_lst.append(row)
            # i += 1

    # with open(d, 'w') as foutput:
    #     writer = csv.writer(foutput, delimiter=delimiter)
    #     writer.writerows(all_lst)
    # if file_name.find('tree') >0:
    #     x = input('WCTE - Please halt the program here and examine the _trees file in the data folder')

def append_to_existing(lst, file_name, header="", index=None, delimiter=';', start_char=None):
    write_columns_csv(lst, file_name, header, index, start_char=start_char, delimiter=delimiter, open_as='a')

def import_csv(file_name, delimiter=';', header=True, indices=None, start_at=0, break_at='\n', ignore=""):
    d = find_path(file_name)
    input_lst = []
    indices_lst = []
    with open(d, 'r') as f:
        reader = csv.reader(clean_lines(f), delimiter=delimiter)
        for _ in range(0, start_at):
            next(reader)
        if header:
            header_row = next(reader)
        for row in reader:
            if row[0] == break_at:
                break
            if row[0] == ignore:
                continue
            if indices:
                input_lst.append(row[indices:])
                indices_lst.append(row[:indices])
            else:
                input_lst.append(row)
    if header and not indices :
        return header_row, np.array(input_lst, dtype="float64")
    elif header and indices:
        return header_row[indices:], indices_lst, np.array(input_lst, dtype="float64")
    return np.array(input_lst, dtype="float64")

##########
### MP ###
##########

def _pickle_method(method):
    func_name = method.__func__.__name__
    obj = method.__self__
    cls = method.__self__.__class__
    if func_name.startswith('__') and not func_name.endswith('__'): #deal with mangled names
        cls_name = cls.__name__.lstrip('_')
        func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)

def _unpickle_method(func_name, obj, cls):
    for cls in cls.__mro__:
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls)

<|endoftext|>