<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, Matt Wright <matt@nobien.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import tempfile
import os.path

DOCUMENTATION = '''
---
module: easy_install
short_description: Installs Python libraries
description:
     - Installs Python libraries, optionally in a I(virtualenv)
version_added: "0.7"
options:
  name:
    description:
      - A Python library name
    required: true
    default: null
    aliases: []
  virtualenv:
    description:
      - an optional I(virtualenv) directory path to install into. If the
        I(virtualenv) does not exist, it is created automatically
    required: false
    default: null
  virtualenv_site_packages:
    version_added: "1.1"
    description:
      - Whether the virtual environment will inherit packages from the
        global site-packages directory.  Note that if this setting is
        changed on an already existing virtual environment it will not
        have any effect, the environment must be deleted and newly
        created.
    required: false
    default: "no"
    choices: [ "yes", "no" ]
  virtualenv_command:
    version_added: "1.1"
    description:
      - The command to create the virtual environment with. For example
        C(pyvenv), C(virtualenv), C(virtualenv2).
    required: false
    default: virtualenv
  executable:
    description:
      - The explicit executable or a pathname to the executable to be used to
        run easy_install for a specific version of Python installed in the
        system. For example C(easy_install-3.3), if there are both Python 2.7
        and 3.3 installations in the system and you want to run easy_install
        for the Python 3.3 installation.
    version_added: "1.3"
    required: false
    default: null
  state:
    description:
      - The desired state of the library. C(latest) ensures that the latest version is installed.
    required: false
    choices: [present, latest]
    default: present
notes:
    - Please note that the M(easy_install) module can only install Python
      libraries. Thus this module is not able to remove libraries. It is
      generally recommended to use the M(pip) module which you can first install
      using M(easy_install).
    - Also note that I(virtualenv) must be installed on the remote host if the
      C(virtualenv) parameter is specified.
requirements: [ "virtualenv" ]
author: Matt Wright
'''

EXAMPLES = '''
# Examples from Ansible Playbooks
- easy_install: name=pip state=latest

# Install Bottle into the specified virtualenv.
- easy_install: name=bottle virtualenv=/webapps/myapp/venv
'''

def _is_package_installed(module, name, easy_install, executable_arguments):
    executable_arguments = executable_arguments + ['--dry-run']
    cmd = '%s %s %s' % (easy_install, ' '.join(executable_arguments), name)
    rc, status_stdout, status_stderr = module.run_command(cmd)
    return not ('Reading' in status_stdout or 'Downloading' in status_stdout)

def _get_easy_install(module, env=None, executable=None):
    candidate_easy_inst_basenames = ['easy_install']
    easy_install = None
    if executable is not None:
        if os.path.isabs(executable):
            easy_install = executable
        else:
            candidate_easy_inst_basenames.insert(0, executable)
    if easy_install is None:
        if env is None:
            opt_dirs = []
        else:
            # Try easy_install with the virtualenv directory first.
            opt_dirs = ['%s/bin' % env]
        for basename in candidate_easy_inst_basenames:
            easy_install = module.get_bin_path(basename, False, opt_dirs)
            if easy_install is not None:
                break
    # easy_install should have been found by now.  The final call to
    # get_bin_path will trigger fail_json.
    if easy_install is None:
        basename = candidate_easy_inst_basenames[0]
        easy_install = module.get_bin_path(basename, True, opt_dirs)
    return easy_install

def main():
    arg_spec = dict(
        name=dict(required=True),
        state=dict(required=False,
                   default='present',
                   choices=['present','latest'],
                   type='str'),
        virtualenv=dict(default=None, required=False),
        virtualenv_site_packages=dict(default='no', type='bool'),
        virtualenv_command=dict(default='virtualenv', required=False),
        executable=dict(default='easy_install', required=False),
    )

    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=True)

    name = module.params['name']
    env = module.params['virtualenv']
    executable = module.params['executable']
    site_packages = module.params['virtualenv_site_packages']
    virtualenv_command = module.params['virtualenv_command']
    executable_arguments = []
    if module.params['state'] == 'latest':
        executable_arguments.append('--upgrade')

    rc = 0
    err = ''
    out = ''

    if env:
        virtualenv = module.get_bin_path(virtualenv_command, True)

        if not os.path.exists(os.path.join(env, 'bin', 'activate')):
            if module.check_mode:
                module.exit_json(changed=True)
            command = '%s %s' % (virtualenv, env)
            if site_packages:
                command += ' --system-site-packages'
            cwd = tempfile.gettempdir()
            rc_venv, out_venv, err_venv = module.run_command(command, cwd=cwd)

            rc += rc_venv
            out += out_venv
            err += err_venv

    easy_install = _get_easy_install(module, env, executable)

    cmd = None
    changed = False
    installed = _is_package_installed(module, name, easy_install)

    if not installed:
        if module.check_mode:
            module.exit_json(changed=True)
        cmd = '%s %s %s' % (easy_install, ' '.join(executable_arguments), name)
        rc_easy_inst, out_easy_inst, err_easy_inst = module.run_command(cmd)

        rc += rc_easy_inst
        out += out_easy_inst
        err += err_easy_inst

        changed = True

    if rc != 0:
        module.fail_json(msg=err, cmd=cmd)

    module.exit_json(changed=changed, binary=easy_install,
                     name=name, virtualenv=env)

# import module snippets
from ansible.module_utils.basic import *

main()
<commit_msg>Add 'version_added' to state documentation for easy_install resource<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, Matt Wright <matt@nobien.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

import tempfile
import os.path

DOCUMENTATION = '''
---
module: easy_install
short_description: Installs Python libraries
description:
     - Installs Python libraries, optionally in a I(virtualenv)
version_added: "0.7"
options:
  name:
    description:
      - A Python library name
    required: true
    default: null
    aliases: []
  virtualenv:
    description:
      - an optional I(virtualenv) directory path to install into. If the
        I(virtualenv) does not exist, it is created automatically
    required: false
    default: null
  virtualenv_site_packages:
    version_added: "1.1"
    description:
      - Whether the virtual environment will inherit packages from the
        global site-packages directory.  Note that if this setting is
        changed on an already existing virtual environment it will not
        have any effect, the environment must be deleted and newly
        created.
    required: false
    default: "no"
    choices: [ "yes", "no" ]
  virtualenv_command:
    version_added: "1.1"
    description:
      - The command to create the virtual environment with. For example
        C(pyvenv), C(virtualenv), C(virtualenv2).
    required: false
    default: virtualenv
  executable:
    description:
      - The explicit executable or a pathname to the executable to be used to
        run easy_install for a specific version of Python installed in the
        system. For example C(easy_install-3.3), if there are both Python 2.7
        and 3.3 installations in the system and you want to run easy_install
        for the Python 3.3 installation.
    version_added: "1.3"
    required: false
    default: null
  state:
    version_added: "2.0"
    description:
      - The desired state of the library. C(latest) ensures that the latest version is installed.
    required: false
    choices: [present, latest]
    default: present
notes:
    - Please note that the M(easy_install) module can only install Python
      libraries. Thus this module is not able to remove libraries. It is
      generally recommended to use the M(pip) module which you can first install
      using M(easy_install).
    - Also note that I(virtualenv) must be installed on the remote host if the
      C(virtualenv) parameter is specified.
requirements: [ "virtualenv" ]
author: Matt Wright
'''

EXAMPLES = '''
# Examples from Ansible Playbooks
- easy_install: name=pip state=latest

# Install Bottle into the specified virtualenv.
- easy_install: name=bottle virtualenv=/webapps/myapp/venv
'''

def _is_package_installed(module, name, easy_install, executable_arguments):
    executable_arguments = executable_arguments + ['--dry-run']
    cmd = '%s %s %s' % (easy_install, ' '.join(executable_arguments), name)
    rc, status_stdout, status_stderr = module.run_command(cmd)
    return not ('Reading' in status_stdout or 'Downloading' in status_stdout)

def _get_easy_install(module, env=None, executable=None):
    candidate_easy_inst_basenames = ['easy_install']
    easy_install = None
    if executable is not None:
        if os.path.isabs(executable):
            easy_install = executable
        else:
            candidate_easy_inst_basenames.insert(0, executable)
    if easy_install is None:
        if env is None:
            opt_dirs = []
        else:
            # Try easy_install with the virtualenv directory first.
            opt_dirs = ['%s/bin' % env]
        for basename in candidate_easy_inst_basenames:
            easy_install = module.get_bin_path(basename, False, opt_dirs)
            if easy_install is not None:
                break
    # easy_install should have been found by now.  The final call to
    # get_bin_path will trigger fail_json.
    if easy_install is None:
        basename = candidate_easy_inst_basenames[0]
        easy_install = module.get_bin_path(basename, True, opt_dirs)
    return easy_install

def main():
    arg_spec = dict(
        name=dict(required=True),
        state=dict(required=False,
                   default='present',
                   choices=['present','latest'],
                   type='str'),
        virtualenv=dict(default=None, required=False),
        virtualenv_site_packages=dict(default='no', type='bool'),
        virtualenv_command=dict(default='virtualenv', required=False),
        executable=dict(default='easy_install', required=False),
    )

    module = AnsibleModule(argument_spec=arg_spec, supports_check_mode=True)

    name = module.params['name']
    env = module.params['virtualenv']
    executable = module.params['executable']
    site_packages = module.params['virtualenv_site_packages']
    virtualenv_command = module.params['virtualenv_command']
    executable_arguments = []
    if module.params['state'] == 'latest':
        executable_arguments.append('--upgrade')

    rc = 0
    err = ''
    out = ''

    if env:
        virtualenv = module.get_bin_path(virtualenv_command, True)

        if not os.path.exists(os.path.join(env, 'bin', 'activate')):
            if module.check_mode:
                module.exit_json(changed=True)
            command = '%s %s' % (virtualenv, env)
            if site_packages:
                command += ' --system-site-packages'
            cwd = tempfile.gettempdir()
            rc_venv, out_venv, err_venv = module.run_command(command, cwd=cwd)

            rc += rc_venv
            out += out_venv
            err += err_venv

    easy_install = _get_easy_install(module, env, executable)

    cmd = None
    changed = False
    installed = _is_package_installed(module, name, easy_install)

    if not installed:
        if module.check_mode:
            module.exit_json(changed=True)
        cmd = '%s %s %s' % (easy_install, ' '.join(executable_arguments), name)
        rc_easy_inst, out_easy_inst, err_easy_inst = module.run_command(cmd)

        rc += rc_easy_inst
        out += out_easy_inst
        err += err_easy_inst

        changed = True

    if rc != 0:
        module.fail_json(msg=err, cmd=cmd)

    module.exit_json(changed=changed, binary=easy_install,
                     name=name, virtualenv=env)

# import module snippets
from ansible.module_utils.basic import *

main()
<|endoftext|>


<commit_before>"""An AdaNet estimator implementation which can run on TPU.

Copyright 2018 The AdaNet Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from adanet.core.ensemble import MixtureWeightType
from adanet.core.estimator import Estimator
from adanet.core.summary import _ScopedSummary
import six
import tensorflow as tf
from tensorflow.contrib.tpu.python.tpu import tpu_function
from tensorflow.python import summary

# TODO: support summaries on TPU during training.
def _rewire_summaries():
  """Rewire Tensorflow summaries to be no-ops when running on TPU.

  Summaries are not currently supported on TPU.
  """

  def tpu_no_op(fn):

    def _fn(*args, **kwargs):
      if tpu_function.get_tpu_context().number_of_shards:
        return None
      return fn(*args, **kwargs)

    return _fn

  summary.audio = tpu_no_op(summary.audio)
  summary.histogram = tpu_no_op(summary.histogram)
  summary.image = tpu_no_op(summary.image)
  summary.scalar = tpu_no_op(summary.scalar)
  summary.tensor_summary = tpu_no_op(summary.tensor_summary)
  summary.text = tpu_no_op(summary.text)

  tf.summary.audio = tpu_no_op(tf.summary.audio)
  tf.summary.histogram = tpu_no_op(tf.summary.histogram)
  tf.summary.image = tpu_no_op(tf.summary.image)
  tf.summary.scalar = tpu_no_op(tf.summary.scalar)
  tf.summary.tensor_summary = tpu_no_op(tf.summary.tensor_summary)
  tf.summary.text = tpu_no_op(tf.summary.text)

  _ScopedSummary.audio = tpu_no_op(_ScopedSummary.audio)
  _ScopedSummary.histogram = tpu_no_op(_ScopedSummary.histogram)
  _ScopedSummary.image = tpu_no_op(_ScopedSummary.image)
  _ScopedSummary.scalar = tpu_no_op(_ScopedSummary.scalar)

# Rewire summaries to be no-ops when running on TPU.
_rewire_summaries()

class TPUEstimator(Estimator, tf.contrib.tpu.TPUEstimator):
  """An adanet.Estimator capable of running on TPU.

  If running on TPU, all summary calls are rewired to be no-ops during training.

  WARNING: this API is highly experimental, unstable, and can change  without
  warning.
  """

  def __init__(self,
               head,
               subnetwork_generator,
               max_iteration_steps,
               mixture_weight_type=MixtureWeightType.SCALAR,
               mixture_weight_initializer=None,
               warm_start_mixture_weights=False,
               adanet_lambda=0.,
               adanet_beta=0.,
               evaluator=None,
               report_materializer=None,
               use_bias=False,
               metric_fn=None,
               force_grow=False,
               replicate_ensemble_in_training=False,
               adanet_loss_decay=.9,
               worker_wait_timeout_secs=7200,
               model_dir=None,
               report_dir=None,
               config=None,
               use_tpu=True,
               batch_size=None):
    """Initializes a `TPUEstimator`. See base classes for details."""

    if not use_tpu:
      tf.logging.warning(
          'This adanet.TPUEstimator is meant to be used for running on TPU. '
          'If you want to run on CPU/GPU, use adanet.Estimator instead.')

    super(TPUEstimator, self).__init__(
        head=head,
        subnetwork_generator=subnetwork_generator,
        max_iteration_steps=max_iteration_steps,
        mixture_weight_type=mixture_weight_type,
        mixture_weight_initializer=mixture_weight_initializer,
        warm_start_mixture_weights=warm_start_mixture_weights,
        adanet_lambda=adanet_lambda,
        adanet_beta=adanet_beta,
        evaluator=evaluator,
        report_materializer=report_materializer,
        use_bias=use_bias,
        metric_fn=metric_fn,
        force_grow=force_grow,
        replicate_ensemble_in_training=replicate_ensemble_in_training,
        adanet_loss_decay=adanet_loss_decay,
        worker_wait_timeout_secs=worker_wait_timeout_secs,
        model_dir=model_dir,
        report_dir=report_dir,
        config=config if config else tf.contrib.tpu.RunConfig(),
        use_tpu=use_tpu,
        eval_on_tpu=False,
        export_to_tpu=False,
        train_batch_size=batch_size or 0)

  def _call_adanet_model_fn(self, input_fn, mode, params):
    """See the `Estimator` base class for details."""

    # Fakes TPU shard context before calling through to the parent to supress
    # warnings by CrossShardOptimizer when running on TPU. Warnings are issued
    # when `_adanet_model_fn` is called directly on CPU during the bookkeeping
    # phase. Since we rebuild the graph each time `_adanet_model_fn` is called,
    # this has no adverse effects.
    with tpu_function.tpu_shard_context(0):
      super(TPUEstimator, self)._call_adanet_model_fn(input_fn, mode, params)

  def _adanet_model_fn(self, features, labels, mode, params):
    """See the `Estimator` base class for details."""

    estimator_spec = super(TPUEstimator, self)._adanet_model_fn(
        features, labels, mode, params)
    if 'use_tpu' in params and mode == tf.estimator.ModeKeys.TRAIN:
      kwargs = {
          key: value
          for key, value in six.iteritems(estimator_spec._asdict())
          if key not in ('eval_metric_ops', 'scaffold', 'training_chief_hooks')
      }
      estimator_spec = tf.contrib.tpu.TPUEstimatorSpec(**kwargs)
    return estimator_spec
<commit_msg>Bind params to input_fn when calling it from _call_adanet_model_fn.<commit_after>"""An AdaNet estimator implementation which can run on TPU.

Copyright 2018 The AdaNet Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    https://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import functools

from adanet.core.ensemble import MixtureWeightType
from adanet.core.estimator import Estimator
from adanet.core.summary import _ScopedSummary
import six
import tensorflow as tf
from tensorflow.contrib.tpu.python.tpu import tpu_function
from tensorflow.python import summary

# TODO: support summaries on TPU during training.
def _rewire_summaries():
  """Rewire Tensorflow summaries to be no-ops when running on TPU.

  Summaries are not currently supported on TPU.
  """

  def tpu_no_op(fn):

    def _fn(*args, **kwargs):
      if tpu_function.get_tpu_context().number_of_shards:
        return None
      return fn(*args, **kwargs)

    return _fn

  summary.audio = tpu_no_op(summary.audio)
  summary.histogram = tpu_no_op(summary.histogram)
  summary.image = tpu_no_op(summary.image)
  summary.scalar = tpu_no_op(summary.scalar)
  summary.tensor_summary = tpu_no_op(summary.tensor_summary)
  summary.text = tpu_no_op(summary.text)

  tf.summary.audio = tpu_no_op(tf.summary.audio)
  tf.summary.histogram = tpu_no_op(tf.summary.histogram)
  tf.summary.image = tpu_no_op(tf.summary.image)
  tf.summary.scalar = tpu_no_op(tf.summary.scalar)
  tf.summary.tensor_summary = tpu_no_op(tf.summary.tensor_summary)
  tf.summary.text = tpu_no_op(tf.summary.text)

  _ScopedSummary.audio = tpu_no_op(_ScopedSummary.audio)
  _ScopedSummary.histogram = tpu_no_op(_ScopedSummary.histogram)
  _ScopedSummary.image = tpu_no_op(_ScopedSummary.image)
  _ScopedSummary.scalar = tpu_no_op(_ScopedSummary.scalar)

# Rewire summaries to be no-ops when running on TPU.
_rewire_summaries()

class TPUEstimator(Estimator, tf.contrib.tpu.TPUEstimator):
  """An adanet.Estimator capable of running on TPU.

  If running on TPU, all summary calls are rewired to be no-ops during training.

  WARNING: this API is highly experimental, unstable, and can change  without
  warning.
  """

  def __init__(self,
               head,
               subnetwork_generator,
               max_iteration_steps,
               mixture_weight_type=MixtureWeightType.SCALAR,
               mixture_weight_initializer=None,
               warm_start_mixture_weights=False,
               adanet_lambda=0.,
               adanet_beta=0.,
               evaluator=None,
               report_materializer=None,
               use_bias=False,
               metric_fn=None,
               force_grow=False,
               replicate_ensemble_in_training=False,
               adanet_loss_decay=.9,
               worker_wait_timeout_secs=7200,
               model_dir=None,
               report_dir=None,
               config=None,
               use_tpu=True,
               batch_size=None):
    """Initializes a `TPUEstimator`. See base classes for details."""

    if not use_tpu:
      tf.logging.warning(
          'This adanet.TPUEstimator is meant to be used for running on TPU. '
          'If you want to run on CPU/GPU, use adanet.Estimator instead.')

    super(TPUEstimator, self).__init__(
        head=head,
        subnetwork_generator=subnetwork_generator,
        max_iteration_steps=max_iteration_steps,
        mixture_weight_type=mixture_weight_type,
        mixture_weight_initializer=mixture_weight_initializer,
        warm_start_mixture_weights=warm_start_mixture_weights,
        adanet_lambda=adanet_lambda,
        adanet_beta=adanet_beta,
        evaluator=evaluator,
        report_materializer=report_materializer,
        use_bias=use_bias,
        metric_fn=metric_fn,
        force_grow=force_grow,
        replicate_ensemble_in_training=replicate_ensemble_in_training,
        adanet_loss_decay=adanet_loss_decay,
        worker_wait_timeout_secs=worker_wait_timeout_secs,
        model_dir=model_dir,
        report_dir=report_dir,
        config=config if config else tf.contrib.tpu.RunConfig(),
        use_tpu=use_tpu,
        eval_on_tpu=False,
        export_to_tpu=False,
        train_batch_size=batch_size or 0)

  def _call_adanet_model_fn(self, input_fn, mode, params):
    """See the `Estimator` base class for details."""

    # Fakes TPU shard context before calling through to the parent to supress
    # warnings by CrossShardOptimizer when running on TPU. Warnings are issued
    # when `_adanet_model_fn` is called directly on CPU during the bookkeeping
    # phase. Since we rebuild the graph each time `_adanet_model_fn` is called,
    # this has no adverse effects.
    with tpu_function.tpu_shard_context(0):
      # Bind params to input_fn since the parent's input_fn is not expected to
      # have any arguments.
      input_fn = functools.partial(input_fn, params)
      super(TPUEstimator, self)._call_adanet_model_fn(input_fn, mode, params)

  def _adanet_model_fn(self, features, labels, mode, params):
    """See the `Estimator` base class for details."""

    estimator_spec = super(TPUEstimator, self)._adanet_model_fn(
        features, labels, mode, params)
    if 'use_tpu' in params and mode == tf.estimator.ModeKeys.TRAIN:
      kwargs = {
          key: value
          for key, value in six.iteritems(estimator_spec._asdict())
          if key not in ('eval_metric_ops', 'scaffold', 'training_chief_hooks')
      }
      estimator_spec = tf.contrib.tpu.TPUEstimatorSpec(**kwargs)
    return estimator_spec
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Addons modules by CLEARCORP S.A.
#    Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
import time
import pooler
from osv import fields, osv
from tools.translate import _
from tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, float_compare
import decimal_precision as dp
import netsvc

class sale_order_line(osv.osv):
    _inherit = 'sale.order.line'
    _description = 'Order Line'

    def _amount_line_no_discount(self, cr, uid, ids, field_name, arg, context=None):
        tax_obj = self.pool.get('account.tax')
        cur_obj = self.pool.get('res.currency')
        res = {}
        if context is None:
            context = {}
        for line in self.browse(cr, uid, ids, context=context):
            price = line.price_unit
            taxes = tax_obj.compute_all(cr, uid, line.tax_id, price, line.product_uom_qty, line.order_id.partner_invoice_id.id, line.product_id, line.order_id.partner_id)
            cur = line.order_id.pricelist_id.currency_id
            res[line.id] = cur_obj.round(cr, uid, cur, taxes['total'])
        return res

    _columns = {
        'price_subtotal_not_discounted': fields.function(_amount_line_no_discount, digits_compute= dp.get_precision('Account'), string='Subtotal',
            store = {
                'sale.order.line': (lambda self, cr, uid, ids, c={}: ids, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 5),
            }),
    }

    def button_dummy(self, cr, uid, ids, context=None):
        #To recalculate function fields
        for line in self.browse(cr, uid, ids, context=context):
            self.write(cr, uid, [line.id], {'id': line.id}, context=context)
        return True

class SaleOrder(osv.osv):
    _inherit = 'sale.order'
    _description = 'Sales Order'

    def _amount_line_tax_no_discount(self, cr, uid, line, context=None):
        val = 0.0
        for c in self.pool.get('account.tax').compute_all(cr, uid, line.tax_id, line.price_unit, line.product_uom_qty, line.order_id.partner_invoice_id.id, line.product_id, line.order_id.partner_id)['taxes']:
            val += c.get('amount', 0.0)
        return val

    def _amount_all(self, cr, uid, ids, field_name, arg, context=None):
        ###Workaround, functional fields###
        ###See sale.order###
        cur_obj = self.pool.get('res.currency')
        res = {}
        for order in self.browse(cr, uid, ids, context=context):
            res[order.id] = {
                'amount_untaxed': 0.0,
                'amount_tax': 0.0,
                'amount_total': 0.0,
                'order_discount': 0.0,
                'amount_discounted': 0.0,
                'amount_untaxed_not_discounted': 0.0,
            }
            val = val1 = 0.0
            cur = order.pricelist_id.currency_id
            for line in order.order_line:
                val1 += line.price_subtotal
                val += self._amount_line_tax(cr, uid, line, context=context)
                res[order.id]['amount_untaxed_not_discounted'] += line.price_subtotal_not_discounted
                res[order.id]['amount_discounted'] += line.price_subtotal_not_discounted - line.price_subtotal
            res[order.id]['amount_tax'] = cur_obj.round(cr, uid, cur, val)
            res[order.id]['amount_untaxed'] = cur_obj.round(cr, uid, cur, val1)
            res[order.id]['amount_total'] = res[order.id]['amount_untaxed'] + res[order.id]['amount_tax']
            if res[order.id]['amount_untaxed_not_discounted'] == 0:
                res[order.id]['order_discount'] = 0
            else:
                res[order.id]['order_discount'] = 100 * res[order.id]['amount_discounted'] / res[order.id]['amount_untaxed_not_discounted']
        return res

    def _get_order_ccorp(self, cr, uid, ids, context=None):
        return self.pool.get('sale.order')._get_order(cr, uid, ids, context)

    _columns = {
        'amount_untaxed': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Untaxed Amount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The amount without tax."),
        'amount_tax': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Taxes',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The tax amount.", track_visibility='always'),
        'amount_total': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Total',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The total amount."),
        'order_discount': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), required=True, string='Untaxed Amount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
        'amount_discounted': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), required=True, string='Discount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
        'amount_untaxed_not_discounted': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), required=True, string='Subtotal',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
                }

    def button_dummy(self, cr, uid, ids, context={}):
        if context is None:
            context = {}
        #To recalculate function fields
        if isinstance(ids, int):
            ids = [ids]
        for sale in self.browse(cr, uid, ids, context=context):
            context.update({'second_time': True})
            self.write(cr, uid, [sale.id], {}, context=context)
        super(SaleOrder, self).button_dummy(cr, uid, ids, context=context)
        return True

    def create(self, cr, uid, vals, context=None):
        sale_id = super(SaleOrder, self).create(cr, uid, vals, context=context)
        self.button_dummy(cr, uid, [sale_id], context=context)
        return sale_id

    def write(self, cr, uid, ids, vals, context={}):
        if context is None:
            context = {}
        if 'second_time' not in context:
             self.button_dummy(cr, uid, ids, context=context)
        else:
            del context['second_time']
        return super(SaleOrder, self).write(cr, uid, ids, vals, context=context)
<commit_msg>[FIX] Fix required functional fields in sale_order_global_discount<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Addons modules by CLEARCORP S.A.
#    Copyright (C) 2009-TODAY CLEARCORP S.A. (<http://clearcorp.co.cr>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
import time
import pooler
from osv import fields, osv
from tools.translate import _
from tools import DEFAULT_SERVER_DATE_FORMAT, DEFAULT_SERVER_DATETIME_FORMAT, float_compare
import decimal_precision as dp
import netsvc

class sale_order_line(osv.osv):
    _inherit = 'sale.order.line'
    _description = 'Order Line'

    def _amount_line_no_discount(self, cr, uid, ids, field_name, arg, context=None):
        tax_obj = self.pool.get('account.tax')
        cur_obj = self.pool.get('res.currency')
        res = {}
        if context is None:
            context = {}
        for line in self.browse(cr, uid, ids, context=context):
            price = line.price_unit
            taxes = tax_obj.compute_all(cr, uid, line.tax_id, price, line.product_uom_qty, line.order_id.partner_invoice_id.id, line.product_id, line.order_id.partner_id)
            cur = line.order_id.pricelist_id.currency_id
            res[line.id] = cur_obj.round(cr, uid, cur, taxes['total'])
        return res

    _columns = {
        'price_subtotal_not_discounted': fields.function(_amount_line_no_discount, digits_compute= dp.get_precision('Account'), string='Subtotal',
            store = {
                'sale.order.line': (lambda self, cr, uid, ids, c={}: ids, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 5),
            }),
    }

    def button_dummy(self, cr, uid, ids, context=None):
        #To recalculate function fields
        for line in self.browse(cr, uid, ids, context=context):
            self.write(cr, uid, [line.id], {'id': line.id}, context=context)
        return True

class SaleOrder(osv.osv):
    _inherit = 'sale.order'
    _description = 'Sales Order'

    def _amount_line_tax_no_discount(self, cr, uid, line, context=None):
        val = 0.0
        for c in self.pool.get('account.tax').compute_all(cr, uid, line.tax_id, line.price_unit, line.product_uom_qty, line.order_id.partner_invoice_id.id, line.product_id, line.order_id.partner_id)['taxes']:
            val += c.get('amount', 0.0)
        return val

    def _amount_all(self, cr, uid, ids, field_name, arg, context=None):
        ###Workaround, functional fields###
        ###See sale.order###
        cur_obj = self.pool.get('res.currency')
        res = {}
        for order in self.browse(cr, uid, ids, context=context):
            res[order.id] = {
                'amount_untaxed': 0.0,
                'amount_tax': 0.0,
                'amount_total': 0.0,
                'order_discount': 0.0,
                'amount_discounted': 0.0,
                'amount_untaxed_not_discounted': 0.0,
            }
            val = val1 = 0.0
            cur = order.pricelist_id.currency_id
            for line in order.order_line:
                val1 += line.price_subtotal
                val += self._amount_line_tax(cr, uid, line, context=context)
                res[order.id]['amount_untaxed_not_discounted'] += line.price_subtotal_not_discounted
                res[order.id]['amount_discounted'] += line.price_subtotal_not_discounted - line.price_subtotal
            res[order.id]['amount_tax'] = cur_obj.round(cr, uid, cur, val)
            res[order.id]['amount_untaxed'] = cur_obj.round(cr, uid, cur, val1)
            res[order.id]['amount_total'] = res[order.id]['amount_untaxed'] + res[order.id]['amount_tax']
            if res[order.id]['amount_untaxed_not_discounted'] == 0:
                res[order.id]['order_discount'] = 0
            else:
                res[order.id]['order_discount'] = 100 * res[order.id]['amount_discounted'] / res[order.id]['amount_untaxed_not_discounted']
        return res

    def _get_order_ccorp(self, cr, uid, ids, context=None):
        return self.pool.get('sale.order')._get_order(cr, uid, ids, context)

    _columns = {
        'amount_untaxed': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Untaxed Amount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The amount without tax."),
        'amount_tax': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Taxes',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The tax amount.", track_visibility='always'),
        'amount_total': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Total',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums', help="The total amount."),
        'order_discount': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Untaxed Amount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
        'amount_discounted': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Discount',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
        'amount_untaxed_not_discounted': fields.function(_amount_all, digits_compute= dp.get_precision('Account'), string='Subtotal',
            store = {
                'sale.order': (lambda self, cr, uid, ids, c={}: ids, ['order_line'], 10),
                'sale.order.line': (_get_order_ccorp, ['price_unit', 'tax_id', 'discount', 'product_uom_qty'], 10),
            },
            multi='sums'),
                }

    def button_dummy(self, cr, uid, ids, context={}):
        if context is None:
            context = {}
        #To recalculate function fields
        if isinstance(ids, int):
            ids = [ids]
        for sale in self.browse(cr, uid, ids, context=context):
            context.update({'second_time': True})
            self.write(cr, uid, [sale.id], {}, context=context)
        super(SaleOrder, self).button_dummy(cr, uid, ids, context=context)
        return True

    def create(self, cr, uid, vals, context=None):
        sale_id = super(SaleOrder, self).create(cr, uid, vals, context=context)
        self.button_dummy(cr, uid, [sale_id], context=context)
        return sale_id

    def write(self, cr, uid, ids, vals, context={}):
        if context is None:
            context = {}
        if 'second_time' not in context:
             self.button_dummy(cr, uid, ids, context=context)
        else:
            del context['second_time']
        return super(SaleOrder, self).write(cr, uid, ids, vals, context=context)
<|endoftext|>


<commit_before>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import itertools

from conary import versions
from conary.build import defaultrecipes, macros, use
from conary.build.errors import CookError
from conary.build.grouprecipe import _BaseGroupRecipe, _SingleGroup
from conary.build.recipe import loadMacros
from conary.conaryclient import troveset
from conary.conaryclient.resolve import PythonDependencyChecker
from conary.repository import searchsource
from conary.deps import deps

class GroupSetTroveCache(object):

    def __init__(self, groupRecipe, cache):
        self.cache = cache
        self.groupRecipe = groupRecipe
        self.depCache = {}

    def __getattr__(self, name):
        return getattr(self.cache, name)

    def iterTroveList(self, troveTup, strongRefs=False, weakRefs=False):
        raise NotImplementedError

    def iterTroveListInfo(self, troveTup):
        if isinstance(troveTup[1], versions.NewVersion):
            sg = self.groupRecipe._getGroup(troveTup[0])

            for x in sg.iterTroveListInfo():
                yield (x[0], x[2], x[1])

            for name, byDefault, explicit in sg.iterNewGroupList():
                yield (name, versions.NewVersion(),
                       self.groupRecipe.flavor), byDefault, explicit
        else:
            for x in self.cache.iterTroveListInfo(troveTup):
                yield x

class GroupActionData(troveset.ActionData):

    def __init__(self, troveCache, groupRecipe):
        troveset.ActionData.__init__(self, troveCache, groupRecipe.flavor)
        self.groupRecipe = groupRecipe

class GroupTupleSetMethods(object):

    def depsNeeded(self, resolveSource, failOnUnresolved = True):
        if isinstance(resolveSource, troveset.SearchPathTroveSet):
            newList = []
            for ts in resolveSource.troveSetList:
                if isinstance(ts, troveset.TroveTupleSet):
                    ts = ts._action(ActionClass = troveset.FetchAction)
                newList.append(ts)

            resolveSource = troveset.SearchPathTroveSet(newList,
                                                        graph = self.g)
        elif isinstance(resolveSource, troveset.TroveTupleSet):
            resolveSource = resolveSource._action(
                                    ActionClass = troveset.FetchAction)

        fetched = self._action(ActionClass = troveset.FetchAction)

        return fetched._action(resolveSource,
                                     failOnUnresolved = failOnUnresolved,
                                     ActionClass = DepsNeededAction)

    def difference(self, other):
        if type(other) == str:
            findSet = self.find(other)
            return self._action(findSet, ActionClass = GroupDifferenceAction,
                                edgeList = [ None, '-' ] )

        return self._action(other, ActionClass = GroupDifferenceAction)

    __sub__ = difference
    remove = difference

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

    def flatten(self):
        return self._action(ActionClass = FlattenAction)

    def getInstall(self):
        return self._action(ActionClass = GetInstalledAction)

    def getOptional(self):
        return self._action(ActionClass = GetOptionalAction)

    def makeInstall(self, installTroveSet = None):
        return self._action(ActionClass = MakeInstallAction,
                            installTroveSet = installTroveSet)

    def makeOptional(self, optionalTroveSet = None):
        return self._action(ActionClass = MakeOptionalAction,
                            optionalTroveSet = optionalTroveSet)

    def members(self):
        return self._action(ActionClass = MembersAction)

    def union(self, *troveSetList):
        return self._action(ActionClass = GroupUnionAction, *troveSetList)

    def createGroup(self, name, checkPathConflicts = True):
        return self._action(name, checkPathConflicts = checkPathConflicts,
                            ActionClass = CreateNewGroupAction)

    def _createGroup(self, name, checkPathConflicts = True):
        return self._action(name, ActionClass = CreateGroupAction,
                            checkPathConflicts = checkPathConflicts)

    __add__ = union
    __or__ = union

class GroupDelayedTroveTupleSet(GroupTupleSetMethods,
                                troveset.DelayedTupleSet):

    pass

class GroupSearchPathTroveSet(troveset.SearchPathTroveSet):

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

class GroupSearchSourceTroveSet(troveset.SearchSourceTroveSet):

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

class GroupFindAction(troveset.FindAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupDelayedTupleSetAction(troveset.DelayedTupleSetAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupDifferenceAction(troveset.DifferenceAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupUnionAction(troveset.UnionAction):

    resultClass = GroupDelayedTroveTupleSet

class CreateGroupAction(GroupDelayedTupleSetAction):

    prefilter = troveset.FetchAction

    def __init__(self, primaryTroveSet, name, checkPathConflicts = True):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet)
        self.name = name
        self.checkPathConflicts = checkPathConflicts

    def __call__(self, data):
        grp = SG(data.groupRecipe.name,
                 checkPathConflicts = self.checkPathConflicts)
        data.groupRecipe._addGroup(self.name, grp)
        data.groupRecipe._setDefaultGroup(grp)

        self._create(data.groupRecipe.defaultGroup,
                     self.primaryTroveSet, self.outSet, data)

    def _create(self, sg, ts, outSet, data):
        sg.populate(ts, data.troveCache)

        outSet._setInstall([ (sg.name, versions.NewVersion(),
                              data.groupRecipe.flavor) ])
        outSet.realized = True

    def __str__(self):
        return self.name

class CreateNewGroupAction(CreateGroupAction):

    def __init__(self, primaryTroveSet, name, checkPathConflicts = True):
        CreateGroupAction.__init__(self, primaryTroveSet, name,
                                   checkPathConflicts = checkPathConflicts)

    def __call__(self, data):
        newGroup = SG(self.name, checkPathConflicts = self.checkPathConflicts)
        data.groupRecipe._addGroup(self.name, newGroup)
        self._create(newGroup, self.primaryTroveSet, self.outSet, data)

class DepsNeededAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, resolveTroveSet,
                 failOnUnresolved = True):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            resolveTroveSet)
        self.failOnUnresolved = failOnUnresolved
        self.resolveTroveSet = resolveTroveSet

    def __call__(self, data):
        checker = PythonDependencyChecker(
                        data.troveCache,
                        ignoreDepClasses = [ deps.AbiDependency,
                                             deps.RpmLibDependencies ])

        troveList = []
        for (troveTuple, isInstall, isExplicit) in \
                    self.primaryTroveSet._walk(data.troveCache,
                                newGroups = False, recurse = True):
            if isInstall:
                troveList.append(troveTuple)

        jobSet = [ (n, (None, None), (v, f), False) for (n,v,f) in troveList ]

        checker.addJobs(jobSet)
        resolveMethod = (self.resolveTroveSet._getResolveSource().
                                    getResolveMethod())

        failedDeps, suggMap = checker.resolve(resolveMethod)

        if self.failOnUnresolved and failedDeps:
            raise CookError("Unresolved Deps:\n" +
                "\n".join(
                [ "\t%s=%s[%s] requires %s" % (name, version, flavor, dep)
                  for ((name, version, flavor), dep) in failedDeps ]))

        installSet = set()
        for requiredBy, requiredSet in suggMap.iteritems():
            installSet.update(requiredSet)

        self.outSet._setInstall(installSet)

class GetInstalledAction(GroupDelayedTupleSetAction):

    def __call__(self, data):
        self.outSet._setInstall(self.primaryTroveSet._getInstallSet())

class GetOptionalAction(GroupDelayedTupleSetAction):

    def __call__(self, data):
        self.outSet._setOptional(self.primaryTroveSet._getOptionalSet())

class MakeInstallAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, installTroveSet = None):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            installTroveSet)
        self.installTroveSet = installTroveSet

    def __call__(self, data):
        if self.installTroveSet:
            self.outSet._setOptional(self.primaryTroveSet._getOptionalSet())
            self.outSet._setInstall(
                    (self.installTroveSet._getInstallSet() |
                     self.installTroveSet._getOptionalSet()))
        else:
            self.outSet._setInstall(self.primaryTroveSet._getInstallSet() |
                                    self.primaryTroveSet._getOptionalSet())

class MakeOptionalAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, optionalTroveSet = None):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            optionalTroveSet)
        self.optionalTroveSet = optionalTroveSet

    def __call__(self, data):
        if self.optionalTroveSet:
            self.outSet._setInstall(self.primaryTroveSet._getInstallSet())
            self.outSet._setOptional(
                    (self.optionalTroveSet._getInstallSet() |
                     self.optionalTroveSet._getOptionalSet()))
        else:
            self.outSet._setOptional(self.primaryTroveSet._getInstallSet() |
                                     self.primaryTroveSet._getOptionalSet())

class MembersAction(GroupDelayedTupleSetAction):

    prefilter = troveset.FetchAction
    justStrong = True

    def __call__(self, data):
        for (troveTuple, installSet) in itertools.chain(
                itertools.izip(self.primaryTroveSet._getInstallSet(),
                               itertools.repeat(True)),
                itertools.izip(self.primaryTroveSet._getOptionalSet(),
                               itertools.repeat(False))):
            installs = []
            available = []

            for (refTrove, byDefault, isStrong) in \
                        data.troveCache.iterTroveListInfo(troveTuple):
                if self.justStrong and not isStrong:
                    continue

                if byDefault:
                    installs.append(refTrove)
                elif not byDefault:
                    available.append(refTrove)

            self.outSet._setInstall(installs)
            self.outSet._setOptional(available)

class FlattenAction(MembersAction):

    justStrong = False

class SG(_SingleGroup):

    def __init__(self, *args, **kwargs):
        _SingleGroup.__init__(self, *args, **kwargs)
        self.autoResolve = False
        self.depCheck = False
        self.imageGroup = False

    def populate(self, troveSet, troveCache):
        seen = set()

        for troveTup, byDefault, explicit in troveSet._walk(troveCache):
            if not explicit:
                continue

            seen.add(troveTup)
            if isinstance(troveTup[1], versions.NewVersion):
                self.addNewGroup(troveTup[0], byDefault = byDefault,
                                 explicit = True)
            else:
                self.addTrove(troveTup, explicit = True, byDefault = byDefault,
                              components = [])

        for troveTup, byDefault, explicit in troveSet._walk(troveCache,
                                                            recurse = True):
            if troveTup in seen:
                # if it's explicit, it's already been seen
                continue

            seen.add(troveTup)

            if isinstance(troveTup[1], versions.NewVersion):
                self.addNewGroup(troveTup[0], byDefault = byDefault,
                                 explicit = False)
            else:
                self.addTrove(troveTup, explicit = False, byDefault = byDefault,
                              components = [])

    def iterAddSpecs(self):
        return []

    def iterAddAllSpecs(self):
        return []

    def iterReplaceSpecs(self):
        return []

    def iterDifferenceSpecs(self):
        return []

    def iterNewGroupDifferenceList(self):
        return []

    def iterCopiedFrom(self):
        return []

    def getComponentsToMove(self):
        return []

    def getRequires(self):
        return deps.DependencySet()

class _GroupSetRecipe(_BaseGroupRecipe):

    Flags = use.LocalFlags
    internalAbstractBaseClass = 1

    def __init__(self, repos, cfg, label, flavor, laReposCache, srcdirs=None,
                 extraMacros={}, lightInstance = False):

        klass = self._getParentClass('_BaseGroupRecipe')
        klass.__init__(self, laReposCache = laReposCache,
                       srcdirs = srcdirs,
                       lightInstance = lightInstance,
                       cfg = cfg)

        self.troveSource = repos
        self.repos = repos

        self.labelPath = [ label ]
        self.buildLabel = label
        self.flavor = flavor
        self.searchSource = searchsource.NetworkSearchSource(
                repos, self.labelPath, flavor)
        self.macros = macros.Macros(ignoreUnknown=lightInstance)
        self.world = GroupSearchSourceTroveSet(self.searchSource)
        self.g = troveset.OperationGraph()

        baseMacros = loadMacros(cfg.defaultMacros)
        self.macros.update(baseMacros)
        for key in cfg.macros:
            self.macros._override(key, cfg['macros'][key])
        self.macros.name = self.name
        self.macros.version = self.version
        if '.' in self.version:
            self.macros.major_version = '.'.join(self.version.split('.')[0:2])
        else:
            self.macros.major_version = self.version
        if extraMacros:
            self.macros.update(extraMacros)

    def _realizeGraph(self, cache, callback):
        data = GroupActionData(troveCache = GroupSetTroveCache(self, cache),
                               groupRecipe = self)
        self.g.realize(data)

    def getLabelPath(self):
        return self.labelPath

    def getSearchFlavor(self):
        return self.flavor

    def iterReplaceSpecs(self):
        return []

    def getResolveTroveSpecs(self):
        return []

    def getChildGroups(self, groupName = None):
        return []

    def getGroupMap(self, groupName = None):
        return {}

    def _getSearchSource(self):
        return self.troveSource

    def getSearchPath(self):
        return [ ]

    def writeDotGraph(self, path):
        self.g.generateDotFile(path, edgeFormatFn = lambda a,b,c: c)

    def Group(self, ts, checkPathConflicts = True):
        return ts._createGroup(self.name,
                               checkPathConflicts = checkPathConflicts)

    def Repository(self, labelList, flavor):
        if type(labelList) == tuple:
            labelList = list(tuple)
        elif type(labelList) != list:
            labelList = [ labelList ]

        for i, label in enumerate(labelList):
            if type(label) == str:
                labelList[i] = versions.Label(label)
            elif not isinstance(label, versions.Label):
                raise CookError("String label or Label object expected")

        if type(flavor) == str:
            flavor = deps.parseFlavor(flavor)

        searchSource = searchsource.NetworkSearchSource(
                                            self.repos, labelList, flavor)
        return GroupSearchSourceTroveSet(searchSource, graph = self.g)

    def SearchPath(self, *troveSets):
        return GroupSearchPathTroveSet(troveSets, graph = self.g)

from conary.build.packagerecipe import BaseRequiresRecipe
exec defaultrecipes.GroupSetRecipe
<commit_msg>added Components action for groupset<commit_after>#
# Copyright (c) 2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

import itertools

from conary import trove, versions
from conary.build import defaultrecipes, macros, use
from conary.build.errors import CookError
from conary.build.grouprecipe import _BaseGroupRecipe, _SingleGroup
from conary.build.recipe import loadMacros
from conary.conaryclient import troveset
from conary.conaryclient.resolve import PythonDependencyChecker
from conary.repository import searchsource
from conary.deps import deps

class GroupSetTroveCache(object):

    def __init__(self, groupRecipe, cache):
        self.cache = cache
        self.groupRecipe = groupRecipe
        self.depCache = {}

    def __getattr__(self, name):
        return getattr(self.cache, name)

    def iterTroveList(self, troveTup, strongRefs=False, weakRefs=False):
        raise NotImplementedError

    def iterTroveListInfo(self, troveTup):
        if isinstance(troveTup[1], versions.NewVersion):
            sg = self.groupRecipe._getGroup(troveTup[0])

            for x in sg.iterTroveListInfo():
                yield (x[0], x[2], x[1])

            for name, byDefault, explicit in sg.iterNewGroupList():
                yield (name, versions.NewVersion(),
                       self.groupRecipe.flavor), byDefault, explicit
        else:
            for x in self.cache.iterTroveListInfo(troveTup):
                yield x

class GroupActionData(troveset.ActionData):

    def __init__(self, troveCache, groupRecipe):
        troveset.ActionData.__init__(self, troveCache, groupRecipe.flavor)
        self.groupRecipe = groupRecipe

class GroupTupleSetMethods(object):

    def depsNeeded(self, resolveSource, failOnUnresolved = True):
        if isinstance(resolveSource, troveset.SearchPathTroveSet):
            newList = []
            for ts in resolveSource.troveSetList:
                if isinstance(ts, troveset.TroveTupleSet):
                    ts = ts._action(ActionClass = troveset.FetchAction)
                newList.append(ts)

            resolveSource = troveset.SearchPathTroveSet(newList,
                                                        graph = self.g)
        elif isinstance(resolveSource, troveset.TroveTupleSet):
            resolveSource = resolveSource._action(
                                    ActionClass = troveset.FetchAction)

        fetched = self._action(ActionClass = troveset.FetchAction)

        return fetched._action(resolveSource,
                                     failOnUnresolved = failOnUnresolved,
                                     ActionClass = DepsNeededAction)

    def difference(self, other):
        if type(other) == str:
            findSet = self.find(other)
            return self._action(findSet, ActionClass = GroupDifferenceAction,
                                edgeList = [ None, '-' ] )

        return self._action(other, ActionClass = GroupDifferenceAction)

    __sub__ = difference
    remove = difference

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

    def components(self, *componentList):
        return self._action(ActionClass = ComponentsAction, *componentList)

    def flatten(self):
        return self._action(ActionClass = FlattenAction)

    def getInstall(self):
        return self._action(ActionClass = GetInstalledAction)

    def getOptional(self):
        return self._action(ActionClass = GetOptionalAction)

    def makeInstall(self, installTroveSet = None):
        return self._action(ActionClass = MakeInstallAction,
                            installTroveSet = installTroveSet)

    def makeOptional(self, optionalTroveSet = None):
        return self._action(ActionClass = MakeOptionalAction,
                            optionalTroveSet = optionalTroveSet)

    def members(self):
        return self._action(ActionClass = MembersAction)

    def union(self, *troveSetList):
        return self._action(ActionClass = GroupUnionAction, *troveSetList)

    def createGroup(self, name, checkPathConflicts = True):
        return self._action(name, checkPathConflicts = checkPathConflicts,
                            ActionClass = CreateNewGroupAction)

    def _createGroup(self, name, checkPathConflicts = True):
        return self._action(name, ActionClass = CreateGroupAction,
                            checkPathConflicts = checkPathConflicts)

    __add__ = union
    __or__ = union

class GroupDelayedTroveTupleSet(GroupTupleSetMethods,
                                troveset.DelayedTupleSet):

    pass

class GroupSearchPathTroveSet(troveset.SearchPathTroveSet):

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

class GroupSearchSourceTroveSet(troveset.SearchSourceTroveSet):

    def find(self, *troveSpecs):
        return self._action(ActionClass = GroupFindAction, *troveSpecs)

    __getitem__ = find

class GroupFindAction(troveset.FindAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupDelayedTupleSetAction(troveset.DelayedTupleSetAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupDifferenceAction(troveset.DifferenceAction):

    resultClass = GroupDelayedTroveTupleSet

class GroupUnionAction(troveset.UnionAction):

    resultClass = GroupDelayedTroveTupleSet

class ComponentsAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, *componentNames):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet)
        self.componentNames = set(componentNames)

    def __call__(self, data):
        installSet = set()
        optionalSet = set()

        for (troveTup), inInstall, explicit in \
                        self.primaryTroveSet._walk(data.troveCache):
            if not trove.troveIsComponent(troveTup[0]):
                continue

            componentName = troveTup[0].split(':')[1]
            if componentName in self.componentNames:
                if inInstall:
                    installSet.add(troveTup)
                else:
                    optionalSet.add(troveTup)

        self.outSet._setInstall(installSet)
        self.outSet._setOptional(optionalSet)

class CreateGroupAction(GroupDelayedTupleSetAction):

    prefilter = troveset.FetchAction

    def __init__(self, primaryTroveSet, name, checkPathConflicts = True):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet)
        self.name = name
        self.checkPathConflicts = checkPathConflicts

    def __call__(self, data):
        grp = SG(data.groupRecipe.name,
                 checkPathConflicts = self.checkPathConflicts)
        data.groupRecipe._addGroup(self.name, grp)
        data.groupRecipe._setDefaultGroup(grp)

        self._create(data.groupRecipe.defaultGroup,
                     self.primaryTroveSet, self.outSet, data)

    def _create(self, sg, ts, outSet, data):
        sg.populate(ts, data.troveCache)

        outSet._setInstall([ (sg.name, versions.NewVersion(),
                              data.groupRecipe.flavor) ])
        outSet.realized = True

    def __str__(self):
        return self.name

class CreateNewGroupAction(CreateGroupAction):

    def __init__(self, primaryTroveSet, name, checkPathConflicts = True):
        CreateGroupAction.__init__(self, primaryTroveSet, name,
                                   checkPathConflicts = checkPathConflicts)

    def __call__(self, data):
        newGroup = SG(self.name, checkPathConflicts = self.checkPathConflicts)
        data.groupRecipe._addGroup(self.name, newGroup)
        self._create(newGroup, self.primaryTroveSet, self.outSet, data)

class DepsNeededAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, resolveTroveSet,
                 failOnUnresolved = True):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            resolveTroveSet)
        self.failOnUnresolved = failOnUnresolved
        self.resolveTroveSet = resolveTroveSet

    def __call__(self, data):
        checker = PythonDependencyChecker(
                        data.troveCache,
                        ignoreDepClasses = [ deps.AbiDependency,
                                             deps.RpmLibDependencies ])

        troveList = []
        for (troveTuple, isInstall, isExplicit) in \
                    self.primaryTroveSet._walk(data.troveCache,
                                newGroups = False, recurse = True):
            if isInstall:
                troveList.append(troveTuple)

        jobSet = [ (n, (None, None), (v, f), False) for (n,v,f) in troveList ]

        checker.addJobs(jobSet)
        resolveMethod = (self.resolveTroveSet._getResolveSource().
                                    getResolveMethod())

        failedDeps, suggMap = checker.resolve(resolveMethod)

        if self.failOnUnresolved and failedDeps:
            raise CookError("Unresolved Deps:\n" +
                "\n".join(
                [ "\t%s=%s[%s] requires %s" % (name, version, flavor, dep)
                  for ((name, version, flavor), dep) in failedDeps ]))

        installSet = set()
        for requiredBy, requiredSet in suggMap.iteritems():
            installSet.update(requiredSet)

        self.outSet._setInstall(installSet)

class GetInstalledAction(GroupDelayedTupleSetAction):

    def __call__(self, data):
        self.outSet._setInstall(self.primaryTroveSet._getInstallSet())

class GetOptionalAction(GroupDelayedTupleSetAction):

    def __call__(self, data):
        self.outSet._setOptional(self.primaryTroveSet._getOptionalSet())

class MakeInstallAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, installTroveSet = None):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            installTroveSet)
        self.installTroveSet = installTroveSet

    def __call__(self, data):
        if self.installTroveSet:
            self.outSet._setOptional(self.primaryTroveSet._getOptionalSet())
            self.outSet._setInstall(
                    (self.installTroveSet._getInstallSet() |
                     self.installTroveSet._getOptionalSet()))
        else:
            self.outSet._setInstall(self.primaryTroveSet._getInstallSet() |
                                    self.primaryTroveSet._getOptionalSet())

class MakeOptionalAction(GroupDelayedTupleSetAction):

    def __init__(self, primaryTroveSet, optionalTroveSet = None):
        GroupDelayedTupleSetAction.__init__(self, primaryTroveSet,
                                            optionalTroveSet)
        self.optionalTroveSet = optionalTroveSet

    def __call__(self, data):
        if self.optionalTroveSet:
            self.outSet._setInstall(self.primaryTroveSet._getInstallSet())
            self.outSet._setOptional(
                    (self.optionalTroveSet._getInstallSet() |
                     self.optionalTroveSet._getOptionalSet()))
        else:
            self.outSet._setOptional(self.primaryTroveSet._getInstallSet() |
                                     self.primaryTroveSet._getOptionalSet())

class MembersAction(GroupDelayedTupleSetAction):

    prefilter = troveset.FetchAction
    justStrong = True

    def __call__(self, data):
        for (troveTuple, installSet) in itertools.chain(
                itertools.izip(self.primaryTroveSet._getInstallSet(),
                               itertools.repeat(True)),
                itertools.izip(self.primaryTroveSet._getOptionalSet(),
                               itertools.repeat(False))):
            installs = []
            available = []

            for (refTrove, byDefault, isStrong) in \
                        data.troveCache.iterTroveListInfo(troveTuple):
                if self.justStrong and not isStrong:
                    continue

                if byDefault:
                    installs.append(refTrove)
                elif not byDefault:
                    available.append(refTrove)

            self.outSet._setInstall(installs)
            self.outSet._setOptional(available)

class FlattenAction(MembersAction):

    justStrong = False

class SG(_SingleGroup):

    def __init__(self, *args, **kwargs):
        _SingleGroup.__init__(self, *args, **kwargs)
        self.autoResolve = False
        self.depCheck = False
        self.imageGroup = False

    def populate(self, troveSet, troveCache):
        seen = set()

        for troveTup, byDefault, explicit in troveSet._walk(troveCache):
            if not explicit:
                continue

            seen.add(troveTup)
            if isinstance(troveTup[1], versions.NewVersion):
                self.addNewGroup(troveTup[0], byDefault = byDefault,
                                 explicit = True)
            else:
                self.addTrove(troveTup, explicit = True, byDefault = byDefault,
                              components = [])

        for troveTup, byDefault, explicit in troveSet._walk(troveCache,
                                                            recurse = True):
            if troveTup in seen:
                # if it's explicit, it's already been seen
                continue

            seen.add(troveTup)

            if isinstance(troveTup[1], versions.NewVersion):
                self.addNewGroup(troveTup[0], byDefault = byDefault,
                                 explicit = False)
            else:
                self.addTrove(troveTup, explicit = False, byDefault = byDefault,
                              components = [])

    def iterAddSpecs(self):
        return []

    def iterAddAllSpecs(self):
        return []

    def iterReplaceSpecs(self):
        return []

    def iterDifferenceSpecs(self):
        return []

    def iterNewGroupDifferenceList(self):
        return []

    def iterCopiedFrom(self):
        return []

    def getComponentsToMove(self):
        return []

    def getRequires(self):
        return deps.DependencySet()

class _GroupSetRecipe(_BaseGroupRecipe):

    Flags = use.LocalFlags
    internalAbstractBaseClass = 1

    def __init__(self, repos, cfg, label, flavor, laReposCache, srcdirs=None,
                 extraMacros={}, lightInstance = False):

        klass = self._getParentClass('_BaseGroupRecipe')
        klass.__init__(self, laReposCache = laReposCache,
                       srcdirs = srcdirs,
                       lightInstance = lightInstance,
                       cfg = cfg)

        self.troveSource = repos
        self.repos = repos

        self.labelPath = [ label ]
        self.buildLabel = label
        self.flavor = flavor
        self.searchSource = searchsource.NetworkSearchSource(
                repos, self.labelPath, flavor)
        self.macros = macros.Macros(ignoreUnknown=lightInstance)
        self.world = GroupSearchSourceTroveSet(self.searchSource)
        self.g = troveset.OperationGraph()

        baseMacros = loadMacros(cfg.defaultMacros)
        self.macros.update(baseMacros)
        for key in cfg.macros:
            self.macros._override(key, cfg['macros'][key])
        self.macros.name = self.name
        self.macros.version = self.version
        if '.' in self.version:
            self.macros.major_version = '.'.join(self.version.split('.')[0:2])
        else:
            self.macros.major_version = self.version
        if extraMacros:
            self.macros.update(extraMacros)

    def _realizeGraph(self, cache, callback):
        data = GroupActionData(troveCache = GroupSetTroveCache(self, cache),
                               groupRecipe = self)
        self.g.realize(data)

    def getLabelPath(self):
        return self.labelPath

    def getSearchFlavor(self):
        return self.flavor

    def iterReplaceSpecs(self):
        return []

    def getResolveTroveSpecs(self):
        return []

    def getChildGroups(self, groupName = None):
        return []

    def getGroupMap(self, groupName = None):
        return {}

    def _getSearchSource(self):
        return self.troveSource

    def getSearchPath(self):
        return [ ]

    def writeDotGraph(self, path):
        self.g.generateDotFile(path, edgeFormatFn = lambda a,b,c: c)

    def Group(self, ts, checkPathConflicts = True):
        return ts._createGroup(self.name,
                               checkPathConflicts = checkPathConflicts)

    def Repository(self, labelList, flavor):
        if type(labelList) == tuple:
            labelList = list(tuple)
        elif type(labelList) != list:
            labelList = [ labelList ]

        for i, label in enumerate(labelList):
            if type(label) == str:
                labelList[i] = versions.Label(label)
            elif not isinstance(label, versions.Label):
                raise CookError("String label or Label object expected")

        if type(flavor) == str:
            flavor = deps.parseFlavor(flavor)

        searchSource = searchsource.NetworkSearchSource(
                                            self.repos, labelList, flavor)
        return GroupSearchSourceTroveSet(searchSource, graph = self.g)

    def SearchPath(self, *troveSets):
        return GroupSearchPathTroveSet(troveSets, graph = self.g)

from conary.build.packagerecipe import BaseRequiresRecipe
exec defaultrecipes.GroupSetRecipe
<|endoftext|>


<commit_before>from flask import jsonify
import requests, operator, math, json, csv, os
import matplotlib.pyplot as plt

from sklearn.tree import DecisionTreeRegressor as DTR
from sklearn.ensemble import RandomForestRegressor as RFR
from sklearn.ensemble import ExtraTreesRegressor as ETR
from sklearn.svm import SVR
from sklearn.ensemble import AdaBoostRegressor as ABR
from sklearn.linear_model import ElasticNet as EN
from sklearn.multioutput import MultiOutputRegressor as MOR
from sklearn.neural_network import MLPRegressor as MLPR

from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score as CVS
from sklearn.model_selection import learning_curve

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

from sklearn.feature_extraction import DictVectorizer

from sklearn.externals import joblib

import pandas as pd
import numpy as np

from collections import Counter
import time

def hello():
    return 'hello machines'

enc_samples = []
y = []
# Read data from the csv
# Place it into a dataframe
def prepare_data(as_generator=False):
    filename = 'austen-sense-400-1N1S1L2U2L(2017-01-07 23:41:04.780)'
    data = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename + '.csv')
    X = pd.DataFrame(data['sample_doc'])
    y = pd.DataFrame(data[data.columns[1:400]])
    # print 'before: ',X.shape

    filename2 = 'belles_lettres-brown-400-1N1S1L2U2L(2017-01-17 17:30:39.071)'
    data2 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename2 + '.csv')
    A = pd.DataFrame(data2['sample_doc'])
    b = pd.DataFrame(data2[data2.columns[1:400]])
    # print A, b

    filename3 = 'learned-brown-400-1N1S1L2U2L(2017-01-15 20:45:24.877)'
    data3 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename3 + '.csv')
    C = pd.DataFrame(data3['sample_doc'])
    d = pd.DataFrame(data3[data3.columns[1:400]])

    filename4 = 'news-brown-400-1N1S1L2U2L(2017-01-14 17:57:42.849)'
    data4 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename4 + '.csv')
    E = pd.DataFrame(data4['sample_doc'])
    f = pd.DataFrame(data4[data4.columns[1:400]])

    X = X.append(A, ignore_index=True)
    y = y.append(b, ignore_index=True)
    X = X.append(C, ignore_index=True)
    y = y.append(d, ignore_index=True)
    X = X.append(E, ignore_index=True)
    y = y.append(f, ignore_index=True)

    # print 'after: ',X.shape

    le = LabelEncoder()
    enc = OneHotEncoder()

    X_array = np.ravel(X)

    # Initialize DictVectorizer and an empty sparse_matrix to store vectors
    vectorizer = DictVectorizer(sparse=False)
    sparse_matrix = []
    # Find unique words, use them as encoding labels
    for sample in X_array:
        words = sample.split(' ')
        counted_sample = Counter()
        counted_sample.update(words)
        sparse_matrix.append(dict(counted_sample))

    # X is the list of 'fit_transformed' vectors
    X = vectorizer.fit_transform(sparse_matrix)
    # for datapoint in X:
    #     print max(datapoint)
    # print X
    # TODO: Return encoded data and labels
    if not as_generator:
        return 'data prepared'
    else:
        return X, y

# Do a shufflesplit or other cross-validation
# Train a classifier on the data and labels
def train_model():
    X,y = prepare_data(True)
    # print y.shape
    # y = y[y.columns[30:90]]
    y = y[y.columns[30:33]]

    print 'this should be y', y
    # print X[:1000]
    split = ShuffleSplit(n_splits=1, test_size=0.09, random_state=42)
    t_1 = time.clock()
    # Initialize model parameters
    estimator = DTR(max_features=1.0, max_depth=18, random_state=12, splitter='random', min_samples_split=.0006, presort=True)

    estimator4 = ETR(n_estimators=100, max_features=0.66, random_state=12, n_jobs=-1, bootstrap=True)

    # Investigate parameters and relation to null output
    estimator7 = MLPR(solver='sgd', max_iter=900, verbose=True, early_stopping=True, hidden_layer_sizes=(3,3), tol=1e-9, alpha=1e-9, warm_start=True)
    # MOR multioutput regression!
    estimator8 = MOR(estimator, n_jobs=-1)

    # scorer = CVS(reg, X[:1000], y[:1000], cv=split, pre_dispatch=3)
    # for score in scorer:
        # print score

    def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None, n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
        """
        Generate a simple plot of the test and training learning curve.

        Parameters
        ----------
        estimator : object type that implements the "fit" and "predict" methods
            An object of that type which is cloned for each validation.

        title : string
            Title for the chart.

        X : array-like, shape (n_samples, n_features)
            Training vector, where n_samples is the number of samples and
            n_features is the number of features.

        y : array-like, shape (n_samples) or (n_samples, n_features), optional
            Target relative to X for classification or regression;
            None for unsupervised learning.

        ylim : tuple, shape (ymin, ymax), optional
            Defines minimum and maximum yvalues plotted.

        cv : int, cross-validation generator or an iterable, optional
            Determines the cross-validation splitting strategy.
            Possible inputs for cv are:
              - None, to use the default 3-fold cross-validation,
              - integer, to specify the number of folds.
              - An object to be used as a cross-validation generator.
              - An iterable yielding train/test splits.

            For integer/None inputs, if ``y`` is binary or multiclass,
            :class:`StratifiedKFold` used. If the estimator is not a classifier
            or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

            Refer :ref:`User Guide <cross_validation>` for the various
            cross-validators that can be used here.

        n_jobs : integer, optional
            Number of jobs to run in parallel (default 1).
        """
        plt.figure()
        plt.title(title)
        if ylim is not None:
            plt.ylim(*ylim)
        plt.xlabel("Training examples")
        plt.ylabel("Score")
        train_sizes, train_scores, test_scores = learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
        train_scores_mean = np.mean(train_scores, axis=1)
        train_scores_std = np.std(train_scores, axis=1)
        test_scores_mean = np.mean(test_scores, axis=1)
        test_scores_std = np.std(test_scores, axis=1)
        plt.grid()

        plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                         train_scores_mean + train_scores_std, alpha=0.1,
                         color="r")
        plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                         test_scores_mean + test_scores_std, alpha=0.1, color="g")
        plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
                 label="Training score")
        plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
                 label="Cross-validation score")
        print test_scores_mean
        plt.legend(loc="best")
        return plt

    # title = "Learning Curves (RFR)"
    # plot_learning_curve(estimator2, title, X[:2000], y[:2000], (0.1, 1.01), split, n_jobs=1)
    # plt.show()
    # Optional: Run plot_learning_curve to generate learning curves for models. Relocate this code elsewhere to improve readability.
    title = "Learning Curves (DTR(depth 18, 1.0 features, random splits, min split .0006, presort)+MOR, 24.5k samples, 0.09 test, 3 columns)"
    # plot_learning_curve(estimator8, title, X[:24500], y[:24500], (-0.1, 1.01), n_jobs=-1, cv=split)
    # plt.show()
    # TODO: Rework this train_model function to focus on training and saving models
    # Fit the model to some data
    estimator8.fit(X,y)
    # Dump the model to persist it.
    joblib.dump(estimator8, title+'.pkl')
    t_2 = time.clock()
    print "Total time: ", t_2-t_1
    return 'training model'

# Measure model performance with CV
def validate_model():
    # Retrieve a model from a .pkl file with joblib.load()
    # Use an unseen dataset to score it
    # Measuure amount of time for predictions
    X,y = prepare_data(True)
    title = 'DTR(depth 18, 1.0 features, random splits, min split .0006, presort)+MOR, 24.5k samples, 0.09 test, 3 columns.pkl'
    estimator = joblib.load(title)
    estimator.fit(X,y)
    joblib.dump(estimator, title)
    print 'prediction', estimator.predict(X[1])
    print 'actual', y[1]
    return 'validating model'
<commit_msg>refactor: Reorganize model training and validation<commit_after>from flask import jsonify
import requests, operator, math, json, csv, os
import matplotlib.pyplot as plt

from sklearn.tree import DecisionTreeRegressor as DTR
from sklearn.ensemble import RandomForestRegressor as RFR
from sklearn.ensemble import ExtraTreesRegressor as ETR
from sklearn.svm import SVR
from sklearn.ensemble import AdaBoostRegressor as ABR
from sklearn.linear_model import ElasticNet as EN
from sklearn.multioutput import MultiOutputRegressor as MOR
from sklearn.neural_network import MLPRegressor as MLPR

from sklearn.model_selection import ShuffleSplit
from sklearn.model_selection import cross_val_score as CVS
from sklearn.model_selection import learning_curve

from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder

from sklearn.feature_extraction import DictVectorizer

from sklearn.externals import joblib

import pandas as pd
import numpy as np

from collections import Counter
import time

def hello():
    return 'hello machines'

enc_samples = []
y = []
# Read data from the csv
# Place it into a dataframe
def prepare_data(as_generator=False, text_encoder=False):
    filename = 'austen-sense-400-1N1S1L2U2L(2017-01-07 23:41:04.780)'
    data = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename + '.csv')
    X = pd.DataFrame(data['sample_doc'])
    y = pd.DataFrame(data[data.columns[1:400]])
    # print 'before: ',X.shape

    filename2 = 'belles_lettres-brown-400-1N1S1L2U2L(2017-01-17 17:30:39.071)'
    data2 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename2 + '.csv')
    A = pd.DataFrame(data2['sample_doc'])
    b = pd.DataFrame(data2[data2.columns[1:400]])
    # print A, b

    filename3 = 'learned-brown-400-1N1S1L2U2L(2017-01-15 20:45:24.877)'
    data3 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename3 + '.csv')
    C = pd.DataFrame(data3['sample_doc'])
    d = pd.DataFrame(data3[data3.columns[1:400]])

    filename4 = 'news-brown-400-1N1S1L2U2L(2017-01-14 17:57:42.849)'
    data4 = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename4 + '.csv')
    E = pd.DataFrame(data4['sample_doc'])
    f = pd.DataFrame(data4[data4.columns[1:400]])

    X = X.append(A, ignore_index=True)
    y = y.append(b, ignore_index=True)
    X = X.append(C, ignore_index=True)
    y = y.append(d, ignore_index=True)
    X = X.append(E, ignore_index=True)
    y = y.append(f, ignore_index=True)

    # print 'after: ',X.shape

    le = LabelEncoder()
    enc = OneHotEncoder()

    X_array = np.ravel(X)

    # Initialize DictVectorizer and an empty sparse_matrix to store vectors
    vectorizer = DictVectorizer(sparse=False)
    sparse_matrix = []
    # Find unique words, use them as encoding labels
    for sample in X_array:
        words = sample.split(' ')
        counted_sample = Counter()
        counted_sample.update(words)
        sparse_matrix.append(dict(counted_sample))

    # X is the list of 'fit_transformed' vectors
    # print sparse_matrix
    X = vectorizer.fit_transform(sparse_matrix)
    # for datapoint in X:
    #     print max(datapoint)
    # print X
    # TODO: Return encoded data and labels
    if not as_generator:
        return 'data prepared'
    else:
        if text_encoder:
            return X, y, vectorizer
        return X, y

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None, n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
    """
    Generate a simple plot of the test and training learning curve.

    Parameters
    ----------
    estimator : object type that implements the "fit" and "predict" methods
        An object of that type which is cloned for each validation.

    title : string
        Title for the chart.

    X : array-like, shape (n_samples, n_features)
        Training vector, where n_samples is the number of samples and
        n_features is the number of features.

    y : array-like, shape (n_samples) or (n_samples, n_features), optional
        Target relative to X for classification or regression;
        None for unsupervised learning.

    ylim : tuple, shape (ymin, ymax), optional
        Defines minimum and maximum yvalues plotted.

    cv : int, cross-validation generator or an iterable, optional
        Determines the cross-validation splitting strategy.
        Possible inputs for cv are:
          - None, to use the default 3-fold cross-validation,
          - integer, to specify the number of folds.
          - An object to be used as a cross-validation generator.
          - An iterable yielding train/test splits.

        For integer/None inputs, if ``y`` is binary or multiclass,
        :class:`StratifiedKFold` used. If the estimator is not a classifier
        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.

        Refer :ref:`User Guide <cross_validation>` for the various
        cross-validators that can be used here.

    n_jobs : integer, optional
        Number of jobs to run in parallel (default 1).
    """
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Cross-validation score")
    print test_scores_mean
    plt.legend(loc="best")
    return plt

# Do a shufflesplit or other cross-validation
# Train a classifier on the data and labels
def train_model():
    X,y = prepare_data(True)
    # print y.shape
    # y = y[y.columns[30:90]]
    y = y[y.columns[30:33]]

    # print 'this should be y', y
    # print X[:1000]
    split = ShuffleSplit(n_splits=1, test_size=0.09, random_state=42)
    t_1 = time.clock()
    # Initialize model parameters
    estimator = DTR(max_features=1.0, max_depth=18, random_state=12, splitter='random', min_samples_split=.0006, presort=True)

    estimator4 = ETR(n_estimators=100, max_features=0.66, random_state=12, n_jobs=-1, bootstrap=True)

    # Investigate parameters and relation to null output
    estimator7 = MLPR(solver='sgd', max_iter=900, verbose=True, early_stopping=True, hidden_layer_sizes=(3,3), tol=1e-9, alpha=1e-9, warm_start=True)
    # MOR multioutput regression!
    estimator8 = MOR(estimator, n_jobs=-1)

    # Optional: Run plot_learning_curve to generate learning curves for models. Relocate this code elsewhere to improve readability.
    title = "Learning Curves (DTR(depth 18, 1.0 features, random splits, min split .0006, presort)+MOR, 24.5k samples, 3 columns)"
    # plot_learning_curve(estimator8, title, X[:24500], y[:24500], (-0.1, 1.01), n_jobs=-1, cv=split)
    # plt.show()
    # TODO: Rework this train_model function to focus on training and saving models
    # Fit the model to some data
    estimator8.fit(X[:24500],y[:24500])
    # Dump the model to persist it.
    joblib.dump(estimator8, title[16:]+'.pkl')
    t_2 = time.clock()
    print "Total time: ", t_2-t_1
    return 'training model'

# Measure model performance with CV
def validate_model():
    # The vectorizer we retrieve here must be the same as that used to train the model we're validating.
    _,_,vectorizer = prepare_data(True, True)

    # Retrieve a model from a .pkl file with joblib.load()
    title = '(DTR(depth 18, 1.0 features, random splits, min split .0006, presort)+MOR, 10k samples, 3 columns).pkl'
    estimator = joblib.load(title)

    # Use an unseen dataset to score it
    filename = 'science_fiction-brown-400-1N1S1L2U2L(2017-01-17 17:30:39.071)'
    data = pd.read_csv(os.path.dirname(__file__) + '/../../data/' + filename + '.csv')
    X = pd.DataFrame(data['sample_doc'])
    y = pd.DataFrame(data[data.columns[30:33]])
    X_array = np.ravel(X)
    sparse_matrix = []

    for sample in X_array:
        words = sample.split(' ')
        counted_sample = Counter()
        counted_sample.update(words)
        sparse_matrix.append(dict(counted_sample))

    X = vectorizer.transform(sparse_matrix)

    # Measure amount of time for predictions
    t_1 = time.clock()
    score = estimator.score(X,y)
    t_2 = time.clock()
    print 'model score: ', score, 'time required for',y.shape,'predictions: ',t_2-t_1
    return 'validating model'
<|endoftext|>