<commit_before># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""mel conversion ops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.signal import shape_ops
from tensorflow.python.util.tf_export import tf_export

# mel spectrum constants.
_MEL_BREAK_FREQUENCY_HERTZ = 700.0
_MEL_HIGH_FREQUENCY_Q = 1127.0

def _mel_to_hertz(mel_values, name=None):
  """Converts frequencies in `mel_values` from the mel scale to linear scale.

  Args:
    mel_values: A `Tensor` of frequencies in the mel scale.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of the same shape and type as `mel_values` containing linear
    scale frequencies in Hertz.
  """
  with ops.name_scope(name, 'mel_to_hertz', [mel_values]):
    mel_values = ops.convert_to_tensor(mel_values)
    return _MEL_BREAK_FREQUENCY_HERTZ * (
        math_ops.exp(mel_values / _MEL_HIGH_FREQUENCY_Q) - 1.0
    )

def _hertz_to_mel(frequencies_hertz, name=None):
  """Converts frequencies in `frequencies_hertz` in Hertz to the mel scale.

  Args:
    frequencies_hertz: A `Tensor` of frequencies in Hertz.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of the same shape and type of `frequencies_hertz` containing
    frequencies in the mel scale.
  """
  with ops.name_scope(name, 'hertz_to_mel', [frequencies_hertz]):
    frequencies_hertz = ops.convert_to_tensor(frequencies_hertz)
    return _MEL_HIGH_FREQUENCY_Q * math_ops.log(
        1.0 + (frequencies_hertz / _MEL_BREAK_FREQUENCY_HERTZ))

def _validate_arguments(num_mel_bins, sample_rate,
                        lower_edge_hertz, upper_edge_hertz, dtype):
  """Checks the inputs to linear_to_mel_weight_matrix."""
  if num_mel_bins <= 0:
    raise ValueError('num_mel_bins must be positive. Got: %s' % num_mel_bins)
  if lower_edge_hertz < 0.0:
    raise ValueError('lower_edge_hertz must be non-negative. Got: %s' %
                     lower_edge_hertz)
  if lower_edge_hertz >= upper_edge_hertz:
    raise ValueError('lower_edge_hertz %.1f >= upper_edge_hertz %.1f' %
                     (lower_edge_hertz, upper_edge_hertz))
  if not isinstance(sample_rate, ops.Tensor):
    if sample_rate <= 0.0:
      raise ValueError('sample_rate must be positive. Got: %s' % sample_rate)
    if upper_edge_hertz > sample_rate / 2:
      raise ValueError('upper_edge_hertz must not be larger than the Nyquist '
                       'frequency (sample_rate / 2). Got %s for sample_rate: %s'
                       % (upper_edge_hertz, sample_rate))
  if not dtype.is_floating:
    raise ValueError('dtype must be a floating point type. Got: %s' % dtype)

@tf_export('signal.linear_to_mel_weight_matrix')
def linear_to_mel_weight_matrix(num_mel_bins=20,
                                num_spectrogram_bins=129,
                                sample_rate=8000,
                                lower_edge_hertz=125.0,
                                upper_edge_hertz=3800.0,
                                dtype=dtypes.float32,
                                name=None):
  """Returns a matrix to warp linear scale spectrograms to the [mel scale][mel].

  Returns a weight matrix that can be used to re-weight a `Tensor` containing
  `num_spectrogram_bins` linearly sampled frequency information from
  `[0, sample_rate / 2]` into `num_mel_bins` frequency information from
  `[lower_edge_hertz, upper_edge_hertz]` on the [mel scale][mel].

  This function follows the [Hidden Markov Model Toolkit
  (HTK)](http://htk.eng.cam.ac.uk/) convention, defining the mel scale in
  terms of a frequency in hertz according to the following formula:

      $$\textrm{mel}(f) = 2595 * \textrm{log}_{10}(1 + \frac{f}{700})$$

  In the returned matrix, all the triangles (filterbanks) have a peak value
  of 1.0.

  For example, the returned matrix `A` can be used to right-multiply a
  spectrogram `S` of shape `[frames, num_spectrogram_bins]` of linear
  scale spectrum values (e.g. STFT magnitudes) to generate a "mel spectrogram"
  `M` of shape `[frames, num_mel_bins]`.

      # `S` has shape [frames, num_spectrogram_bins]
      # `M` has shape [frames, num_mel_bins]
      M = tf.matmul(S, A)

  The matrix can be used with `tf.tensordot` to convert an arbitrary rank
  `Tensor` of linear-scale spectral bins into the mel scale.

      # S has shape [..., num_spectrogram_bins].
      # M has shape [..., num_mel_bins].
      M = tf.tensordot(S, A, 1)
      # tf.tensordot does not support shape inference for this case yet.
      M.set_shape(S.shape[:-1].concatenate(A.shape[-1:]))

  Args:
    num_mel_bins: Python int. How many bands in the resulting mel spectrum.
    num_spectrogram_bins: An integer `Tensor`. How many bins there are in the
      source spectrogram data, which is understood to be `fft_size // 2 + 1`,
      i.e. the spectrogram only contains the nonredundant FFT bins.
    sample_rate: An integer or float `Tensor`. Samples per second of the input
      signal used to create the spectrogram. Used to figure out the frequencies
      corresponding to each spectrogram bin, which dictates how they are mapped
      into the mel scale.
    lower_edge_hertz: Python float. Lower bound on the frequencies to be
      included in the mel spectrum. This corresponds to the lower edge of the
      lowest triangular band.
    upper_edge_hertz: Python float. The desired top edge of the highest
      frequency band.
    dtype: The `DType` of the result matrix. Must be a floating point type.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of shape `[num_spectrogram_bins, num_mel_bins]`.

  Raises:
    ValueError: If `num_mel_bins`/`num_spectrogram_bins`/`sample_rate` are not
      positive, `lower_edge_hertz` is negative, frequency edges are incorrectly
      ordered, `upper_edge_hertz` is larger than the Nyquist frequency.

  [mel]: https://en.wikipedia.org/wiki/Mel_scale
  """
  with ops.name_scope(name, 'linear_to_mel_weight_matrix') as name:
    # Convert Tensor `sample_rate` to float, if possible.
    if isinstance(sample_rate, ops.Tensor):
      maybe_const_val = tensor_util.constant_value(sample_rate)
      if maybe_const_val is not None:
        sample_rate = maybe_const_val

    # Note: As num_spectrogram_bins is passed to `math_ops.linspace`
    # and the validation is already done in linspace (both in shape function
    # and in kernel), there is no need to validate num_spectrogram_bins here.
    _validate_arguments(num_mel_bins, sample_rate,
                        lower_edge_hertz, upper_edge_hertz, dtype)

    # This function can be constant folded by graph optimization since there are
    # no Tensor inputs.
    sample_rate = math_ops.cast(
        sample_rate, dtype, name='sample_rate')
    lower_edge_hertz = ops.convert_to_tensor(
        lower_edge_hertz, dtype, name='lower_edge_hertz')
    upper_edge_hertz = ops.convert_to_tensor(
        upper_edge_hertz, dtype, name='upper_edge_hertz')
    zero = ops.convert_to_tensor(0.0, dtype)

    # HTK excludes the spectrogram DC bin.
    bands_to_zero = 1
    nyquist_hertz = sample_rate / 2.0
    linear_frequencies = math_ops.linspace(
        zero, nyquist_hertz, num_spectrogram_bins)[bands_to_zero:]
    spectrogram_bins_mel = array_ops.expand_dims(
        _hertz_to_mel(linear_frequencies), 1)

    # Compute num_mel_bins triples of (lower_edge, center, upper_edge). The
    # center of each band is the lower and upper edge of the adjacent bands.
    # Accordingly, we divide [lower_edge_hertz, upper_edge_hertz] into
    # num_mel_bins + 2 pieces.
    band_edges_mel = shape_ops.frame(
        math_ops.linspace(_hertz_to_mel(lower_edge_hertz),
                          _hertz_to_mel(upper_edge_hertz),
                          num_mel_bins + 2), frame_length=3, frame_step=1)

    # Split the triples up and reshape them into [1, num_mel_bins] tensors.
    lower_edge_mel, center_mel, upper_edge_mel = tuple(array_ops.reshape(
        t, [1, num_mel_bins]) for t in array_ops.split(
            band_edges_mel, 3, axis=1))

    # Calculate lower and upper slopes for every spectrogram bin.
    # Line segments are linear in the mel domain, not Hertz.
    lower_slopes = (spectrogram_bins_mel - lower_edge_mel) / (
        center_mel - lower_edge_mel)
    upper_slopes = (upper_edge_mel - spectrogram_bins_mel) / (
        upper_edge_mel - center_mel)

    # Intersect the line segments with each other and zero.
    mel_weights_matrix = math_ops.maximum(
        zero, math_ops.minimum(lower_slopes, upper_slopes))

    # Re-add the zeroed lower bins we sliced out above.
    return array_ops.pad(
        mel_weights_matrix, [[bands_to_zero, 0], [0, 0]], name=name)
<commit_msg>Fix for //tensorflow/python/kernel_tests/signal:mel_ops_test<commit_after># Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""mel conversion ops."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import math_ops
from tensorflow.python.ops.signal import shape_ops
from tensorflow.python.util.tf_export import tf_export

# mel spectrum constants.
_MEL_BREAK_FREQUENCY_HERTZ = 700.0
_MEL_HIGH_FREQUENCY_Q = 1127.0

def _mel_to_hertz(mel_values, name=None):
  """Converts frequencies in `mel_values` from the mel scale to linear scale.

  Args:
    mel_values: A `Tensor` of frequencies in the mel scale.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of the same shape and type as `mel_values` containing linear
    scale frequencies in Hertz.
  """
  with ops.name_scope(name, 'mel_to_hertz', [mel_values]):
    mel_values = ops.convert_to_tensor(mel_values)
    return _MEL_BREAK_FREQUENCY_HERTZ * math_ops.expm1(mel_values / _MEL_HIGH_FREQUENCY_Q)

def _hertz_to_mel(frequencies_hertz, name=None):
  """Converts frequencies in `frequencies_hertz` in Hertz to the mel scale.

  Args:
    frequencies_hertz: A `Tensor` of frequencies in Hertz.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of the same shape and type of `frequencies_hertz` containing
    frequencies in the mel scale.
  """
  with ops.name_scope(name, 'hertz_to_mel', [frequencies_hertz]):
    frequencies_hertz = ops.convert_to_tensor(frequencies_hertz)
    return _MEL_HIGH_FREQUENCY_Q * math_ops.log1p(frequencies_hertz / _MEL_BREAK_FREQUENCY_HERTZ)

def _validate_arguments(num_mel_bins, sample_rate,
                        lower_edge_hertz, upper_edge_hertz, dtype):
  """Checks the inputs to linear_to_mel_weight_matrix."""
  if num_mel_bins <= 0:
    raise ValueError('num_mel_bins must be positive. Got: %s' % num_mel_bins)
  if lower_edge_hertz < 0.0:
    raise ValueError('lower_edge_hertz must be non-negative. Got: %s' %
                     lower_edge_hertz)
  if lower_edge_hertz >= upper_edge_hertz:
    raise ValueError('lower_edge_hertz %.1f >= upper_edge_hertz %.1f' %
                     (lower_edge_hertz, upper_edge_hertz))
  if not isinstance(sample_rate, ops.Tensor):
    if sample_rate <= 0.0:
      raise ValueError('sample_rate must be positive. Got: %s' % sample_rate)
    if upper_edge_hertz > sample_rate / 2:
      raise ValueError('upper_edge_hertz must not be larger than the Nyquist '
                       'frequency (sample_rate / 2). Got %s for sample_rate: %s'
                       % (upper_edge_hertz, sample_rate))
  if not dtype.is_floating:
    raise ValueError('dtype must be a floating point type. Got: %s' % dtype)

@tf_export('signal.linear_to_mel_weight_matrix')
def linear_to_mel_weight_matrix(num_mel_bins=20,
                                num_spectrogram_bins=129,
                                sample_rate=8000,
                                lower_edge_hertz=125.0,
                                upper_edge_hertz=3800.0,
                                dtype=dtypes.float32,
                                name=None):
  """Returns a matrix to warp linear scale spectrograms to the [mel scale][mel].

  Returns a weight matrix that can be used to re-weight a `Tensor` containing
  `num_spectrogram_bins` linearly sampled frequency information from
  `[0, sample_rate / 2]` into `num_mel_bins` frequency information from
  `[lower_edge_hertz, upper_edge_hertz]` on the [mel scale][mel].

  This function follows the [Hidden Markov Model Toolkit
  (HTK)](http://htk.eng.cam.ac.uk/) convention, defining the mel scale in
  terms of a frequency in hertz according to the following formula:

      $$\textrm{mel}(f) = 2595 * \textrm{log}_{10}(1 + \frac{f}{700})$$

  In the returned matrix, all the triangles (filterbanks) have a peak value
  of 1.0.

  For example, the returned matrix `A` can be used to right-multiply a
  spectrogram `S` of shape `[frames, num_spectrogram_bins]` of linear
  scale spectrum values (e.g. STFT magnitudes) to generate a "mel spectrogram"
  `M` of shape `[frames, num_mel_bins]`.

      # `S` has shape [frames, num_spectrogram_bins]
      # `M` has shape [frames, num_mel_bins]
      M = tf.matmul(S, A)

  The matrix can be used with `tf.tensordot` to convert an arbitrary rank
  `Tensor` of linear-scale spectral bins into the mel scale.

      # S has shape [..., num_spectrogram_bins].
      # M has shape [..., num_mel_bins].
      M = tf.tensordot(S, A, 1)
      # tf.tensordot does not support shape inference for this case yet.
      M.set_shape(S.shape[:-1].concatenate(A.shape[-1:]))

  Args:
    num_mel_bins: Python int. How many bands in the resulting mel spectrum.
    num_spectrogram_bins: An integer `Tensor`. How many bins there are in the
      source spectrogram data, which is understood to be `fft_size // 2 + 1`,
      i.e. the spectrogram only contains the nonredundant FFT bins.
    sample_rate: An integer or float `Tensor`. Samples per second of the input
      signal used to create the spectrogram. Used to figure out the frequencies
      corresponding to each spectrogram bin, which dictates how they are mapped
      into the mel scale.
    lower_edge_hertz: Python float. Lower bound on the frequencies to be
      included in the mel spectrum. This corresponds to the lower edge of the
      lowest triangular band.
    upper_edge_hertz: Python float. The desired top edge of the highest
      frequency band.
    dtype: The `DType` of the result matrix. Must be a floating point type.
    name: An optional name for the operation.

  Returns:
    A `Tensor` of shape `[num_spectrogram_bins, num_mel_bins]`.

  Raises:
    ValueError: If `num_mel_bins`/`num_spectrogram_bins`/`sample_rate` are not
      positive, `lower_edge_hertz` is negative, frequency edges are incorrectly
      ordered, `upper_edge_hertz` is larger than the Nyquist frequency.

  [mel]: https://en.wikipedia.org/wiki/Mel_scale
  """
  with ops.name_scope(name, 'linear_to_mel_weight_matrix') as name:
    # Convert Tensor `sample_rate` to float, if possible.
    if isinstance(sample_rate, ops.Tensor):
      maybe_const_val = tensor_util.constant_value(sample_rate)
      if maybe_const_val is not None:
        sample_rate = maybe_const_val

    # Note: As num_spectrogram_bins is passed to `math_ops.linspace`
    # and the validation is already done in linspace (both in shape function
    # and in kernel), there is no need to validate num_spectrogram_bins here.
    _validate_arguments(num_mel_bins, sample_rate,
                        lower_edge_hertz, upper_edge_hertz, dtype)

    # This function can be constant folded by graph optimization since there are
    # no Tensor inputs.
    sample_rate = math_ops.cast(
        sample_rate, dtype, name='sample_rate')
    lower_edge_hertz = ops.convert_to_tensor(
        lower_edge_hertz, dtype, name='lower_edge_hertz')
    upper_edge_hertz = ops.convert_to_tensor(
        upper_edge_hertz, dtype, name='upper_edge_hertz')
    zero = ops.convert_to_tensor(0.0, dtype)

    # HTK excludes the spectrogram DC bin.
    bands_to_zero = 1
    nyquist_hertz = sample_rate / 2.0
    linear_frequencies = math_ops.linspace(
        zero, nyquist_hertz, num_spectrogram_bins)[bands_to_zero:]
    spectrogram_bins_mel = array_ops.expand_dims(
        _hertz_to_mel(linear_frequencies), 1)

    # Compute num_mel_bins triples of (lower_edge, center, upper_edge). The
    # center of each band is the lower and upper edge of the adjacent bands.
    # Accordingly, we divide [lower_edge_hertz, upper_edge_hertz] into
    # num_mel_bins + 2 pieces.
    band_edges_mel = shape_ops.frame(
        math_ops.linspace(_hertz_to_mel(lower_edge_hertz),
                          _hertz_to_mel(upper_edge_hertz),
                          num_mel_bins + 2), frame_length=3, frame_step=1)

    # Split the triples up and reshape them into [1, num_mel_bins] tensors.
    lower_edge_mel, center_mel, upper_edge_mel = tuple(array_ops.reshape(
        t, [1, num_mel_bins]) for t in array_ops.split(
            band_edges_mel, 3, axis=1))

    # Calculate lower and upper slopes for every spectrogram bin.
    # Line segments are linear in the mel domain, not Hertz.
    lower_slopes = (spectrogram_bins_mel - lower_edge_mel) / (
        center_mel - lower_edge_mel)
    upper_slopes = (upper_edge_mel - spectrogram_bins_mel) / (
        upper_edge_mel - center_mel)

    # Intersect the line segments with each other and zero.
    mel_weights_matrix = math_ops.maximum(
        zero, math_ops.minimum(lower_slopes, upper_slopes))

    # Re-add the zeroed lower bins we sliced out above.
    return array_ops.pad(
        mel_weights_matrix, [[bands_to_zero, 0], [0, 0]], name=name)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Released as open source by Gabriel Caudrelier
# Developed by Gabriel Caudrelier, gabriel dot caudrelier at gmail dot com
# https://github.com/metrodango/pip3line
# Released under AGPL see LICENSE for more information

# This class is just an example on how to communicate with the pip3line gui via sockets
# The method process expect a reponse from the pip3line server

import socket
import base64

class Pip3line:
	DEFAULT_TCP = ('127.0.0.1',45632)
	DEFAULT_UNIX_SOCKET="/tmp/pip3lineMass"
	DEFAULT_WINDOWS_PIPE="pip3lineMass"

	def __init__(self):
		self.DEFAULT_TCP = ('127.0.0.1',45632)
		self.DEFAULT_UNIX_SOCKET="/tmp/pip3lineMass"
		self.DEFAULT_WINDOWS_PIPE="pip3lineMass"
		self.tcpparam = self.DEFAULT_TCP
		self.base64inUse = False
		self.conn = None
	def __del__(self):
		if self.conn != None:
			self.conn.close()

	def setup(self,tcp=None, pipe=None):
		if (tcp != None):
			self.tcpparam = tcp
			if self.connect():
				print("TCP connection opened on",str(tcp))
		elif (pipe != None):
			if os.path.exists(pipe):
			        self.conn = open(pipe, 'r')
			else:
				print("no Unix Socket/Named Pipe for  " + pipe)

			print("Pipe connection opened on",pipe)
	def connect(self):
		try:
			self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		except socket.error as msg:
			self.networkError(msg)
			self.conn = None
			return False

		try:
			self.conn.connect(self.tcpparam)
		except socket.error as msg:
			self.networkError(msg)
			self.conn.close()
			self.conn = None
			return False

		return True

	def __sending(self,data):
		if self.base64inUse:
			data = base64.b64encode(data)

		self.conn.send(data)
		self.conn.send(b'\n')
		endBlock = False
		rdata = b''
		while not endBlock:
			rdata += self.conn.recv(1024)
			if rdata.count(b'\n') > 0:
				rdata = rdata.split(b'\n')[0]
				endBlock = True

		if rdata != b'' and self.base64inUse:
			rdata = base64.b64decode(rdata)

		return rdata

	def process(self,data):
		if isinstance(data, b''):
			print("Need array of bytes")
			return b''

		if (self.conn == None):
			if not self.connect():
				return b''

		try:
			rdata = self.__sending(data)
			if rdata==b'':
				print("Connection died (retrying)")
				if self.connect():
					rdata = self.__sending(data)
					if rdata==b'':
						self.networkError("Server down?")
		except socket.error as msg:
			self.networkError(msg)
			if self.connect():
				rdata = self.__sending(data)

		return rdata
	def networkError(self,msg):
		print(msg,str(self.tcpparam))

if __name__ == "__main__":
	print("Running tests")
<commit_msg>Codacity style minor issue<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Released as open source by Gabriel Caudrelier
# Developed by Gabriel Caudrelier, gabriel dot caudrelier at gmail dot com
# https://github.com/metrodango/pip3line
# Released under AGPL see LICENSE for more information

# This class is just an example on how to communicate with the pip3line gui via sockets
# The method process expect a reponse from the pip3line server

import socket
import base64

class Pip3line:
	DEFAULT_TCP = ('127.0.0.1',45632)
	DEFAULT_UNIX_SOCKET="/tmp/pip3lineMass"
	DEFAULT_WINDOWS_PIPE="pip3lineMass"

	def __init__(self):
		self.DEFAULT_TCP = ('127.0.0.1',45632)
		self.DEFAULT_UNIX_SOCKET="/tmp/pip3lineMass"
		self.DEFAULT_WINDOWS_PIPE="pip3lineMass"
		self.tcpparam = self.DEFAULT_TCP
		self.base64inUse = False
		self.conn = None
	def __del__(self):
		if self.conn != None:
			self.conn.close()

	def setup(self,tcp=None, pipe=None):
		if (tcp != None):
			self.tcpparam = tcp
			if self.connect():
				print("TCP connection opened on",str(tcp))
		elif (pipe != None):
			if os.path.exists(pipe):
			        self.conn = open(pipe, 'r')
			else:
				print("no Unix Socket/Named Pipe for  " + pipe)

			print("Pipe connection opened on",pipe)
	def connect(self):
		try:
			self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		except socket.error as msg:
			self.networkError(msg)
			self.conn = None
			return False

		try:
			self.conn.connect(self.tcpparam)
		except socket.error as msg:
			self.networkError(msg)
			self.conn.close()
			self.conn = None
			return False

		return True

	def __sending(self,data):
		if self.base64inUse:
			data = base64.b64encode(data)

		self.conn.send(data)
		self.conn.send(b'\n')
		endBlock = False
		rdata = b''
		while not endBlock:
			rdata += self.conn.recv(1024)
			if rdata.count(b'\n') > 0:
				rdata = rdata.split(b'\n')[0]
				endBlock = True

		if rdata != b'' and self.base64inUse:
			rdata = base64.b64decode(rdata)

		return rdata

	def process(self,data):
		if isinstance(data, b''):
			print("Need array of bytes")
			return b''

		if (self.conn == None):
			if not self.connect():
				return b''

		try:
			rdata = self.__sending(data)
			if rdata==b'':
				print("Connection died (retrying)")
				if self.connect():
					rdata = self.__sending(data)
					if rdata==b'':
						self.networkError("Server down?")
		except socket.error as msg:
			self.networkError(msg)
			if self.connect():
				rdata = self.__sending(data)

		return rdata
	def networkError(self,msg):
		print(msg,str(self.tcpparam))

if __name__ == "__main__":
	print("Running tests")
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import logging

from django.db import models as django_models
from django.http import Http404
import django_filters
from django_fsm import TransitionNotAllowed
from rest_framework import filters as rf_filter
from rest_framework import mixins
from rest_framework import permissions, status
from rest_framework import viewsets
from rest_framework.exceptions import PermissionDenied
from rest_framework.response import Response
from rest_framework_extensions.decorators import action, link

from nodeconductor.cloud.models import Cloud, Flavor
from nodeconductor.core import mixins as core_mixins
from nodeconductor.core import models as core_models
from nodeconductor.core import viewsets as core_viewsets
from nodeconductor.iaas import models
from nodeconductor.iaas import serializers
from nodeconductor.structure import filters
from nodeconductor.structure.filters import filter_queryset_for_user
from nodeconductor.structure.models import ProjectRole

logger = logging.getLogger(__name__)

class InstanceFilter(django_filters.FilterSet):
    project_group = django_filters.CharFilter(
        name='project__project_groups__name',
        distinct=True,
        lookup_type='icontains',
    )
    project = django_filters.CharFilter(
        name='project__name',
        distinct=True,
        lookup_type='icontains',
    )

    customer_name = django_filters.CharFilter(
        name='project__customer__name',
        distinct=True,
        lookup_type='icontains',
    )

    hostname = django_filters.CharFilter(lookup_type='icontains')
    state = django_filters.CharFilter()

    class Meta(object):
        model = models.Instance
        fields = [
            'hostname',
            'customer_name',
            'state',
            'project',
            'project_group',
        ]
        order_by = [
            'hostname',
            'state',
        ]

class InstanceViewSet(mixins.CreateModelMixin,
                      mixins.RetrieveModelMixin,
                      core_mixins.ListModelMixin,
                      core_mixins.UpdateOnlyModelMixin,
                      viewsets.GenericViewSet):
    """List of VM instances that are accessible by this user.

    VM instances are launched in clouds, whereas the instance may belong to one cloud only, and the cloud may have
    multiple VM instances.

    VM instance may be in one of the following states:
     - creating
     - created
     - starting
     - started
     - stopping
     - stopped
     - restarting
     - deleting
     - deleted
     - erred

    Staff members can list all available VM instances in any cloud.
    Customer owners can list all VM instances in all the clouds that belong to any of the customers they own.
    Project administrators can list all VM instances, create new instances and start/stop/restart instances in all the
    clouds that are connected to any of the projects they are administrators in.
    Project managers can list all VM instances in all the clouds that are connected to any of the projects they are
    managers in.
    """

    queryset = models.Instance.objects.all()
    serializer_class = serializers.InstanceSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter, rf_filter.DjangoFilterBackend)
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    filter_class = InstanceFilter

    def get_serializer_class(self):
        if self.request.method in ('POST', 'PUT', 'PATCH'):
            return serializers.InstanceCreateSerializer

        return super(InstanceViewSet, self).get_serializer_class()

    def get_serializer_context(self):
        """
        Extra context provided to the serializer class.
        """
        context = super(InstanceViewSet, self).get_serializer_context()
        context['user'] = self.request.user
        return context

    def get_queryset(self):
        queryset = super(InstanceViewSet, self).get_queryset()
        queryset = queryset.exclude(state=models.Instance.States.DELETED)
        return queryset

    def _schedule_transition(self, request, uuid, operation, **kwargs):
        # Importing here to avoid circular imports
        from nodeconductor.iaas import tasks
        # XXX: this should be testing for actions/role pairs as well
        instance = filter_queryset_for_user(models.Instance.objects.filter(uuid=uuid), request.user).first()

        if instance is None:
            raise Http404()

        is_admin = instance.project.roles.filter(permission_group__user=request.user,
                                                 role_type=ProjectRole.ADMINISTRATOR).exists()
        if not is_admin:
            raise PermissionDenied()

        supported_operations = {
            # code: (scheduled_celery_task, instance_marker_state)
            'start': (instance.schedule_starting, tasks.schedule_starting),
            'stop': (instance.schedule_stopping, tasks.schedule_stopping),
            'destroy': (instance.schedule_deletion, tasks.schedule_deleting),
            'resize': (instance.schedule_resizing, tasks.schedule_resizing),
        }

        logger.info('Scheduling provisioning instance with uuid %s', uuid)
        processing_task = supported_operations[operation][1]
        instance_schedule_transition = supported_operations[operation][0]
        try:
            instance_schedule_transition()
            instance.save()
            processing_task.delay(uuid, **kwargs)
        except TransitionNotAllowed:
            return Response({'status': 'Performing %s operation from instance state \'%s\' is not allowed'
                            % (operation, instance.get_state_display())},
                            status=status.HTTP_409_CONFLICT)

        return Response({'status': '%s was scheduled' % operation})

    @action()
    def stop(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'stop')

    @action()
    def start(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'start')

    def destroy(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'destroy')

    @action()
    def resize(self, request, uuid=None):
        try:
            instance = models.Instance.objects.get(uuid=uuid)
        except models.Instance.DoesNotExist:
            raise Http404()

        try:
            flavor_uuid = request.DATA['flavor']
        except KeyError:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        instance_cloud = instance.flavor.cloud

        new_flavor = Flavor.objects.filter(cloud=instance_cloud, uuid=flavor_uuid)

        if new_flavor.exists():
            return self._schedule_transition(request, uuid, 'resize', new_flavor=flavor_uuid)

        return Response({'status': "New flavor is not within the same cloud"},
                        status=status.HTTP_400_BAD_REQUEST)

class TemplateViewSet(core_viewsets.UpdateModelViewSet):
    """List of VM templates that are accessible by this user.

    VM template is a description of a system installed on VM instances: OS, disk partition etc.

    VM template is not to be confused with VM instance flavor -- template is a definition of a system to be installed (set of software) whereas flavor is a set of virtual hardware parameters.

    VM templates are connected to clouds, whereas the template may belong to one cloud only, and the cloud may have multiple VM templates.

    Staff members can list all available VM templates in any cloud and create new templates.

    Customer owners can list all VM templates in all the clouds that belong to any of the customers they own.

    Project administrators can list all VM templates and create new VM instances using these templates in all the clouds that are connected to any of the projects they are administrators in.

    Project managers can list all VM templates in all the clouds that are connected to any of the projects they are managers in.

    Staff members can add licenses to template by sending POST request with list of licenses uuids.
    Example POST data: {'licenses': [license1_uuid, licenses2_uuid ...]}
    """

    queryset = models.Template.objects.all()
    serializer_class = serializers.TemplateSerializer
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    lookup_field = 'uuid'

    def get_serializer_class(self):
        if self.request.method in ('POST', 'PUT', 'PATCH'):
            return serializers.TemplateCreateSerializer

        return super(TemplateViewSet, self).get_serializer_class()

    def get_queryset(self):
        queryset = super(TemplateViewSet, self).get_queryset()

        user = self.request.user

        if not user.is_staff:
            queryset = queryset.exclude(is_active=False)

        if self.request.method == 'GET':
            cloud_uuid = self.request.QUERY_PARAMS.get('cloud')
            if cloud_uuid is not None:
                cloud_queryset = filter_queryset_for_user(
                    Cloud.objects.all(), user)

                try:
                    cloud = cloud_queryset.get(uuid=cloud_uuid)
                except Cloud.DoesNotExist:
                    return queryset.none()

                queryset = queryset.filter(images__cloud=cloud)

        return queryset

class SshKeyViewSet(core_viewsets.ModelViewSet):
    """List of SSH public keys that are accessible by this user.

    SSH public keys are injected to VM instances during creation, so that holder of corresponding SSH private key can log in to that instance.

    SSH public keys are connected to user accounts, whereas the key may belong to one user only, and the user may have multiple SSH keys.

    Users can only access SSH keys connected to their accounts.

    Project administrators can select what SSH key will be injected to VM instance during instance provisioning.
    """

    queryset = core_models.SshPublicKey.objects.all()
    serializer_class = serializers.SshKeySerializer
    lookup_field = 'uuid'

    def pre_save(self, key):
        key.user = self.request.user

    def get_queryset(self):
        queryset = super(SshKeyViewSet, self).get_queryset()
        user = self.request.user
        return queryset.filter(user=user)

class PurchaseViewSet(core_viewsets.ReadOnlyModelViewSet):
    """
    List of operations with VM templates.

    TODO: list supported operation types.

    TODO: describe permissions for different user types.
    """

    queryset = models.Purchase.objects.all()
    serializer_class = serializers.PurchaseSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter,)

class ImageViewSet(core_viewsets.ReadOnlyModelViewSet):
    """TODO: add documentation.

    TODO: describe permissions for different user types.
    """

    queryset = models.Image.objects.all()
    serializer_class = serializers.ImageSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter,)

class TemplateLicenseViewSet(core_viewsets.ModelViewSet):
    """
    Every template is potentially connected to zero or more consumed licenses.
    License is defined as an abstract consumable.

    Only staff can view all licenses, edit and delete them.

    Customer owners, managers and administrators can view license only with templates

    Add customer uuid as `customer` GET parameter to filter licenses for customer
    """
    queryset = models.TemplateLicense.objects.all()
    serializer_class = serializers.TemplateLicenseSerializer
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    lookup_field = 'uuid'

    def get_queryset(self):
        if not self.request.user.is_staff:
            raise Http404

        queryset = super(TemplateLicenseViewSet, self).get_queryset()

        if 'customer' in self.request.QUERY_PARAMS:
            customer_uuid = self.request.QUERY_PARAMS['customer']
            customer_templates_ids = models.Template.objects.filter(
                images__cloud__projects__customer__uuid=customer_uuid).values_list('id', flat=True)
            queryset = queryset.filter(templates__in=customer_templates_ids)

        return queryset

    @link(is_for_list=True)
    def stats(self, request):
        aggregate = self.request.QUERY_PARAMS.get('aggregate', 'name')
        if aggregate == 'project_name':
            aggregate_field = 'instance__project__name'
        elif aggregate == 'project_group':
            aggregate_field = 'instance__project__project_groups__name'
        elif aggregate == 'license_type':
            aggregate_field = 'template_license__license_type'
        else:
            aggregate_field = 'template_license__name'

        queryset = filters.filter_queryset_for_user(models.InstanceLicense.objects.all(), request.user)
        queryset = queryset.values(aggregate_field).annotate(count=django_models.Count('id'))

        # This hack can be removed when https://code.djangoproject.com/ticket/16735 will be closed
        for d in queryset:
            d[aggregate] = d[aggregate_field]
            del d[aggregate_field]

        return Response(queryset)

class ServiceFilter(django_filters.FilterSet):
    project_groups = django_filters.CharFilter(
        name='project__project_groups__name',
        distinct=True,
        lookup_type='icontains',
    )
    project_name = django_filters.CharFilter(
        name='project__name',
        distinct=True,
        lookup_type='icontains',
    )

    name = django_filters.CharFilter(name='hostname', lookup_type='icontains')
    agreed_sla = django_filters.NumberFilter()
    actual_sla = django_filters.NumberFilter()

    class Meta(object):
        model = models.Instance
        fields = [
            'project_name',
            'name',
            'project_groups',
        ]

# XXX: This view has to be rewritten or removed after haystack implementation
class ServiceViewSet(core_viewsets.ReadOnlyModelViewSet):

    queryset = models.Instance.objects.all()
    serializer_class = serializers.ServiceSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter, rf_filter.DjangoFilterBackend)
    filter_class = ServiceFilter
<commit_msg>stats distinct count (NC-158)<commit_after>from __future__ import unicode_literals

import logging

from django.db import models as django_models
from django.http import Http404
import django_filters
from django_fsm import TransitionNotAllowed
from rest_framework import filters as rf_filter
from rest_framework import mixins
from rest_framework import permissions, status
from rest_framework import viewsets
from rest_framework.exceptions import PermissionDenied
from rest_framework.response import Response
from rest_framework_extensions.decorators import action, link

from nodeconductor.cloud.models import Cloud, Flavor
from nodeconductor.core import mixins as core_mixins
from nodeconductor.core import models as core_models
from nodeconductor.core import viewsets as core_viewsets
from nodeconductor.iaas import models
from nodeconductor.iaas import serializers
from nodeconductor.structure import filters
from nodeconductor.structure.filters import filter_queryset_for_user
from nodeconductor.structure.models import ProjectRole

logger = logging.getLogger(__name__)

class InstanceFilter(django_filters.FilterSet):
    project_group = django_filters.CharFilter(
        name='project__project_groups__name',
        distinct=True,
        lookup_type='icontains',
    )
    project = django_filters.CharFilter(
        name='project__name',
        distinct=True,
        lookup_type='icontains',
    )

    customer_name = django_filters.CharFilter(
        name='project__customer__name',
        distinct=True,
        lookup_type='icontains',
    )

    hostname = django_filters.CharFilter(lookup_type='icontains')
    state = django_filters.CharFilter()

    class Meta(object):
        model = models.Instance
        fields = [
            'hostname',
            'customer_name',
            'state',
            'project',
            'project_group',
        ]
        order_by = [
            'hostname',
            'state',
        ]

class InstanceViewSet(mixins.CreateModelMixin,
                      mixins.RetrieveModelMixin,
                      core_mixins.ListModelMixin,
                      core_mixins.UpdateOnlyModelMixin,
                      viewsets.GenericViewSet):
    """List of VM instances that are accessible by this user.

    VM instances are launched in clouds, whereas the instance may belong to one cloud only, and the cloud may have
    multiple VM instances.

    VM instance may be in one of the following states:
     - creating
     - created
     - starting
     - started
     - stopping
     - stopped
     - restarting
     - deleting
     - deleted
     - erred

    Staff members can list all available VM instances in any cloud.
    Customer owners can list all VM instances in all the clouds that belong to any of the customers they own.
    Project administrators can list all VM instances, create new instances and start/stop/restart instances in all the
    clouds that are connected to any of the projects they are administrators in.
    Project managers can list all VM instances in all the clouds that are connected to any of the projects they are
    managers in.
    """

    queryset = models.Instance.objects.all()
    serializer_class = serializers.InstanceSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter, rf_filter.DjangoFilterBackend)
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    filter_class = InstanceFilter

    def get_serializer_class(self):
        if self.request.method in ('POST', 'PUT', 'PATCH'):
            return serializers.InstanceCreateSerializer

        return super(InstanceViewSet, self).get_serializer_class()

    def get_serializer_context(self):
        """
        Extra context provided to the serializer class.
        """
        context = super(InstanceViewSet, self).get_serializer_context()
        context['user'] = self.request.user
        return context

    def get_queryset(self):
        queryset = super(InstanceViewSet, self).get_queryset()
        queryset = queryset.exclude(state=models.Instance.States.DELETED)
        return queryset

    def _schedule_transition(self, request, uuid, operation, **kwargs):
        # Importing here to avoid circular imports
        from nodeconductor.iaas import tasks
        # XXX: this should be testing for actions/role pairs as well
        instance = filter_queryset_for_user(models.Instance.objects.filter(uuid=uuid), request.user).first()

        if instance is None:
            raise Http404()

        is_admin = instance.project.roles.filter(permission_group__user=request.user,
                                                 role_type=ProjectRole.ADMINISTRATOR).exists()
        if not is_admin:
            raise PermissionDenied()

        supported_operations = {
            # code: (scheduled_celery_task, instance_marker_state)
            'start': (instance.schedule_starting, tasks.schedule_starting),
            'stop': (instance.schedule_stopping, tasks.schedule_stopping),
            'destroy': (instance.schedule_deletion, tasks.schedule_deleting),
            'resize': (instance.schedule_resizing, tasks.schedule_resizing),
        }

        logger.info('Scheduling provisioning instance with uuid %s', uuid)
        processing_task = supported_operations[operation][1]
        instance_schedule_transition = supported_operations[operation][0]
        try:
            instance_schedule_transition()
            instance.save()
            processing_task.delay(uuid, **kwargs)
        except TransitionNotAllowed:
            return Response({'status': 'Performing %s operation from instance state \'%s\' is not allowed'
                            % (operation, instance.get_state_display())},
                            status=status.HTTP_409_CONFLICT)

        return Response({'status': '%s was scheduled' % operation})

    @action()
    def stop(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'stop')

    @action()
    def start(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'start')

    def destroy(self, request, uuid=None):
        return self._schedule_transition(request, uuid, 'destroy')

    @action()
    def resize(self, request, uuid=None):
        try:
            instance = models.Instance.objects.get(uuid=uuid)
        except models.Instance.DoesNotExist:
            raise Http404()

        try:
            flavor_uuid = request.DATA['flavor']
        except KeyError:
            return Response(status=status.HTTP_400_BAD_REQUEST)

        instance_cloud = instance.flavor.cloud

        new_flavor = Flavor.objects.filter(cloud=instance_cloud, uuid=flavor_uuid)

        if new_flavor.exists():
            return self._schedule_transition(request, uuid, 'resize', new_flavor=flavor_uuid)

        return Response({'status': "New flavor is not within the same cloud"},
                        status=status.HTTP_400_BAD_REQUEST)

class TemplateViewSet(core_viewsets.UpdateModelViewSet):
    """List of VM templates that are accessible by this user.

    VM template is a description of a system installed on VM instances: OS, disk partition etc.

    VM template is not to be confused with VM instance flavor -- template is a definition of a system to be installed (set of software) whereas flavor is a set of virtual hardware parameters.

    VM templates are connected to clouds, whereas the template may belong to one cloud only, and the cloud may have multiple VM templates.

    Staff members can list all available VM templates in any cloud and create new templates.

    Customer owners can list all VM templates in all the clouds that belong to any of the customers they own.

    Project administrators can list all VM templates and create new VM instances using these templates in all the clouds that are connected to any of the projects they are administrators in.

    Project managers can list all VM templates in all the clouds that are connected to any of the projects they are managers in.

    Staff members can add licenses to template by sending POST request with list of licenses uuids.
    Example POST data: {'licenses': [license1_uuid, licenses2_uuid ...]}
    """

    queryset = models.Template.objects.all()
    serializer_class = serializers.TemplateSerializer
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    lookup_field = 'uuid'

    def get_serializer_class(self):
        if self.request.method in ('POST', 'PUT', 'PATCH'):
            return serializers.TemplateCreateSerializer

        return super(TemplateViewSet, self).get_serializer_class()

    def get_queryset(self):
        queryset = super(TemplateViewSet, self).get_queryset()

        user = self.request.user

        if not user.is_staff:
            queryset = queryset.exclude(is_active=False)

        if self.request.method == 'GET':
            cloud_uuid = self.request.QUERY_PARAMS.get('cloud')
            if cloud_uuid is not None:
                cloud_queryset = filter_queryset_for_user(
                    Cloud.objects.all(), user)

                try:
                    cloud = cloud_queryset.get(uuid=cloud_uuid)
                except Cloud.DoesNotExist:
                    return queryset.none()

                queryset = queryset.filter(images__cloud=cloud)

        return queryset

class SshKeyViewSet(core_viewsets.ModelViewSet):
    """List of SSH public keys that are accessible by this user.

    SSH public keys are injected to VM instances during creation, so that holder of corresponding SSH private key can log in to that instance.

    SSH public keys are connected to user accounts, whereas the key may belong to one user only, and the user may have multiple SSH keys.

    Users can only access SSH keys connected to their accounts.

    Project administrators can select what SSH key will be injected to VM instance during instance provisioning.
    """

    queryset = core_models.SshPublicKey.objects.all()
    serializer_class = serializers.SshKeySerializer
    lookup_field = 'uuid'

    def pre_save(self, key):
        key.user = self.request.user

    def get_queryset(self):
        queryset = super(SshKeyViewSet, self).get_queryset()
        user = self.request.user
        return queryset.filter(user=user)

class PurchaseViewSet(core_viewsets.ReadOnlyModelViewSet):
    """
    List of operations with VM templates.

    TODO: list supported operation types.

    TODO: describe permissions for different user types.
    """

    queryset = models.Purchase.objects.all()
    serializer_class = serializers.PurchaseSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter,)

class ImageViewSet(core_viewsets.ReadOnlyModelViewSet):
    """TODO: add documentation.

    TODO: describe permissions for different user types.
    """

    queryset = models.Image.objects.all()
    serializer_class = serializers.ImageSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter,)

class TemplateLicenseViewSet(core_viewsets.ModelViewSet):
    """
    Every template is potentially connected to zero or more consumed licenses.
    License is defined as an abstract consumable.

    Only staff can view all licenses, edit and delete them.

    Customer owners, managers and administrators can view license only with templates

    Add customer uuid as `customer` GET parameter to filter licenses for customer
    """
    queryset = models.TemplateLicense.objects.all()
    serializer_class = serializers.TemplateLicenseSerializer
    permission_classes = (permissions.IsAuthenticated, permissions.DjangoObjectPermissions)
    lookup_field = 'uuid'

    def get_queryset(self):
        if not self.request.user.is_staff:
            raise Http404

        queryset = super(TemplateLicenseViewSet, self).get_queryset()

        if 'customer' in self.request.QUERY_PARAMS:
            customer_uuid = self.request.QUERY_PARAMS['customer']
            customer_templates_ids = models.Template.objects.filter(
                images__cloud__projects__customer__uuid=customer_uuid).values_list('id', flat=True)
            queryset = queryset.filter(templates__in=customer_templates_ids)

        return queryset

    @link(is_for_list=True)
    def stats(self, request):
        aggregate = self.request.QUERY_PARAMS.get('aggregate', 'name')
        if aggregate == 'project_name':
            aggregate_field = 'instance__project__name'
        elif aggregate == 'project_group':
            aggregate_field = 'instance__project__project_groups__name'
        elif aggregate == 'license_type':
            aggregate_field = 'template_license__license_type'
        else:
            aggregate_field = 'template_license__name'

        queryset = filters.filter_queryset_for_user(models.InstanceLicense.objects.all(), request.user)
        queryset = queryset.values(aggregate_field).annotate(count=django_models.Count('id', distinct=True))

        # This hack can be removed when https://code.djangoproject.com/ticket/16735 will be closed
        for d in queryset:
            d[aggregate] = d[aggregate_field]
            del d[aggregate_field]

        return Response(queryset)

class ServiceFilter(django_filters.FilterSet):
    project_groups = django_filters.CharFilter(
        name='project__project_groups__name',
        distinct=True,
        lookup_type='icontains',
    )
    project_name = django_filters.CharFilter(
        name='project__name',
        distinct=True,
        lookup_type='icontains',
    )

    name = django_filters.CharFilter(name='hostname', lookup_type='icontains')
    agreed_sla = django_filters.NumberFilter()
    actual_sla = django_filters.NumberFilter()

    class Meta(object):
        model = models.Instance
        fields = [
            'project_name',
            'name',
            'project_groups',
        ]

# XXX: This view has to be rewritten or removed after haystack implementation
class ServiceViewSet(core_viewsets.ReadOnlyModelViewSet):

    queryset = models.Instance.objects.all()
    serializer_class = serializers.ServiceSerializer
    lookup_field = 'uuid'
    filter_backends = (filters.GenericRoleFilter, rf_filter.DjangoFilterBackend)
    filter_class = ServiceFilter
<|endoftext|>


<commit_before>try:
    from decimal import Decimal
except:
    from django.utils._decimal import Decimal

import logging

from django.core import urlresolvers
from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.datastructures import MultiValueDictKeyError
from django.utils.safestring import mark_safe
from django.utils.simplejson.encoder import JSONEncoder
from django.utils.translation import ugettext as _
from django.views.decorators.cache import never_cache

from livesettings import config_value
from product.models import Product, OptionManager
from product.utils import find_best_auto_discount
from product.views import find_product_template, optionids_from_post
from satchmo_store.shop import forms
from satchmo_store.shop.exceptions import CartAddProhibited
from satchmo_store.shop.models import Cart, CartItem, NullCart, NullCartItem
from satchmo_store.shop.signals import satchmo_cart_changed, satchmo_cart_add_complete, satchmo_cart_details_query
from satchmo_utils.views import bad_or_missing
from satchmo_utils.numbers import RoundedDecimalError, round_decimal

log = logging.getLogger('shop.views.cart')

NOTSET = object()

def _set_quantity(request, force_delete=False):
    """Set the quantity for a specific cartitem.
    Checks to make sure the item is actually in the user's cart.
    """
    cart = Cart.objects.from_request(request, create=False)
    if isinstance(cart, NullCart):
        return (False, None, None, _("No cart to update."))

    cartplaces = config_value('SHOP', 'CART_PRECISION')
    if force_delete:
        qty = Decimal('0')
    else:
        try:
            roundfactor = config_value('SHOP', 'CART_ROUNDING')
            qty = round_decimal(request.POST.get('quantity', 0), places=cartplaces, roundfactor=roundfactor, normalize=True)
        except RoundedDecimalError, P:
            return (False, cart, None, _("Bad quantity."))

        if qty < Decimal('0'):
            qty = Decimal('0')

    try:
        itemid = int(request.POST.get('cartitem'))
    except (TypeError, ValueError):
        return (False, cart, None, _("Bad item number."))

    try:
        cartitem = CartItem.objects.get(pk=itemid, cart=cart)
    except CartItem.DoesNotExist:
        return (False, cart, None, _("No such item in your cart."))

    if qty == Decimal('0'):
        cartitem.delete()
        cartitem = NullCartItem(itemid)
    else:
        from satchmo_store.shop.models import Config
        config = Config.objects.get_current()
        if config_value('PRODUCT','NO_STOCK_CHECKOUT') == False:
            stock = cartitem.product.items_in_stock
            log.debug('checking stock quantity.  Have %d, need %d', stock, qty)
            if stock < qty:
                return (False, cart, cartitem, _("Not enough items of '%s' in stock.") % cartitem.product.translated_name())

        cartitem.quantity = round_decimal(qty, places=cartplaces)
        cartitem.save()

    satchmo_cart_changed.send(cart, cart=cart, request=request)
    return (True, cart, cartitem, "")

def display(request, cart=None, error_message='', default_view_tax=NOTSET):
    """Display the items in the cart."""

    if default_view_tax == NOTSET:
        default_view_tax = config_value('TAX', 'DEFAULT_VIEW_TAX')

    if not cart:
        cart = Cart.objects.from_request(request)

    if cart.numItems > 0:
        products = [item.product for item in cart.cartitem_set.all()]
        sale = find_best_auto_discount(products)
    else:
        sale = None

    context = RequestContext(request, {
        'cart': cart,
        'error_message': error_message,
        'default_view_tax' : default_view_tax,
        'sale' : sale,
        })
    return render_to_response('shop/cart.html', context)

display = never_cache(display)

def add(request, id=0, redirect_to='satchmo_cart'):
    """Add an item to the cart."""
    log.debug('FORM: %s', request.POST)
    formdata = request.POST.copy()
    productslug = None

    cartplaces = config_value('SHOP', 'CART_PRECISION')
    roundfactor = config_value('SHOP', 'CART_ROUNDING')

    if formdata.has_key('productname'):
        productslug = formdata['productname']
    try:
        product, details = product_from_post(productslug, formdata)

        if not (product and product.active):
            log.debug("product %s is not active" % productslug)
            return bad_or_missing(request, _("That product is not available at the moment."))
        else:
            log.debug("product %s is active" % productslug)

    except (Product.DoesNotExist, MultiValueDictKeyError):
        log.debug("Could not find product: %s", productslug)
        return bad_or_missing(request, _('The product you have requested does not exist.'))

    try:
        quantity = round_decimal(formdata['quantity'], places=cartplaces, roundfactor=roundfactor)
    except RoundedDecimalError, P:
        return _product_error(request, product,
            _("Invalid quantity."))

    if quantity <= Decimal('0'):
        return _product_error(request, product,
            _("Please enter a positive number."))

    cart = Cart.objects.from_request(request, create=True)
    # send a signal so that listeners can update product details before we add it to the cart.
    satchmo_cart_details_query.send(
            cart,
            product=product,
            quantity=quantity,
            details=details,
            request=request,
            form=formdata
            )
    try:
        added_item = cart.add_item(product, number_added=quantity, details=details)

    except CartAddProhibited, cap:
        return _product_error(request, product, cap.message)

    # got to here with no error, now send a signal so that listeners can also operate on this form.
    satchmo_cart_add_complete.send(cart, cart=cart, cartitem=added_item, product=product, request=request, form=formdata)
    satchmo_cart_changed.send(cart, cart=cart, request=request)

    url = urlresolvers.reverse(redirect_to)
    return HttpResponseRedirect(url)

def add_ajax(request, id=0, template="shop/json.html"):
    cartplaces = config_value('SHOP', 'CART_PRECISION')
    roundfactor = config_value('SHOP', 'CART_ROUNDING')

    data = {'errors': []}
    product = None
    formdata = request.POST.copy()
    if not formdata.has_key('productname'):
        data['errors'].append(('product', _('No product requested')))
    else:
        productslug = formdata['productname']
        log.debug('CART_AJAX: slug=%s', productslug)
        try:
            product, details = product_from_post(productslug, formdata)

        except Product.DoesNotExist:
            log.warn("Could not find product: %s", productslug)
            product = None

        if not product:
            data['errors'].append(('product', _('The product you have requested does not exist.')))

        else:
            if not product.active:
                data['errors'].append(('product', _('That product is not available at the moment.')))

            else:
                data['id'] = product.id
                data['name'] = product.translated_name()

                if not formdata.has_key('quantity'):
                    quantity = Decimal('-1')
                else:
                    quantity = formdata['quantity']

                try:
                    quantity = round_decimal(quantity, places=cartplaces, roundfactor=roundfactor)

                    if quantity < Decimal('0'):
                        data['errors'].append(('quantity', _('Choose a quantity.')))

                except RoundedDecimalError:
                    data['errors'].append(('quantity', _('Invalid quantity.')))

    tempCart = Cart.objects.from_request(request, create=True)

    if not data['errors']:
        # send a signal so that listeners can update product details before we add it to the cart.
        satchmo_cart_details_query.send(
                tempCart,
                product=product,
                quantity=quantity,
                details=details,
                request=request,
                form=formdata
                )
        try:
            added_item = tempCart.add_item(product, number_added=quantity)
            request.session['cart'] = tempCart.id
            data['results'] = _('Success')
            if added_item:
                # send a signal so that listeners can also operate on this form and item.
                satchmo_cart_add_complete.send(
                        tempCart,
                        cartitem=added_item,
                        product=product,
                        request=request,
                        form=formdata
                        )

        except CartAddProhibited, cap:
            data['results'] = _('Error')
            data['errors'].append(('product', cap.message))

    else:
        data['results'] = _('Error')

    data['cart_count'] = str(round_decimal(tempCart.numItems, 2))

    encoded = JSONEncoder().encode(data)
    encoded = mark_safe(encoded)
    log.debug('CART AJAX: %s', data)

    satchmo_cart_changed.send(tempCart, cart=tempCart, request=request)
    return render_to_response(template, {'json' : encoded})

def add_multiple(request, redirect_to='satchmo_cart', products=None, template="shop/multiple_product_form.html"):
    """Add multiple items to the cart.
    """
    if request.method == "POST":
        log.debug('FORM: %s', request.POST)
        formdata = request.POST.copy()
        form = forms.MultipleProductForm(formdata, products=products)

        if form.is_valid():
            cart = Cart.objects.from_request(request, create=True)
            form.save(cart, request)
            satchmo_cart_changed.send(cart, cart=cart, request=request)

            url = urlresolvers.reverse(redirect_to)
            return HttpResponseRedirect(url)
    else:
        form = forms.MultipleProductForm(products=products)

    return render_to_response(template, RequestContext(request, {'form' : form}))

def agree_terms(request):
    """Agree to terms"""
    if request.method == "POST":
        if request.POST.get('agree_terms', False):
            url = urlresolvers.reverse('satchmo_checkout-step1')
            return HttpResponseRedirect(url)

    return display(request, error_message=_('You must accept the terms and conditions.'))

def remove(request):
    """Remove an item from the cart."""
    success, cart, cartitem, errors = _set_quantity(request, force_delete=True)
    if errors:
        return display(request, cart=cart, error_message=errors)
    else:
        url = urlresolvers.reverse('satchmo_cart')
        return HttpResponseRedirect(url)

def remove_ajax(request, template="shop/json.html"):
    """Remove an item from the cart. Returning JSON formatted results."""
    data = {}
    if not request.POST:
        data['results'] = False
        data['errors'] = _('Internal error: please submit as a POST')

    else:
        success, cart, cartitem, errors = _set_quantity(request, force_delete=True)

        data['results'] = success
        data['errors'] = errors

        # note we have to convert Decimals to strings, since simplejson doesn't know about Decimals
        if cart and cartitem:
            data['cart_total'] = str(cart.total)
            data['cart_count'] = str(round_decimal(cart.numItems, 2))
            data['item_id'] = cartitem.id

        return render_to_response(template, {'json': JSONEncoder().encode(data)})

def set_quantity(request):
    """Set the quantity for a cart item.

    Intended to be called via the cart itself, returning to the cart after done.
    """
    cart_url = urlresolvers.reverse('satchmo_cart')

    if not request.POST:
        return HttpResponseRedirect(cart_url)

    success, cart, cartitem, errors = _set_quantity(request)
    if success:
        return HttpResponseRedirect(cart_url)
    else:
        return display(request, cart = cart, error_message = errors)

def set_quantity_ajax(request, template="shop/json.html"):
    """Set the quantity for a cart item, returning results formatted for handling by script.
    """
    data = {}
    if not request.POST:
        data['results'] = False
        data['errors'] = _('Internal error: please submit as a POST')

    else:
        success, cart, cartitem, errors = _set_quantity(request)

        data['results'] = success
        data['errors'] = errors

        # note we have to convert Decimals to strings, since simplejson doesn't know about Decimals
        if cart:
            carttotal = str(round_decimal(cart.total, 2))
            cartqty = cart.numItems
        else:
            carttotal = "0.00"
            cartqty = Decimal('0')

        data['cart_total'] = carttotal
        data['cart_count'] = str(round_decimal(cartqty, 2))

        if cartitem:
            itemid = cartitem.id
            itemqty = cartitem.quantity
            price = str(round_decimal(cartitem.line_total, 2))
        else:
            itemid = -1
            itemqty = Decimal('0')
            price = "0.00"

        data['item_id'] = itemid
        data['item_qty'] = str(round_decimal(itemqty, 2))
        data['item_price'] = price

    encoded = JSONEncoder().encode(data)
    encoded = mark_safe(encoded)
    return render_to_response(template, {'json': encoded})

def product_from_post(productslug, formdata):
    product = Product.objects.get_by_site(slug=productslug)
    log.debug('found product: %s', product)
    p_types = product.get_subtypes()
    details = []
    zero = Decimal("0.00")

    if 'ConfigurableProduct' in p_types:
        # This happens when productname cannot be updated by javascript.
        cp = product.configurableproduct
        chosenOptions = optionids_from_post(cp, formdata)
        optproduct = cp.get_product_from_options(chosenOptions)
        if not optproduct:
            log.debug('Could not fully configure a ConfigurableProduct [%s] with [%s]', product, chosenOptions)
            raise Product.DoesNotExist()
        else:
            product = optproduct

    if 'CustomProduct' in p_types:
        cp = product.customproduct
        for customfield in cp.custom_text_fields.all():
            if customfield.price_change is not None:
                price_change = customfield.price_change
            else:
                price_change = zero
            data = { 'name' : customfield.translated_name(),
                     'value' : formdata["custom_%s" % customfield.slug],
                     'sort_order': customfield.sort_order,
                     'price_change': price_change }
            details.append(data)
            data = {}
        chosenOptions = optionids_from_post(cp, formdata)
        manager = OptionManager()
        for choice in chosenOptions:
            result = manager.from_unique_id(choice)
            if result.price_change is not None:
                price_change = result.price_change
            else:
                price_change = zero
            data = { 'name': unicode(result.option_group),
                      'value': unicode(result.translated_name()),
                      'sort_order': result.sort_order,
                      'price_change': price_change
            }
            details.append(data)
            data = {}

    if 'GiftCertificateProduct' in p_types:
        ix = 0
        for field in ('email', 'message'):
            data = {
                'name' : field,
                'value' : formdata.get("custom_%s" % field, ""),
                'sort_order' : ix,
                'price_change' : zero,
            }
            ix += 1
            details.append(data)
        log.debug("Gift Certificate details: %s", details)
        data = {}

    return product, details

def _product_error(request, product, msg):
    request.session['ERRORS'] = msg
    return HttpResponseRedirect(product.get_absolute_url())

<commit_msg>Fix an indentation mistake. Closes #806<commit_after>try:
    from decimal import Decimal
except:
    from django.utils._decimal import Decimal

import logging

from django.core import urlresolvers
from django.http import HttpResponseRedirect, HttpResponse, Http404
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.datastructures import MultiValueDictKeyError
from django.utils.safestring import mark_safe
from django.utils.simplejson.encoder import JSONEncoder
from django.utils.translation import ugettext as _
from django.views.decorators.cache import never_cache

from livesettings import config_value
from product.models import Product, OptionManager
from product.utils import find_best_auto_discount
from product.views import find_product_template, optionids_from_post
from satchmo_store.shop import forms
from satchmo_store.shop.exceptions import CartAddProhibited
from satchmo_store.shop.models import Cart, CartItem, NullCart, NullCartItem
from satchmo_store.shop.signals import satchmo_cart_changed, satchmo_cart_add_complete, satchmo_cart_details_query
from satchmo_utils.views import bad_or_missing
from satchmo_utils.numbers import RoundedDecimalError, round_decimal

log = logging.getLogger('shop.views.cart')

NOTSET = object()

def _set_quantity(request, force_delete=False):
    """Set the quantity for a specific cartitem.
    Checks to make sure the item is actually in the user's cart.
    """
    cart = Cart.objects.from_request(request, create=False)
    if isinstance(cart, NullCart):
        return (False, None, None, _("No cart to update."))

    cartplaces = config_value('SHOP', 'CART_PRECISION')
    if force_delete:
        qty = Decimal('0')
    else:
        try:
            roundfactor = config_value('SHOP', 'CART_ROUNDING')
            qty = round_decimal(request.POST.get('quantity', 0), places=cartplaces, roundfactor=roundfactor, normalize=True)
        except RoundedDecimalError, P:
            return (False, cart, None, _("Bad quantity."))

        if qty < Decimal('0'):
            qty = Decimal('0')

    try:
        itemid = int(request.POST.get('cartitem'))
    except (TypeError, ValueError):
        return (False, cart, None, _("Bad item number."))

    try:
        cartitem = CartItem.objects.get(pk=itemid, cart=cart)
    except CartItem.DoesNotExist:
        return (False, cart, None, _("No such item in your cart."))

    if qty == Decimal('0'):
        cartitem.delete()
        cartitem = NullCartItem(itemid)
    else:
        from satchmo_store.shop.models import Config
        config = Config.objects.get_current()
        if config_value('PRODUCT','NO_STOCK_CHECKOUT') == False:
            stock = cartitem.product.items_in_stock
            log.debug('checking stock quantity.  Have %d, need %d', stock, qty)
            if stock < qty:
                return (False, cart, cartitem, _("Not enough items of '%s' in stock.") % cartitem.product.translated_name())

        cartitem.quantity = round_decimal(qty, places=cartplaces)
        cartitem.save()

    satchmo_cart_changed.send(cart, cart=cart, request=request)
    return (True, cart, cartitem, "")

def display(request, cart=None, error_message='', default_view_tax=NOTSET):
    """Display the items in the cart."""

    if default_view_tax == NOTSET:
        default_view_tax = config_value('TAX', 'DEFAULT_VIEW_TAX')

    if not cart:
        cart = Cart.objects.from_request(request)

    if cart.numItems > 0:
        products = [item.product for item in cart.cartitem_set.all()]
        sale = find_best_auto_discount(products)
    else:
        sale = None

    context = RequestContext(request, {
        'cart': cart,
        'error_message': error_message,
        'default_view_tax' : default_view_tax,
        'sale' : sale,
        })
    return render_to_response('shop/cart.html', context)

display = never_cache(display)

def add(request, id=0, redirect_to='satchmo_cart'):
    """Add an item to the cart."""
    log.debug('FORM: %s', request.POST)
    formdata = request.POST.copy()
    productslug = None

    cartplaces = config_value('SHOP', 'CART_PRECISION')
    roundfactor = config_value('SHOP', 'CART_ROUNDING')

    if formdata.has_key('productname'):
        productslug = formdata['productname']
    try:
        product, details = product_from_post(productslug, formdata)

        if not (product and product.active):
            log.debug("product %s is not active" % productslug)
            return bad_or_missing(request, _("That product is not available at the moment."))
        else:
            log.debug("product %s is active" % productslug)

    except (Product.DoesNotExist, MultiValueDictKeyError):
        log.debug("Could not find product: %s", productslug)
        return bad_or_missing(request, _('The product you have requested does not exist.'))

    try:
        quantity = round_decimal(formdata['quantity'], places=cartplaces, roundfactor=roundfactor)
    except RoundedDecimalError, P:
        return _product_error(request, product,
            _("Invalid quantity."))

    if quantity <= Decimal('0'):
        return _product_error(request, product,
            _("Please enter a positive number."))

    cart = Cart.objects.from_request(request, create=True)
    # send a signal so that listeners can update product details before we add it to the cart.
    satchmo_cart_details_query.send(
            cart,
            product=product,
            quantity=quantity,
            details=details,
            request=request,
            form=formdata
            )
    try:
        added_item = cart.add_item(product, number_added=quantity, details=details)

    except CartAddProhibited, cap:
        return _product_error(request, product, cap.message)

    # got to here with no error, now send a signal so that listeners can also operate on this form.
    satchmo_cart_add_complete.send(cart, cart=cart, cartitem=added_item, product=product, request=request, form=formdata)
    satchmo_cart_changed.send(cart, cart=cart, request=request)

    url = urlresolvers.reverse(redirect_to)
    return HttpResponseRedirect(url)

def add_ajax(request, id=0, template="shop/json.html"):
    cartplaces = config_value('SHOP', 'CART_PRECISION')
    roundfactor = config_value('SHOP', 'CART_ROUNDING')

    data = {'errors': []}
    product = None
    formdata = request.POST.copy()
    if not formdata.has_key('productname'):
        data['errors'].append(('product', _('No product requested')))
    else:
        productslug = formdata['productname']
        log.debug('CART_AJAX: slug=%s', productslug)
        try:
            product, details = product_from_post(productslug, formdata)

        except Product.DoesNotExist:
            log.warn("Could not find product: %s", productslug)
            product = None

        if not product:
            data['errors'].append(('product', _('The product you have requested does not exist.')))

        else:
            if not product.active:
                data['errors'].append(('product', _('That product is not available at the moment.')))

            else:
                data['id'] = product.id
                data['name'] = product.translated_name()

                if not formdata.has_key('quantity'):
                    quantity = Decimal('-1')
                else:
                    quantity = formdata['quantity']

                try:
                    quantity = round_decimal(quantity, places=cartplaces, roundfactor=roundfactor)

                    if quantity < Decimal('0'):
                        data['errors'].append(('quantity', _('Choose a quantity.')))

                except RoundedDecimalError:
                    data['errors'].append(('quantity', _('Invalid quantity.')))

    tempCart = Cart.objects.from_request(request, create=True)

    if not data['errors']:
        # send a signal so that listeners can update product details before we add it to the cart.
        satchmo_cart_details_query.send(
                tempCart,
                product=product,
                quantity=quantity,
                details=details,
                request=request,
                form=formdata
                )
        try:
            added_item = tempCart.add_item(product, number_added=quantity)
            request.session['cart'] = tempCart.id
            data['results'] = _('Success')
            if added_item:
                # send a signal so that listeners can also operate on this form and item.
                satchmo_cart_add_complete.send(
                        tempCart,
                        cartitem=added_item,
                        product=product,
                        request=request,
                        form=formdata
                        )

        except CartAddProhibited, cap:
            data['results'] = _('Error')
            data['errors'].append(('product', cap.message))

    else:
        data['results'] = _('Error')

    data['cart_count'] = str(round_decimal(tempCart.numItems, 2))

    encoded = JSONEncoder().encode(data)
    encoded = mark_safe(encoded)
    log.debug('CART AJAX: %s', data)

    satchmo_cart_changed.send(tempCart, cart=tempCart, request=request)
    return render_to_response(template, {'json' : encoded})

def add_multiple(request, redirect_to='satchmo_cart', products=None, template="shop/multiple_product_form.html"):
    """Add multiple items to the cart.
    """
    if request.method == "POST":
        log.debug('FORM: %s', request.POST)
        formdata = request.POST.copy()
        form = forms.MultipleProductForm(formdata, products=products)

        if form.is_valid():
            cart = Cart.objects.from_request(request, create=True)
            form.save(cart, request)
            satchmo_cart_changed.send(cart, cart=cart, request=request)

            url = urlresolvers.reverse(redirect_to)
            return HttpResponseRedirect(url)
    else:
        form = forms.MultipleProductForm(products=products)

    return render_to_response(template, RequestContext(request, {'form' : form}))

def agree_terms(request):
    """Agree to terms"""
    if request.method == "POST":
        if request.POST.get('agree_terms', False):
            url = urlresolvers.reverse('satchmo_checkout-step1')
            return HttpResponseRedirect(url)

    return display(request, error_message=_('You must accept the terms and conditions.'))

def remove(request):
    """Remove an item from the cart."""
    success, cart, cartitem, errors = _set_quantity(request, force_delete=True)
    if errors:
        return display(request, cart=cart, error_message=errors)
    else:
        url = urlresolvers.reverse('satchmo_cart')
        return HttpResponseRedirect(url)

def remove_ajax(request, template="shop/json.html"):
    """Remove an item from the cart. Returning JSON formatted results."""
    data = {}
    if not request.POST:
        data['results'] = False
        data['errors'] = _('Internal error: please submit as a POST')

    else:
        success, cart, cartitem, errors = _set_quantity(request, force_delete=True)

        data['results'] = success
        data['errors'] = errors

        # note we have to convert Decimals to strings, since simplejson doesn't know about Decimals
        if cart and cartitem:
            data['cart_total'] = str(cart.total)
            data['cart_count'] = str(round_decimal(cart.numItems, 2))
            data['item_id'] = cartitem.id

    return render_to_response(template, {'json': JSONEncoder().encode(data)})

def set_quantity(request):
    """Set the quantity for a cart item.

    Intended to be called via the cart itself, returning to the cart after done.
    """
    cart_url = urlresolvers.reverse('satchmo_cart')

    if not request.POST:
        return HttpResponseRedirect(cart_url)

    success, cart, cartitem, errors = _set_quantity(request)
    if success:
        return HttpResponseRedirect(cart_url)
    else:
        return display(request, cart = cart, error_message = errors)

def set_quantity_ajax(request, template="shop/json.html"):
    """Set the quantity for a cart item, returning results formatted for handling by script.
    """
    data = {}
    if not request.POST:
        data['results'] = False
        data['errors'] = _('Internal error: please submit as a POST')

    else:
        success, cart, cartitem, errors = _set_quantity(request)

        data['results'] = success
        data['errors'] = errors

        # note we have to convert Decimals to strings, since simplejson doesn't know about Decimals
        if cart:
            carttotal = str(round_decimal(cart.total, 2))
            cartqty = cart.numItems
        else:
            carttotal = "0.00"
            cartqty = Decimal('0')

        data['cart_total'] = carttotal
        data['cart_count'] = str(round_decimal(cartqty, 2))

        if cartitem:
            itemid = cartitem.id
            itemqty = cartitem.quantity
            price = str(round_decimal(cartitem.line_total, 2))
        else:
            itemid = -1
            itemqty = Decimal('0')
            price = "0.00"

        data['item_id'] = itemid
        data['item_qty'] = str(round_decimal(itemqty, 2))
        data['item_price'] = price

    encoded = JSONEncoder().encode(data)
    encoded = mark_safe(encoded)
    return render_to_response(template, {'json': encoded})

def product_from_post(productslug, formdata):
    product = Product.objects.get_by_site(slug=productslug)
    log.debug('found product: %s', product)
    p_types = product.get_subtypes()
    details = []
    zero = Decimal("0.00")

    if 'ConfigurableProduct' in p_types:
        # This happens when productname cannot be updated by javascript.
        cp = product.configurableproduct
        chosenOptions = optionids_from_post(cp, formdata)
        optproduct = cp.get_product_from_options(chosenOptions)
        if not optproduct:
            log.debug('Could not fully configure a ConfigurableProduct [%s] with [%s]', product, chosenOptions)
            raise Product.DoesNotExist()
        else:
            product = optproduct

    if 'CustomProduct' in p_types:
        cp = product.customproduct
        for customfield in cp.custom_text_fields.all():
            if customfield.price_change is not None:
                price_change = customfield.price_change
            else:
                price_change = zero
            data = { 'name' : customfield.translated_name(),
                     'value' : formdata["custom_%s" % customfield.slug],
                     'sort_order': customfield.sort_order,
                     'price_change': price_change }
            details.append(data)
            data = {}
        chosenOptions = optionids_from_post(cp, formdata)
        manager = OptionManager()
        for choice in chosenOptions:
            result = manager.from_unique_id(choice)
            if result.price_change is not None:
                price_change = result.price_change
            else:
                price_change = zero
            data = { 'name': unicode(result.option_group),
                      'value': unicode(result.translated_name()),
                      'sort_order': result.sort_order,
                      'price_change': price_change
            }
            details.append(data)
            data = {}

    if 'GiftCertificateProduct' in p_types:
        ix = 0
        for field in ('email', 'message'):
            data = {
                'name' : field,
                'value' : formdata.get("custom_%s" % field, ""),
                'sort_order' : ix,
                'price_change' : zero,
            }
            ix += 1
            details.append(data)
        log.debug("Gift Certificate details: %s", details)
        data = {}

    return product, details

def _product_error(request, product, msg):
    request.session['ERRORS'] = msg
    return HttpResponseRedirect(product.get_absolute_url())

<|endoftext|>


<commit_before>#!/usr/bin/env python
# python
# see: http://amp.pharm.mssm.edu/Enrichr/help#api for API docs
# Usage: gseapy enrichr  <genelist> <listdesrciption> <enrichr_library> <enrichr_results>

from __future__ import print_function

import json
import requests
import sys

print("Enrichr API : Start")
print("Enrichr API : Reading Command Options")
## get command line args
genelist = sys.argv[1]
listdesrciption = sys.argv[2]
enrichr_library = sys.argv[3]
enrichr_results = sys.argv[4]

# testing
# genelist = "gene_list.txt"
# enrichr_library = "KEGG_2016"
# listdesrciption = "TEST"
# enrichr_results = "example_enrichment_KEGG_2016"

## Print options
print('Enrichr API : Input file is:', genelist)
print('Enrichr API : Analysis name: ', listdesrciption)
print('Enrichr API : Enrichr Library: ', enrichr_library)
print('Enrichr API : Enrichr Results File: ', enrichr_results)

# get gene lits
print("Enrichr API : Reading:",genelist)
with open(genelist) as f:
    genes = f.read()

## enrichr url
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'

# stick gene list here
genes_str = str(genes)

# name of analysis or list
description = str(listdesrciption)

# payload
payload = {
  'list': (None, genes_str),
  'description': (None, description)
}

# response
print("Enrichr API : requests.post")
response = requests.post(ENRICHR_URL, files=payload)

if not response.ok:
  raise Exception('Error analyzing gene list')

job_id = json.loads(response.text)

print('Enrichr API : Job ID:', job_id)

################################################################################
# View added gene list
#
ENRICHR_URL_A = 'http://amp.pharm.mssm.edu/Enrichr/view?userListId=%s'

user_list_id = job_id['userListId']
#print(user_list_id)

response_gene_list = requests.get(ENRICHR_URL_A % str(user_list_id))

if not response_gene_list.ok:
    raise Exception('Error getting gene list')

print('Enrichr API : View added gene list:', job_id)
added_gene_list = json.loads(response_gene_list.text)
print(added_gene_list)

################################################################################
# Get enrichment results
#
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/enrich'
query_string = '?userListId=%s&backgroundType=%s'

## get id data
user_list_id = job_id['userListId']

## Libraray
gene_set_library = str(enrichr_library)

response = requests.get(
    ENRICHR_URL + query_string % (str(user_list_id), gene_set_library)
 )
if not response.ok:
    raise Exception('Error fetching enrichment results')

print('Enrichr API : Get enrichment results: Job Id:', job_id)
data = json.loads(response.text)
print(data)

################################################################################
## Download file of enrichment results
#
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/export'

query_string = '?userListId=%s&filename=%s&backgroundType=%s'

user_list_id = str(job_id['userListId'])

filename = enrichr_results

gene_set_library = str(enrichr_library)

url = ENRICHR_URL + query_string % (user_list_id, filename, gene_set_library)

response = requests.get(url, stream=True)

print('Enrichr API : Downloading file of enrichment results: Job Id:', job_id)
with open(filename + '.txt', 'wb') as f:
    for chunk in response.iter_content(chunk_size=1024):
        if chunk:
            f.write(chunk)
################################################
print('Enrichr API : Results written to:', enrichr_results + ".txt")
print("Enrichr API : Done")
<commit_msg>add usage of enrichr<commit_after>#!/usr/bin/env python
# python
# see: http://amp.pharm.mssm.edu/Enrichr/help#api for API docs
# Usage: gseapy enrichr  <genelist> <listdesrciption> <enrichr_library> <enrichr_results>

#genelist : flat file with list of genes, one gene id per row
#listdesrciption : name of analysis
#enrichr_library : Enrichr Library to query.
#enrichr_results : out put file prefix

from __future__ import print_function

import json
import requests
import sys

print("Enrichr API : Start")
print("Enrichr API : Reading Command Options")
## get command line args
genelist = sys.argv[1]
listdesrciption = sys.argv[2]
enrichr_library = sys.argv[3]
enrichr_results = sys.argv[4]

# testing
# genelist = "gene_list.txt"
# enrichr_library = "KEGG_2016"
# listdesrciption = "TEST"
# enrichr_results = "example_enrichment_KEGG_2016"

## Print options
print('Enrichr API : Input file is:', genelist)
print('Enrichr API : Analysis name: ', listdesrciption)
print('Enrichr API : Enrichr Library: ', enrichr_library)
print('Enrichr API : Enrichr Results File: ', enrichr_results)

# get gene lits
print("Enrichr API : Reading:",genelist)
with open(genelist) as f:
    genes = f.read()

## enrichr url
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/addList'

# stick gene list here
genes_str = str(genes)

# name of analysis or list
description = str(listdesrciption)

# payload
payload = {
  'list': (None, genes_str),
  'description': (None, description)
}

# response
print("Enrichr API : requests.post")
response = requests.post(ENRICHR_URL, files=payload)

if not response.ok:
  raise Exception('Error analyzing gene list')

job_id = json.loads(response.text)

print('Enrichr API : Job ID:', job_id)

################################################################################
# View added gene list
#
ENRICHR_URL_A = 'http://amp.pharm.mssm.edu/Enrichr/view?userListId=%s'

user_list_id = job_id['userListId']
#print(user_list_id)

response_gene_list = requests.get(ENRICHR_URL_A % str(user_list_id))

if not response_gene_list.ok:
    raise Exception('Error getting gene list')

print('Enrichr API : View added gene list:', job_id)
added_gene_list = json.loads(response_gene_list.text)
print(added_gene_list)

################################################################################
# Get enrichment results
#
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/enrich'
query_string = '?userListId=%s&backgroundType=%s'

## get id data
user_list_id = job_id['userListId']

## Libraray
gene_set_library = str(enrichr_library)

response = requests.get(
    ENRICHR_URL + query_string % (str(user_list_id), gene_set_library)
 )
if not response.ok:
    raise Exception('Error fetching enrichment results')

print('Enrichr API : Get enrichment results: Job Id:', job_id)
data = json.loads(response.text)
print(data)

################################################################################
## Download file of enrichment results
#
ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/export'

query_string = '?userListId=%s&filename=%s&backgroundType=%s'

user_list_id = str(job_id['userListId'])

filename = enrichr_results

gene_set_library = str(enrichr_library)

url = ENRICHR_URL + query_string % (user_list_id, filename, gene_set_library)

response = requests.get(url, stream=True)

print('Enrichr API : Downloading file of enrichment results: Job Id:', job_id)
with open(filename + '.txt', 'wb') as f:
    for chunk in response.iter_content(chunk_size=1024):
        if chunk:
            f.write(chunk)
################################################
print('Enrichr API : Results written to:', enrichr_results + ".txt")
print("Enrichr API : Done")
<|endoftext|>