<commit_before># -*- coding: utf-8 -*-

# Copyright 2017, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

"""
Interface to C++ quantum circuit simulator with realistic noise.
"""

import uuid
import json
import logging
import os
import subprocess
from subprocess import PIPE
import platform

import numpy as np

from qiskit.result._utils import copy_qasm_from_qobj_into_result, result_from_old_style_dict
from qiskit.backends import BaseBackend
from qiskit.backends.aer.aerjob import AerJob
from qiskit.qobj import qobj_to_dict

logger = logging.getLogger(__name__)

EXTENSION = '.exe' if platform.system() == 'Windows' else ''

# Add path to compiled qasm simulator
DEFAULT_SIMULATOR_PATHS = [
    # This is the path where Makefile creates the simulator by default
    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                 '../../../out/src/qasm-simulator-cpp/qasm_simulator_cpp'
                                 + EXTENSION)),
    # This is the path where PIP installs the simulator
    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                 'qasm_simulator_cpp' + EXTENSION)),
]

class QasmSimulator(BaseBackend):
    """C++ quantum circuit simulator with realistic noise"""

    DEFAULT_CONFIGURATION = {
        'name': 'qasm_simulator',
        'url': 'https://github.com/QISKit/qiskit-terra/src/qasm-simulator-cpp',
        'simulator': True,
        'local': True,
        'description': 'A C++ realistic noise simulator for qobj files',
        'coupling_map': 'all-to-all',
        "basis_gates": 'u0,u1,u2,u3,cx,cz,id,x,y,z,h,s,sdg,t,tdg,rzz,' +
                       'snapshot,wait,noise,save,load'
    }

    def __init__(self, configuration=None, provider=None):
        super().__init__(configuration=configuration or self.DEFAULT_CONFIGURATION.copy(),
                         provider=provider)

        # Try to use the default executable if not specified.
        if self._configuration.get('exe'):
            paths = [self._configuration.get('exe')]
        else:
            paths = DEFAULT_SIMULATOR_PATHS

        # Ensure that the executable is available.
        try:
            self._configuration['exe'] = next(
                path for path in paths if (os.path.exists(path) and
                                           os.path.getsize(path) > 100))
        except StopIteration:
            raise FileNotFoundError('Simulator executable not found (using %s)' %
                                    self._configuration.get('exe', 'default locations'))

    def run(self, qobj):
        """Run a qobj on the backend."""
        job_id = str(uuid.uuid4())
        aer_job = AerJob(self, job_id, self._run_job, qobj)
        aer_job.submit()
        return aer_job

    def _run_job(self, job_id, qobj):
        self._validate(qobj)
        result = run(qobj, self._configuration['exe'])
        result['job_id'] = job_id
        copy_qasm_from_qobj_into_result(qobj, result)

        return result_from_old_style_dict(
            result, [circuit.header.name for circuit in qobj.experiments])

    def _validate(self, qobj):
        for experiment in qobj.experiments:
            if 'measure' not in [op.name for
                                 op in experiment.instructions]:
                logger.warning("no measurements in circuit '%s', "
                               "classical register will remain all zeros.",
                               experiment.header.name)

class CliffordSimulator(BaseBackend):
    """"C++ Clifford circuit simulator with realistic noise."""

    DEFAULT_CONFIGURATION = {
        'name': 'clifford_simulator',
        'url': 'https://github.com/QISKit/qiskit-terra/src/qasm-simulator',
        'simulator': True,
        'local': True,
        'description': 'A C++ Clifford simulator with approximate noise',
        'coupling_map': 'all-to-all',
        'basis_gates': 'cx,id,x,y,z,h,s,sdg,snapshot,wait,noise,save,load'
    }

    def __init__(self, configuration=None, provider=None):
        super().__init__(configuration=configuration or self.DEFAULT_CONFIGURATION.copy(),
                         provider=provider)

        # Try to use the default executable if not specified.
        if self._configuration.get('exe'):
            paths = [self._configuration.get('exe')]
        else:
            paths = DEFAULT_SIMULATOR_PATHS

        # Ensure that the executable is available.
        try:
            self._configuration['exe'] = next(
                path for path in paths if (os.path.exists(path) and
                                           os.path.getsize(path) > 100))
        except StopIteration:
            raise FileNotFoundError('Simulator executable not found (using %s)' %
                                    self._configuration.get('exe', 'default locations'))

    def run(self, qobj):
        """Run a Qobj on the backend.

        Args:
            qobj (dict): job description

        Returns:
            AerJob: derived from BaseJob
        """
        job_id = str(uuid.uuid4())
        aer_job = AerJob(self, job_id, self._run_job, qobj)
        aer_job.submit()
        return aer_job

    def _run_job(self, job_id, qobj):
        self._validate()
        # set backend to Clifford simulator
        if 'config' in qobj:
            qobj['config']['simulator'] = 'clifford'
        else:
            qobj['config'] = {'simulator': 'clifford'}

        result = run(qobj, self._configuration['exe'])
        result['job_id'] = job_id

        return result_from_old_style_dict(
            result, [circuit.header.name for circuit in qobj.experiments])

    def _validate(self):
        return

class QASMSimulatorEncoder(json.JSONEncoder):
    """
    JSON encoder for NumPy arrays and complex numbers.

    This functions as the standard JSON Encoder but adds support
    for encoding:

        * complex numbers z as lists [z.real, z.imag]
        * ndarrays as nested lists.
    """

    # pylint: disable=method-hidden,arguments-differ
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        if isinstance(obj, complex):
            return [obj.real, obj.imag]
        return json.JSONEncoder.default(self, obj)

class QASMSimulatorDecoder(json.JSONDecoder):
    """
    JSON decoder for the output from C++ qasm_simulator.

    This converts complex vectors and matrices into numpy arrays
    for the following keys.
    """
    def __init__(self, *args, **kwargs):
        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)

    # pylint: disable=method-hidden
    def object_hook(self, obj):
        """Special decoding rules for simulator output."""

        for key in ['U_error', 'density_matrix']:
            # JSON is a complex matrix
            if key in obj and isinstance(obj[key], list):
                tmp = np.array(obj[key])
                obj[key] = tmp[::, ::, 0] + 1j * tmp[::, ::, 1]
        for key in ['statevector', 'inner_products']:
            # JSON is a list of complex vectors
            if key in obj:
                for j in range(len(obj[key])):
                    if isinstance(obj[key][j], list):
                        tmp = np.array(obj[key][j])
                        obj[key][j] = tmp[::, 0] + 1j * tmp[::, 1]
        return obj

def run(qobj, executable):
    """
    Run simulation on C++ simulator inside a subprocess.

    Args:
        qobj (Qobj): qobj dictionary defining the simulation to run
        executable (string): filename (with path) of the simulator executable
    Returns:
        dict: A dict of simulation results
    """

    # Open subprocess and execute external command
    try:
        with subprocess.Popen([executable, '-'],
                              stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
            cin = json.dumps(qobj_to_dict(qobj, version='0.0.1'),
                             cls=QASMSimulatorEncoder).encode()
            cout, cerr = proc.communicate(cin)
        if cerr:
            logger.error('ERROR: Simulator encountered a runtime error: %s',
                         cerr.decode())
        sim_output = cout.decode()
        return json.loads(sim_output, cls=QASMSimulatorDecoder)

    except FileNotFoundError:
        msg = "ERROR: Simulator exe not found at: %s" % executable
        logger.error(msg)
        return {"status": msg, "success": False}

def cx_error_matrix(cal_error, zz_error):
    """
    Return the coherent error matrix for CR error model of a CNOT gate.

    Args:
        cal_error (double): calibration error of rotation
        zz_error (double): ZZ interaction term error

    Returns:
        numpy.ndarray: A coherent error matrix U_error for the CNOT gate.

    Details:

    The ideal cross-resonsance (CR) gate corresponds to a 2-qubit rotation
        U_CR_ideal = exp(-1j * (pi/2) * XZ/2)

    where qubit-0 is the control, and qubit-1 is the target. This can be
    converted to a CNOT gate by single-qubit rotations::

        U_CX = U_L * U_CR_ideal * U_R

    The noisy rotation is implemented as
        U_CR_noise = exp(-1j * (pi/2 + cal_error) * (XZ + zz_error ZZ)/2)

    The retured error matrix is given by
        U_error = U_L * U_CR_noise * U_R * U_CX^dagger
    """
    # pylint: disable=invalid-name
    if cal_error == 0 and zz_error == 0:
        return np.eye(4)

    cx_ideal = np.array([[1, 0, 0, 0],
                         [0, 0, 0, 1],
                         [0, 0, 1, 0],
                         [0, 1, 0, 0]])
    b = np.sqrt(1.0 + zz_error * zz_error)
    a = b * (np.pi / 2.0 + cal_error) / 2.0
    sp = (1.0 + 1j * zz_error) * np.sin(a) / b
    sm = (1.0 - 1j * zz_error) * np.sin(a) / b
    c = np.cos(a)
    cx_noise = np.array([[c + sm, 0, -1j * (c - sm), 0],
                         [0, 1j * (c - sm), 0, c + sm],
                         [-1j * (c - sp), 0, c + sp, 0],
                         [0, c + sp, 0, 1j * (c - sp)]]) / np.sqrt(2)
    return cx_noise.dot(cx_ideal.conj().T)

def x90_error_matrix(cal_error, detuning_error):
    """
    Return the coherent error matrix for a X90 rotation gate.

    Args:
        cal_error (double): calibration error of rotation
        detuning_error (double): detuning amount for rotation axis error

    Returns:
        numpy.ndarray: A coherent error matrix U_error for the X90 gate.

    Details:

    The ideal X90 rotation is a pi/2 rotation about the X-axis:
        U_X90_ideal = exp(-1j (pi/2) X/2)

    The noisy rotation is implemented as
        U_X90_noise = exp(-1j (pi/2 + cal_error) (cos(d) X + sin(d) Y)/2)

    where d is the detuning_error.

    The retured error matrix is given by
        U_error = U_X90_noise * U_X90_ideal^dagger
    """
    # pylint: disable=invalid-name
    if cal_error == 0 and detuning_error == 0:
        return np.eye(2)
    else:
        x90_ideal = np.array([[1., -1.j], [-1.j, 1]]) / np.sqrt(2)
        c = np.cos(0.5 * cal_error)
        s = np.sin(0.5 * cal_error)
        gamma = np.exp(-1j * detuning_error)
        x90_noise = np.array([[c - s, -1j * (c + s) * gamma],
                              [-1j * (c + s) * np.conj(gamma), c - s]]) / np.sqrt(2)
    return x90_noise.dot(x90_ideal.conj().T)

def _generate_coherent_error_matrix(config):
    """
    Generate U_error matrix for CX and X90 gates.

    Args:
        config (dict): the config of a qobj circuit

    This parses the config for the following noise parameter keys and returns a
    coherent error matrix for simulation coherent noise::

        * 'CX' gate: 'calibration_error', 'zz_error'
        * 'X90' gate: 'calibration_error', 'detuning_error'
    """
    # pylint: disable=invalid-name
    if 'noise_params' in config:
        # Check for CR coherent error parameters
        if 'CX' in config['noise_params']:
            noise_cx = config['noise_params']['CX']
            cal_error = noise_cx.pop('calibration_error', 0)
            zz_error = noise_cx.pop('zz_error', 0)
            # Add to current coherent error matrix
            if not cal_error == 0 or not zz_error == 0:
                u_error = noise_cx.get('U_error', np.eye(4))
                u_error = u_error.dot(cx_error_matrix(cal_error, zz_error))
                config['noise_params']['CX']['U_error'] = u_error
        # Check for X90 coherent error parameters
        if 'X90' in config['noise_params']:
            noise_x90 = config['noise_params']['X90']
            cal_error = noise_x90.pop('calibration_error', 0)
            detuning_error = noise_x90.pop('detuning_error', 0)
            # Add to current coherent error matrix
            if not cal_error == 0 or not detuning_error == 0:
                u_error = noise_x90.get('U_error', np.eye(2))
                u_error = u_error.dot(x90_error_matrix(cal_error,
                                                       detuning_error))
                config['noise_params']['X90']['U_error'] = u_error
<commit_msg>Fix clifford simulator run for qobj (#1125)<commit_after># -*- coding: utf-8 -*-

# Copyright 2017, IBM.
#
# This source code is licensed under the Apache License, Version 2.0 found in
# the LICENSE.txt file in the root directory of this source tree.

"""
Interface to C++ quantum circuit simulator with realistic noise.
"""

import uuid
import json
import logging
import os
import subprocess
from subprocess import PIPE
import platform

import numpy as np

from qiskit.result._utils import copy_qasm_from_qobj_into_result, result_from_old_style_dict
from qiskit.backends import BaseBackend
from qiskit.backends.aer.aerjob import AerJob
from qiskit.qobj import Qobj
from qiskit.qobj import qobj_to_dict

logger = logging.getLogger(__name__)

EXTENSION = '.exe' if platform.system() == 'Windows' else ''

# Add path to compiled qasm simulator
DEFAULT_SIMULATOR_PATHS = [
    # This is the path where Makefile creates the simulator by default
    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                 '../../../out/src/qasm-simulator-cpp/qasm_simulator_cpp'
                                 + EXTENSION)),
    # This is the path where PIP installs the simulator
    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                 'qasm_simulator_cpp' + EXTENSION)),
]

class QasmSimulator(BaseBackend):
    """C++ quantum circuit simulator with realistic noise"""

    DEFAULT_CONFIGURATION = {
        'name': 'qasm_simulator',
        'url': 'https://github.com/QISKit/qiskit-terra/src/qasm-simulator-cpp',
        'simulator': True,
        'local': True,
        'description': 'A C++ realistic noise simulator for qobj files',
        'coupling_map': 'all-to-all',
        "basis_gates": 'u0,u1,u2,u3,cx,cz,id,x,y,z,h,s,sdg,t,tdg,rzz,' +
                       'snapshot,wait,noise,save,load'
    }

    def __init__(self, configuration=None, provider=None):
        super().__init__(configuration=configuration or self.DEFAULT_CONFIGURATION.copy(),
                         provider=provider)

        # Try to use the default executable if not specified.
        if self._configuration.get('exe'):
            paths = [self._configuration.get('exe')]
        else:
            paths = DEFAULT_SIMULATOR_PATHS

        # Ensure that the executable is available.
        try:
            self._configuration['exe'] = next(
                path for path in paths if (os.path.exists(path) and
                                           os.path.getsize(path) > 100))
        except StopIteration:
            raise FileNotFoundError('Simulator executable not found (using %s)' %
                                    self._configuration.get('exe', 'default locations'))

    def run(self, qobj):
        """Run a qobj on the backend."""
        job_id = str(uuid.uuid4())
        aer_job = AerJob(self, job_id, self._run_job, qobj)
        aer_job.submit()
        return aer_job

    def _run_job(self, job_id, qobj):
        self._validate(qobj)
        result = run(qobj, self._configuration['exe'])
        result['job_id'] = job_id
        copy_qasm_from_qobj_into_result(qobj, result)

        return result_from_old_style_dict(
            result, [circuit.header.name for circuit in qobj.experiments])

    def _validate(self, qobj):
        for experiment in qobj.experiments:
            if 'measure' not in [op.name for
                                 op in experiment.instructions]:
                logger.warning("no measurements in circuit '%s', "
                               "classical register will remain all zeros.",
                               experiment.header.name)

class CliffordSimulator(BaseBackend):
    """"C++ Clifford circuit simulator with realistic noise."""

    DEFAULT_CONFIGURATION = {
        'name': 'clifford_simulator',
        'url': 'https://github.com/QISKit/qiskit-terra/src/qasm-simulator',
        'simulator': True,
        'local': True,
        'description': 'A C++ Clifford simulator with approximate noise',
        'coupling_map': 'all-to-all',
        'basis_gates': 'cx,id,x,y,z,h,s,sdg,snapshot,wait,noise,save,load'
    }

    def __init__(self, configuration=None, provider=None):
        super().__init__(configuration=configuration or self.DEFAULT_CONFIGURATION.copy(),
                         provider=provider)

        # Try to use the default executable if not specified.
        if self._configuration.get('exe'):
            paths = [self._configuration.get('exe')]
        else:
            paths = DEFAULT_SIMULATOR_PATHS

        # Ensure that the executable is available.
        try:
            self._configuration['exe'] = next(
                path for path in paths if (os.path.exists(path) and
                                           os.path.getsize(path) > 100))
        except StopIteration:
            raise FileNotFoundError('Simulator executable not found (using %s)' %
                                    self._configuration.get('exe', 'default locations'))

    def run(self, qobj):
        """Run a Qobj on the backend.

        Args:
            qobj (dict): job description

        Returns:
            AerJob: derived from BaseJob
        """
        job_id = str(uuid.uuid4())
        aer_job = AerJob(self, job_id, self._run_job, qobj)
        aer_job.submit()
        return aer_job

    def _run_job(self, job_id, qobj):
        if isinstance(qobj, Qobj):
            qobj_dict = qobj.as_dict()
        else:
            qobj_dict = qobj
        self._validate()
        # set backend to Clifford simulator
        if 'config' in qobj_dict:
            qobj_dict['config']['simulator'] = 'clifford'
        else:
            qobj_dict['config'] = {'simulator': 'clifford'}

        qobj = Qobj.from_dict(qobj_dict)
        result = run(qobj, self._configuration['exe'])
        result['job_id'] = job_id

        return result_from_old_style_dict(
            result, [circuit.header.name for circuit in qobj.experiments])

    def _validate(self):
        return

class QASMSimulatorEncoder(json.JSONEncoder):
    """
    JSON encoder for NumPy arrays and complex numbers.

    This functions as the standard JSON Encoder but adds support
    for encoding:

        * complex numbers z as lists [z.real, z.imag]
        * ndarrays as nested lists.
    """

    # pylint: disable=method-hidden,arguments-differ
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        if isinstance(obj, complex):
            return [obj.real, obj.imag]
        return json.JSONEncoder.default(self, obj)

class QASMSimulatorDecoder(json.JSONDecoder):
    """
    JSON decoder for the output from C++ qasm_simulator.

    This converts complex vectors and matrices into numpy arrays
    for the following keys.
    """
    def __init__(self, *args, **kwargs):
        json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)

    # pylint: disable=method-hidden
    def object_hook(self, obj):
        """Special decoding rules for simulator output."""

        for key in ['U_error', 'density_matrix']:
            # JSON is a complex matrix
            if key in obj and isinstance(obj[key], list):
                tmp = np.array(obj[key])
                obj[key] = tmp[::, ::, 0] + 1j * tmp[::, ::, 1]
        for key in ['statevector', 'inner_products']:
            # JSON is a list of complex vectors
            if key in obj:
                for j in range(len(obj[key])):
                    if isinstance(obj[key][j], list):
                        tmp = np.array(obj[key][j])
                        obj[key][j] = tmp[::, 0] + 1j * tmp[::, 1]
        return obj

def run(qobj, executable):
    """
    Run simulation on C++ simulator inside a subprocess.

    Args:
        qobj (Qobj): qobj dictionary defining the simulation to run
        executable (string): filename (with path) of the simulator executable
    Returns:
        dict: A dict of simulation results
    """

    # Open subprocess and execute external command
    try:
        with subprocess.Popen([executable, '-'],
                              stdin=PIPE, stdout=PIPE, stderr=PIPE) as proc:
            cin = json.dumps(qobj_to_dict(qobj, version='0.0.1'),
                             cls=QASMSimulatorEncoder).encode()
            cout, cerr = proc.communicate(cin)
        if cerr:
            logger.error('ERROR: Simulator encountered a runtime error: %s',
                         cerr.decode())
        sim_output = cout.decode()
        return json.loads(sim_output, cls=QASMSimulatorDecoder)

    except FileNotFoundError:
        msg = "ERROR: Simulator exe not found at: %s" % executable
        logger.error(msg)
        return {"status": msg, "success": False}

def cx_error_matrix(cal_error, zz_error):
    """
    Return the coherent error matrix for CR error model of a CNOT gate.

    Args:
        cal_error (double): calibration error of rotation
        zz_error (double): ZZ interaction term error

    Returns:
        numpy.ndarray: A coherent error matrix U_error for the CNOT gate.

    Details:

    The ideal cross-resonsance (CR) gate corresponds to a 2-qubit rotation
        U_CR_ideal = exp(-1j * (pi/2) * XZ/2)

    where qubit-0 is the control, and qubit-1 is the target. This can be
    converted to a CNOT gate by single-qubit rotations::

        U_CX = U_L * U_CR_ideal * U_R

    The noisy rotation is implemented as
        U_CR_noise = exp(-1j * (pi/2 + cal_error) * (XZ + zz_error ZZ)/2)

    The retured error matrix is given by
        U_error = U_L * U_CR_noise * U_R * U_CX^dagger
    """
    # pylint: disable=invalid-name
    if cal_error == 0 and zz_error == 0:
        return np.eye(4)

    cx_ideal = np.array([[1, 0, 0, 0],
                         [0, 0, 0, 1],
                         [0, 0, 1, 0],
                         [0, 1, 0, 0]])
    b = np.sqrt(1.0 + zz_error * zz_error)
    a = b * (np.pi / 2.0 + cal_error) / 2.0
    sp = (1.0 + 1j * zz_error) * np.sin(a) / b
    sm = (1.0 - 1j * zz_error) * np.sin(a) / b
    c = np.cos(a)
    cx_noise = np.array([[c + sm, 0, -1j * (c - sm), 0],
                         [0, 1j * (c - sm), 0, c + sm],
                         [-1j * (c - sp), 0, c + sp, 0],
                         [0, c + sp, 0, 1j * (c - sp)]]) / np.sqrt(2)
    return cx_noise.dot(cx_ideal.conj().T)

def x90_error_matrix(cal_error, detuning_error):
    """
    Return the coherent error matrix for a X90 rotation gate.

    Args:
        cal_error (double): calibration error of rotation
        detuning_error (double): detuning amount for rotation axis error

    Returns:
        numpy.ndarray: A coherent error matrix U_error for the X90 gate.

    Details:

    The ideal X90 rotation is a pi/2 rotation about the X-axis:
        U_X90_ideal = exp(-1j (pi/2) X/2)

    The noisy rotation is implemented as
        U_X90_noise = exp(-1j (pi/2 + cal_error) (cos(d) X + sin(d) Y)/2)

    where d is the detuning_error.

    The retured error matrix is given by
        U_error = U_X90_noise * U_X90_ideal^dagger
    """
    # pylint: disable=invalid-name
    if cal_error == 0 and detuning_error == 0:
        return np.eye(2)
    else:
        x90_ideal = np.array([[1., -1.j], [-1.j, 1]]) / np.sqrt(2)
        c = np.cos(0.5 * cal_error)
        s = np.sin(0.5 * cal_error)
        gamma = np.exp(-1j * detuning_error)
        x90_noise = np.array([[c - s, -1j * (c + s) * gamma],
                              [-1j * (c + s) * np.conj(gamma), c - s]]) / np.sqrt(2)
    return x90_noise.dot(x90_ideal.conj().T)

def _generate_coherent_error_matrix(config):
    """
    Generate U_error matrix for CX and X90 gates.

    Args:
        config (dict): the config of a qobj circuit

    This parses the config for the following noise parameter keys and returns a
    coherent error matrix for simulation coherent noise::

        * 'CX' gate: 'calibration_error', 'zz_error'
        * 'X90' gate: 'calibration_error', 'detuning_error'
    """
    # pylint: disable=invalid-name
    if 'noise_params' in config:
        # Check for CR coherent error parameters
        if 'CX' in config['noise_params']:
            noise_cx = config['noise_params']['CX']
            cal_error = noise_cx.pop('calibration_error', 0)
            zz_error = noise_cx.pop('zz_error', 0)
            # Add to current coherent error matrix
            if not cal_error == 0 or not zz_error == 0:
                u_error = noise_cx.get('U_error', np.eye(4))
                u_error = u_error.dot(cx_error_matrix(cal_error, zz_error))
                config['noise_params']['CX']['U_error'] = u_error
        # Check for X90 coherent error parameters
        if 'X90' in config['noise_params']:
            noise_x90 = config['noise_params']['X90']
            cal_error = noise_x90.pop('calibration_error', 0)
            detuning_error = noise_x90.pop('detuning_error', 0)
            # Add to current coherent error matrix
            if not cal_error == 0 or not detuning_error == 0:
                u_error = noise_x90.get('U_error', np.eye(2))
                u_error = u_error.dot(x90_error_matrix(cal_error,
                                                       detuning_error))
                config['noise_params']['X90']['U_error'] = u_error
<|endoftext|>


<commit_before>import importlib

from django.conf import settings
from django.db import models
from django.utils.http import urlencode
from django.urls import reverse
from django.core.paginator import Paginator
from django.utils.safestring import mark_safe

from django_elasticsearch_dsl import Index
from django_elasticsearch_dsl.registries import registry
from django_elasticsearch_dsl.signals import RealTimeSignalProcessor

from elasticsearch_dsl import analyzer, tokenizer, A
from elasticsearch_dsl.query import Q

from .tasks import (
    search_instance_save, search_instance_pre_delete,
    search_instance_delete
)

def get_index(name):
    index = Index('%s_%s' % (
        settings.ELASTICSEARCH_INDEX_PREFIX,
        name
    ))

    # See Elasticsearch Indices API reference for available settings
    index.settings(
        number_of_shards=1,
        number_of_replicas=0
    )
    return index

def get_pagination_vars(data):
    d = data.copy()
    d.pop('page', None)
    return '&' + urlencode(d)

def get_default_text_analyzer():
    return analyzer(
        'froide_analyzer',
        tokenizer='standard',
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_default_ngram_analyzer():
    return analyzer(
        'froide_ngram_analyzer',
        tokenizer=tokenizer(
            'froide_ngram_tokenzier',
            type='edge_ngram',
            min_gram=1,
            max_gram=15,
            token_chars=['letter', 'digit']
        ),
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_func(config_name, default_func):
    def get_it():
        from django.conf import settings
        func_path = settings.FROIDE_CONFIG.get(config_name, None)
        if not func_path:
            return default_func()

        module, func = func_path.rsplit('.', 1)
        module = importlib.import_module(module)
        analyzer_func = getattr(module, func)
        return analyzer_func()
    return get_it

get_text_analyzer = get_func('search_text_analyzer', get_default_text_analyzer)
get_ngram_analyzer = get_func('search_ngram_analyzer', get_default_ngram_analyzer)

def make_filter_url(url_name, data, get_active_filters=None):
    data = dict(data)
    url_kwargs = {}

    if get_active_filters is None:
        active_filters = {}
    else:
        active_filters = get_active_filters(data)

    for key in active_filters:
        url_kwargs[key] = data.pop(key)

    query_string = ''
    data = {k: v for k, v in data.items() if v}
    if data:
        query_string = '?' + urlencode(data)
    return reverse(url_name, kwargs=url_kwargs) + query_string

def get_facet_with_label(info, model=None, attr='name'):
    pks = [item['key'] for item in info['buckets']]
    objs = {
        o.pk: o for o in model.objects.filter(pk__in=pks)
    }
    for item in info['buckets']:
        yield {
            'label': getattr(objs[item['key']], attr),
            'id': item['key'],
            'count': item['doc_count']
        }

def key_getter(item):
    return item['key']

def resolve_facet(data, getter=None, label_getter=None, model=None, make_url=None):
    if getter is None:
        getter = key_getter

    if label_getter is None:
        label_getter = key_getter

    def resolve(key, info):
        if model is not None:
            pks = [item['key'] for item in info['buckets']]
            objs = {
                o.pk: o for o in model.objects.filter(pk__in=pks)
            }
            for item in info['buckets']:
                item['object'] = objs[item['key']]
        for item in info['buckets']:
            item['active'] = getter(item) == data.get(key)
            item['label'] = label_getter(item)
            d = data.copy()
            d[key] = getter(item)
            item['url'] = make_url(d)
            d.pop(key)
            item['clear_url'] = make_url(d)
        return info
    return resolve

def _make_values_lists(kwargs):
    return {
        k: v if isinstance(v, (list, tuple)) else [v]
        for k, v in kwargs.items()
    }

class EmtpyResponse(list):
    class hits:
        total = 0

class SearchQuerySetWrapper(object):
    """
    Decorates a SearchQuerySet object using a generator for efficient iteration
    """
    def __init__(self, sqs, model):
        self.sqs = sqs
        self.sqs.model = model
        self.model = model
        self.filters = []
        self.query = None
        self.aggs = []

    def count(self):
        return self.response.hits.total

    def to_queryset(self):
        return self.sqs.to_queryset()

    def wrap_queryset(self, qs):
        return ESQuerySetWrapper(qs, self.response)

    def update_query(self):
        if self.filters:
            self.sqs.post_filter = Q('bool', must=self.filters)

    def all(self):
        return self

    @property
    def response(self):
        if not hasattr(self.sqs, '_response'):
            self.sqs = self.sqs.source(excludes=['*'])
        try:
            return self.sqs.execute()
        except Exception:
            return EmtpyResponse()

    def add_aggregation(self, aggs):
        for field in aggs:
            a = A('terms', field=field)
            self.sqs.aggs.bucket(field, a)
        return self

    def get_facets(self, resolvers=None):
        if resolvers is None:
            resolvers = {}
        return {
            k: resolvers[k](k, self.response['aggregations'][k])
            for k in self.response['aggregations']
            if k in resolvers
        }

    def get_aggregations(self):
        return {'fields': {
            k: [
                [i['key'], i['doc_count']]
                for i in self.response['aggregations'][k]['buckets']
            ]
            for k in self.response['aggregations']
        }}

    def filter(self, *args, **kwargs):
        if kwargs:
            value = Q('terms', **_make_values_lists(kwargs))
        else:
            value = args[0]
        self.filters.append(value)
        self.update_query()
        return self

    def set_query(self, q):
        self.query = q
        self.sqs = self.sqs.query(self.query)
        return self

    def add_sort(self, *sorts):
        self.sqs = self.sqs.sort(*sorts)
        return self

    def __getitem__(self, key):
        self.sqs = self.sqs[key]
        return self

    def __iter__(self):
        return iter(self.sqs)

class ESQuerySetWrapper(object):
    def __init__(self, qs, es_response):
        self.__class__ = type(
            qs.__class__.__name__,
            (self.__class__, qs.__class__),
            {}
        )
        self.__dict__ = qs.__dict__
        self._qs = qs
        self._es_response = es_response
        self._es_map = {
            int(hit.meta.id): hit for hit in es_response
        }

    def __iter__(self):
        for obj in self._qs:
            hit = self._es_map[obj.pk]
            # mark_safe should work because highlight_options
            # has been set with encoder="html"
            obj.es_highlight = mark_safe(' '.join(self._get_highlight(hit)))
            yield obj

    def _get_highlight(self, hit):
        if hasattr(hit.meta, 'highlight'):
            for key in hit.meta.highlight:
                yield from hit.meta.highlight[key]

class ElasticsearchPaginator(Paginator):
    """
    Paginator that prevents two queries to ES (for count and objects)
    as ES gives count with objects
    """

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        bottom = (number - 1) * self.per_page
        top = bottom + self.per_page
        self.object_list = self.object_list[bottom:top]

        # ignore top boundary
        # if top + self.orphans >= self.count:
        #     top = self.count

        # Validate number after limit/offset has been set
        number = self.validate_number(number)
        return self._get_page(self.object_list, number, self)

class CelerySignalProcessor(RealTimeSignalProcessor):
    def setup(self):
        for model in registry.get_models():
            models.signals.post_save.connect(self.handle_save, sender=model)
            models.signals.post_delete.connect(self.handle_delete, sender=model)

            models.signals.m2m_changed.connect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.connect(self.handle_pre_delete, sender=model)

    def teardown(self):
        # Listen to all model saves.
        for model in registry.get_models():
            models.signals.post_save.disconnect(self.handle_save, sender=model)
            models.signals.post_delete.disconnect(self.handle_delete, sender=model)
            models.signals.m2m_changed.disconnect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.disconnect(self.handle_pre_delete, sender=model)

    def handle_save(self, sender, instance, **kwargs):
        """Handle save.

        Given an individual model instance, update the object in the index.
        Update the related objects either.
        """
        search_instance_save.delay(instance)

    def handle_pre_delete(self, sender, instance, **kwargs):
        """Handle removing of instance object from related models instance.
        We need to do this before the real delete otherwise the relation
        doesn't exists anymore and we can't get the related models instance.
        """
        search_instance_pre_delete.delay(instance)

    def handle_delete(self, sender, instance, **kwargs):
        """Handle delete.

        Given an individual model instance, delete the object from index.
        """
        search_instance_delete.delay(instance)
<commit_msg>Check pagination offset against ES max<commit_after>import importlib

from django.conf import settings
from django.db import models
from django.utils.http import urlencode
from django.urls import reverse
from django.core.paginator import Paginator
from django.utils.safestring import mark_safe

from django_elasticsearch_dsl import Index
from django_elasticsearch_dsl.registries import registry
from django_elasticsearch_dsl.signals import RealTimeSignalProcessor

from elasticsearch_dsl import analyzer, tokenizer, A
from elasticsearch_dsl.query import Q

from .tasks import (
    search_instance_save, search_instance_pre_delete,
    search_instance_delete
)

def get_index(name):
    index = Index('%s_%s' % (
        settings.ELASTICSEARCH_INDEX_PREFIX,
        name
    ))

    # See Elasticsearch Indices API reference for available settings
    index.settings(
        number_of_shards=1,
        number_of_replicas=0
    )
    return index

def get_pagination_vars(data):
    d = data.copy()
    d.pop('page', None)
    return '&' + urlencode(d)

def get_default_text_analyzer():
    return analyzer(
        'froide_analyzer',
        tokenizer='standard',
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_default_ngram_analyzer():
    return analyzer(
        'froide_ngram_analyzer',
        tokenizer=tokenizer(
            'froide_ngram_tokenzier',
            type='edge_ngram',
            min_gram=1,
            max_gram=15,
            token_chars=['letter', 'digit']
        ),
        filter=[
            'standard',
            'lowercase',
            'asciifolding',
        ]
    )

def get_func(config_name, default_func):
    def get_it():
        from django.conf import settings
        func_path = settings.FROIDE_CONFIG.get(config_name, None)
        if not func_path:
            return default_func()

        module, func = func_path.rsplit('.', 1)
        module = importlib.import_module(module)
        analyzer_func = getattr(module, func)
        return analyzer_func()
    return get_it

get_text_analyzer = get_func('search_text_analyzer', get_default_text_analyzer)
get_ngram_analyzer = get_func('search_ngram_analyzer', get_default_ngram_analyzer)

def make_filter_url(url_name, data, get_active_filters=None):
    data = dict(data)
    url_kwargs = {}

    if get_active_filters is None:
        active_filters = {}
    else:
        active_filters = get_active_filters(data)

    for key in active_filters:
        url_kwargs[key] = data.pop(key)

    query_string = ''
    data = {k: v for k, v in data.items() if v}
    if data:
        query_string = '?' + urlencode(data)
    return reverse(url_name, kwargs=url_kwargs) + query_string

def get_facet_with_label(info, model=None, attr='name'):
    pks = [item['key'] for item in info['buckets']]
    objs = {
        o.pk: o for o in model.objects.filter(pk__in=pks)
    }
    for item in info['buckets']:
        yield {
            'label': getattr(objs[item['key']], attr),
            'id': item['key'],
            'count': item['doc_count']
        }

def key_getter(item):
    return item['key']

def resolve_facet(data, getter=None, label_getter=None, model=None, make_url=None):
    if getter is None:
        getter = key_getter

    if label_getter is None:
        label_getter = key_getter

    def resolve(key, info):
        if model is not None:
            pks = [item['key'] for item in info['buckets']]
            objs = {
                o.pk: o for o in model.objects.filter(pk__in=pks)
            }
            for item in info['buckets']:
                item['object'] = objs[item['key']]
        for item in info['buckets']:
            item['active'] = getter(item) == data.get(key)
            item['label'] = label_getter(item)
            d = data.copy()
            d[key] = getter(item)
            item['url'] = make_url(d)
            d.pop(key)
            item['clear_url'] = make_url(d)
        return info
    return resolve

def _make_values_lists(kwargs):
    return {
        k: v if isinstance(v, (list, tuple)) else [v]
        for k, v in kwargs.items()
    }

class EmtpyResponse(list):
    class hits:
        total = 0

class SearchQuerySetWrapper(object):
    """
    Decorates a SearchQuerySet object using a generator for efficient iteration
    """
    def __init__(self, sqs, model):
        self.sqs = sqs
        self.sqs.model = model
        self.model = model
        self.filters = []
        self.query = None
        self.aggs = []

    def count(self):
        return self.response.hits.total

    def to_queryset(self):
        return self.sqs.to_queryset()

    def wrap_queryset(self, qs):
        return ESQuerySetWrapper(qs, self.response)

    def update_query(self):
        if self.filters:
            self.sqs.post_filter = Q('bool', must=self.filters)

    def all(self):
        return self

    @property
    def response(self):
        if not hasattr(self.sqs, '_response'):
            self.sqs = self.sqs.source(excludes=['*'])
        try:
            return self.sqs.execute()
        except Exception:
            return EmtpyResponse()

    def add_aggregation(self, aggs):
        for field in aggs:
            a = A('terms', field=field)
            self.sqs.aggs.bucket(field, a)
        return self

    def get_facets(self, resolvers=None):
        if resolvers is None:
            resolvers = {}
        return {
            k: resolvers[k](k, self.response['aggregations'][k])
            for k in self.response['aggregations']
            if k in resolvers
        }

    def get_aggregations(self):
        return {'fields': {
            k: [
                [i['key'], i['doc_count']]
                for i in self.response['aggregations'][k]['buckets']
            ]
            for k in self.response['aggregations']
        }}

    def filter(self, *args, **kwargs):
        if kwargs:
            value = Q('terms', **_make_values_lists(kwargs))
        else:
            value = args[0]
        self.filters.append(value)
        self.update_query()
        return self

    def set_query(self, q):
        self.query = q
        self.sqs = self.sqs.query(self.query)
        return self

    def add_sort(self, *sorts):
        self.sqs = self.sqs.sort(*sorts)
        return self

    def __getitem__(self, key):
        self.sqs = self.sqs[key]
        return self

    def __iter__(self):
        return iter(self.sqs)

class ESQuerySetWrapper(object):
    def __init__(self, qs, es_response):
        self.__class__ = type(
            qs.__class__.__name__,
            (self.__class__, qs.__class__),
            {}
        )
        self.__dict__ = qs.__dict__
        self._qs = qs
        self._es_response = es_response
        self._es_map = {
            int(hit.meta.id): hit for hit in es_response
        }

    def __iter__(self):
        for obj in self._qs:
            hit = self._es_map[obj.pk]
            # mark_safe should work because highlight_options
            # has been set with encoder="html"
            obj.es_highlight = mark_safe(' '.join(self._get_highlight(hit)))
            yield obj

    def _get_highlight(self, hit):
        if hasattr(hit.meta, 'highlight'):
            for key in hit.meta.highlight:
                yield from hit.meta.highlight[key]

class ElasticsearchPaginator(Paginator):
    """
    Paginator that prevents two queries to ES (for count and objects)
    as ES gives count with objects
    """
    MAX_ES_OFFSET = 10000

    def page(self, number):
        """
        Returns a Page object for the given 1-based page number.
        """
        bottom = (number - 1) * self.per_page
        if bottom >= self.MAX_ES_OFFSET:
            # Only validate if bigger than offset
            number = self.validate_number(number)
            bottom = (number - 1) * self.per_page

        top = bottom + self.per_page
        self.object_list = self.object_list[bottom:top]

        # ignore top boundary
        # if top + self.orphans >= self.count:
        #     top = self.count

        # Validate number after limit/offset has been set
        number = self.validate_number(number)
        return self._get_page(self.object_list, number, self)

class CelerySignalProcessor(RealTimeSignalProcessor):
    def setup(self):
        for model in registry.get_models():
            models.signals.post_save.connect(self.handle_save, sender=model)
            models.signals.post_delete.connect(self.handle_delete, sender=model)

            models.signals.m2m_changed.connect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.connect(self.handle_pre_delete, sender=model)

    def teardown(self):
        # Listen to all model saves.
        for model in registry.get_models():
            models.signals.post_save.disconnect(self.handle_save, sender=model)
            models.signals.post_delete.disconnect(self.handle_delete, sender=model)
            models.signals.m2m_changed.disconnect(self.handle_m2m_changed, sender=model)
            models.signals.pre_delete.disconnect(self.handle_pre_delete, sender=model)

    def handle_save(self, sender, instance, **kwargs):
        """Handle save.

        Given an individual model instance, update the object in the index.
        Update the related objects either.
        """
        search_instance_save.delay(instance)

    def handle_pre_delete(self, sender, instance, **kwargs):
        """Handle removing of instance object from related models instance.
        We need to do this before the real delete otherwise the relation
        doesn't exists anymore and we can't get the related models instance.
        """
        search_instance_pre_delete.delay(instance)

    def handle_delete(self, sender, instance, **kwargs):
        """Handle delete.

        Given an individual model instance, delete the object from index.
        """
        search_instance_delete.delay(instance)
<|endoftext|>


<commit_before>import json

from django.apps import AppConfig
from django.urls import reverse
from django.utils.translation import gettext_lazy as _

class FroideCampaignConfig(AppConfig):
    name = 'froide_campaign'
    verbose_name = _("Froide Campaign App")

    def ready(self):
        from .listeners import connect_info_object

        from froide.foirequest.models import FoiRequest
        FoiRequest.request_created.connect(connect_info_object)

        from froide.account.menu import menu_registry, MenuItem
        from froide.account.export import registry
        from froide.account import account_merged

        registry.register(export_user_data)
        account_merged.connect(merge_user)

        @menu_registry.register
        def get_campaign_menu_item(request):
            if not request.user.has_perm('froide_campaign.can_use_campaigns'):
                return None
            return MenuItem(
                section='before_settings', order=10,
                url=reverse('campaign-list'),
                label=_('My campaigns')
            )

def merge_user(sender, old_user=None, new_user=None, **kwargs):
    from froide.account.utils import move_ownership
    from .models import CampaignPage

    move_ownership(CampaignPage, 'user', old_user, new_user)

def export_user_data(user):
    from froide.foirequest.models.request import get_absolute_domain_short_url
    from .models import CampaignPage, InformationObject

    campaign_pages = (
        CampaignPage.objects
        .filter(user=user)
    )
    if campaign_pages:
        yield ('campaign_pages.json', json.dumps([
            {
                'title': p.title,
                'slug': p.slug,
                'team_id': p.team_id,
                'settings': p.settings,
                'embed': p.get_absolute_domain_embed_url(),
                'public': p.public,
                'description': p.description,
                'campaigns': [c.id for c in p.campaigns.all()]
            }
            for p in campaign_pages]).encode('utf-8')
        )

    iobjs = InformationObject.objects.filter(
        foirequest__user=user
    ).select_related('campaign')
    if iobjs:
        yield ('campaign_requests.json', json.dumps([
            {
                'campaign': i.campaign.title,
                'ident': i.ident,
                'title': i.title,
                'slug': i.slug,
                'ordering': i.ordering,
                'context': i.context,
                'request': (
                    get_absolute_domain_short_url(i.foirequest_id)
                    if i.foirequest_id else None
                ),
                'resolved': i.resolved,
                'resolution_text': i.resolution_text,
                'resolution_link': i.resolution_link,
            }
            for i in iobjs]).encode('utf-8')
        )
<commit_msg>Use new signal to connect request to campaign<commit_after>import json

from django.apps import AppConfig
from django.urls import reverse
from django.utils.translation import gettext_lazy as _

class FroideCampaignConfig(AppConfig):
    name = 'froide_campaign'
    verbose_name = _("Froide Campaign App")

    def ready(self):
        from .listeners import connect_info_object

        from froide.foirequest.models import FoiRequest
        FoiRequest.request_sent.connect(connect_info_object)

        from froide.account.menu import menu_registry, MenuItem
        from froide.account.export import registry
        from froide.account import account_merged

        registry.register(export_user_data)
        account_merged.connect(merge_user)

        @menu_registry.register
        def get_campaign_menu_item(request):
            if not request.user.has_perm('froide_campaign.can_use_campaigns'):
                return None
            return MenuItem(
                section='before_settings', order=10,
                url=reverse('campaign-list'),
                label=_('My campaigns')
            )

def merge_user(sender, old_user=None, new_user=None, **kwargs):
    from froide.account.utils import move_ownership
    from .models import CampaignPage

    move_ownership(CampaignPage, 'user', old_user, new_user)

def export_user_data(user):
    from froide.foirequest.models.request import get_absolute_domain_short_url
    from .models import CampaignPage, InformationObject

    campaign_pages = (
        CampaignPage.objects
        .filter(user=user)
    )
    if campaign_pages:
        yield ('campaign_pages.json', json.dumps([
            {
                'title': p.title,
                'slug': p.slug,
                'team_id': p.team_id,
                'settings': p.settings,
                'embed': p.get_absolute_domain_embed_url(),
                'public': p.public,
                'description': p.description,
                'campaigns': [c.id for c in p.campaigns.all()]
            }
            for p in campaign_pages]).encode('utf-8')
        )

    iobjs = InformationObject.objects.filter(
        foirequest__user=user
    ).select_related('campaign')
    if iobjs:
        yield ('campaign_requests.json', json.dumps([
            {
                'campaign': i.campaign.title,
                'ident': i.ident,
                'title': i.title,
                'slug': i.slug,
                'ordering': i.ordering,
                'context': i.context,
                'request': (
                    get_absolute_domain_short_url(i.foirequest_id)
                    if i.foirequest_id else None
                ),
                'resolved': i.resolved,
                'resolution_text': i.resolution_text,
                'resolution_link': i.resolution_link,
            }
            for i in iobjs]).encode('utf-8')
        )
<|endoftext|>


<commit_before>"""\
Includes tests that check the setup for the tests.
If the library is compiled and if there is a js interpreter.
"""
import os
import sys
if sys.version_info < (2, 7):
    import unittest2 as unittest
else:
    import unittest
import tempfile
class EnviromentTest(unittest.TestCase):
    "Test case that makes sure that the environment is up and working"
    def reportProgres(self):
        """Should be overloaded by the test result class"""

    def stop(self):
        """Should be overloaded by the test result class"""

    def runTest(self):
        """The actual test goes here."""
        if os.system(
            "js --help > %s" %
            os.path.join(
                tempfile.gettempdir(),
                tempfile.gettempprefix()
                )
            ):
            self.stop()
            raise RuntimeError("""Can't find the "js" command.""")
        self.reportProgres()
        if not os.path.exists("py-builtins.js"):
            self.stop()
            raise RuntimeError("""Can't find the "py-builtins.js" command.""")
        self.reportProgres()

    def __str__(self):
        return 'Looking for "js" and "py-builtins" [2]: '

<commit_msg> * Changed the test for "js" detection so it works with both Mozilla and Google javascript engines<commit_after>"""\
Includes tests that check the setup for the tests.
If the library is compiled and if there is a js interpreter.
"""
import os
import sys
if sys.version_info < (2, 7):
    import unittest2 as unittest
else:
    import unittest
import tempfile
class EnviromentTest(unittest.TestCase):
    "Test case that makes sure that the environment is up and working"
    def reportProgres(self):
        """Should be overloaded by the test result class"""

    def stop(self):
        """Should be overloaded by the test result class"""

    def runTest(self):
        """The actual test goes here."""
        if os.system(
            "echo | js > %s" %
            os.path.join(
                tempfile.gettempdir(),
                tempfile.gettempprefix()
                )
            ):
            self.stop()
            raise RuntimeError("""Can't find the "js" command.""")
        self.reportProgres()
        if not os.path.exists("py-builtins.js"):
            self.stop()
            raise RuntimeError("""Can't find the "py-builtins.js" command.""")
        self.reportProgres()

    def __str__(self):
        return 'Looking for "js" and "py-builtins" [2]: '

<|endoftext|>


<commit_before>import os
# Tell gevent not to patch os.waitpid() since it is susceptible to race
# conditions. See:
# http://www.gevent.org/gevent.monkey.html#gevent.monkey.patch_os
os.environ['GEVENT_NOWAITPID'] = 'true'

# Use manhole to give us a way to debug hung processes
# https://pypi.python.org/pypi/manhole
import manhole
manhole.install(
    verbose=False,
    # Listen for SIGUSR1
    oneshot_on="USR1"
)
from gevent import monkey
monkey.patch_all(
    dns=False,
    # Don't patch subprocess to avoid http://tracker.ceph.com/issues/14990
    subprocess=False,
)
import sys
from gevent.hub import Hub

# Don't write pyc files
sys.dont_write_bytecode = True

from .orchestra import monkey
monkey.patch_all()

import logging
import subprocess

__version__ = '1.0.0'

# do our best, but if it fails, continue with above

try:
    __version__ += '-' + subprocess.check_output(
        'git rev-parse --short HEAD'.split(),
        cwd=os.path.dirname(os.path.realpath(__file__))
    ).strip()
except Exception as e:
    # before logging; should be unusual
    print >>sys.stderr, 'Can\'t get version from git rev-parse', e

# If we are running inside a virtualenv, ensure we have its 'bin' directory in
# our PATH. This doesn't happen automatically if scripts are called without
# first activating the virtualenv.
exec_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
if os.path.split(exec_dir)[-1] == 'bin' and exec_dir not in os.environ['PATH']:
    os.environ['PATH'] = ':'.join((exec_dir, os.environ['PATH']))

# We don't need to see log entries for each connection opened
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(
    logging.WARN)
# if requests doesn't bundle it, shut it up anyway
logging.getLogger('urllib3.connectionpool').setLevel(
    logging.WARN)

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s')
log = logging.getLogger(__name__)

log.debug('teuthology version: %s', __version__)

def setup_log_file(log_path):
    root_logger = logging.getLogger()
    handlers = root_logger.handlers
    for handler in handlers:
        if isinstance(handler, logging.FileHandler) and \
                handler.stream.name == log_path:
            log.debug("Already logging to %s; not adding new handler",
                      log_path)
            return
    formatter = logging.Formatter(
        fmt=u'%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
        datefmt='%Y-%m-%dT%H:%M:%S')
    handler = logging.FileHandler(filename=log_path)
    handler.setFormatter(formatter)
    root_logger.addHandler(handler)
    root_logger.info('teuthology version: %s', __version__)

def install_except_hook():
    """
    Install an exception hook that first logs any uncaught exception, then
    raises it.
    """
    def log_exception(exc_type, exc_value, exc_traceback):
        if not issubclass(exc_type, KeyboardInterrupt):
            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
                                                         exc_traceback))
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
    sys.excepthook = log_exception

def patch_gevent_hub_error_handler():
    Hub._origin_handle_error = Hub.handle_error

    def custom_handle_error(self, context, type, value, tb):
        if not issubclass(type, Hub.SYSTEM_ERROR + Hub.NOT_ERROR):
            log.error("Uncaught exception (Hub)", exc_info=(type, value, tb))

        self._origin_handle_error(context, type, value, tb)

    Hub.handle_error = custom_handle_error

patch_gevent_hub_error_handler()
<commit_msg>only pass system errors to hub handler<commit_after>import os
# Tell gevent not to patch os.waitpid() since it is susceptible to race
# conditions. See:
# http://www.gevent.org/gevent.monkey.html#gevent.monkey.patch_os
os.environ['GEVENT_NOWAITPID'] = 'true'

# Use manhole to give us a way to debug hung processes
# https://pypi.python.org/pypi/manhole
import manhole
manhole.install(
    verbose=False,
    # Listen for SIGUSR1
    oneshot_on="USR1"
)
from gevent import monkey
monkey.patch_all(
    dns=False,
    # Don't patch subprocess to avoid http://tracker.ceph.com/issues/14990
    subprocess=False,
)
import sys
from gevent.hub import Hub

# Don't write pyc files
sys.dont_write_bytecode = True

from .orchestra import monkey
monkey.patch_all()

import logging
import subprocess

__version__ = '1.0.0'

# do our best, but if it fails, continue with above

try:
    __version__ += '-' + subprocess.check_output(
        'git rev-parse --short HEAD'.split(),
        cwd=os.path.dirname(os.path.realpath(__file__))
    ).strip()
except Exception as e:
    # before logging; should be unusual
    print >>sys.stderr, 'Can\'t get version from git rev-parse', e

# If we are running inside a virtualenv, ensure we have its 'bin' directory in
# our PATH. This doesn't happen automatically if scripts are called without
# first activating the virtualenv.
exec_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
if os.path.split(exec_dir)[-1] == 'bin' and exec_dir not in os.environ['PATH']:
    os.environ['PATH'] = ':'.join((exec_dir, os.environ['PATH']))

# We don't need to see log entries for each connection opened
logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(
    logging.WARN)
# if requests doesn't bundle it, shut it up anyway
logging.getLogger('urllib3.connectionpool').setLevel(
    logging.WARN)

logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s')
log = logging.getLogger(__name__)

log.debug('teuthology version: %s', __version__)

def setup_log_file(log_path):
    root_logger = logging.getLogger()
    handlers = root_logger.handlers
    for handler in handlers:
        if isinstance(handler, logging.FileHandler) and \
                handler.stream.name == log_path:
            log.debug("Already logging to %s; not adding new handler",
                      log_path)
            return
    formatter = logging.Formatter(
        fmt=u'%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(message)s',
        datefmt='%Y-%m-%dT%H:%M:%S')
    handler = logging.FileHandler(filename=log_path)
    handler.setFormatter(formatter)
    root_logger.addHandler(handler)
    root_logger.info('teuthology version: %s', __version__)

def install_except_hook():
    """
    Install an exception hook that first logs any uncaught exception, then
    raises it.
    """
    def log_exception(exc_type, exc_value, exc_traceback):
        if not issubclass(exc_type, KeyboardInterrupt):
            log.critical("Uncaught exception", exc_info=(exc_type, exc_value,
                                                         exc_traceback))
        sys.__excepthook__(exc_type, exc_value, exc_traceback)
    sys.excepthook = log_exception

def patch_gevent_hub_error_handler():
    Hub._origin_handle_error = Hub.handle_error

    def custom_handle_error(self, context, type, value, tb):
        if context is None or issubclass(type, Hub.SYSTEM_ERROR):
            self.handle_system_error(type, value)
        elif issubclass(type, Hub.NOT_ERROR):
            pass
        else:
            log.error("Uncaught exception (Hub)", exc_info=(type, value, tb))

    Hub.handle_error = custom_handle_error

patch_gevent_hub_error_handler()
<|endoftext|>


<commit_before>import contextlib
import logging

log = logging.getLogger(__name__)

def default_image_name(role):
    return 'testimage.{role}'.format(role=role)

@contextlib.contextmanager
def create_image(ctx, config):
    """
    Create an rbd image.

    For example::

        tasks:
        - ceph:
        - rbd.create_image:
            client.0:
                image_name: testimage
                image_size: 100
            client.1:
    """
    assert isinstance(config, dict) or isinstance(config, list), \
        "task create_image only supports a list or dictionary for configuration"

    if isinstance(config, dict):
        images = config.items()
    else:
        images = [(role, None) for role in config]

    for role, properties in images:
        if properties is None:
            properties = {}
        name = properties.get('image_name', default_image_name(role))
        size = properties.get('image_size', 1024)
        (remote,) = ctx.cluster.only(role).remotes.keys()
        log.info('Creating image {name} with size {size}'.format(name=name,
                                                                 size=size))
        remote.run(
            args=[
                'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib',
                '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                '/tmp/cephtest/archive/coverage',
                '/tmp/cephtest/binary/usr/local/bin/rbd',
                '-c', '/tmp/cephtest/ceph.conf',
                '-p', 'rbd',
                'create',
                '-s', str(size),
                name,
                ],
            )
    try:
        yield
    finally:
        log.info('Deleting rbd images...')
        for role, properties in images:
            if properties is None:
                properties = {}
            name = properties.get('image_name', default_image_name(role))
            (remote,) = ctx.cluster.only(role).remotes.keys()
            remote.run(
                args=[
                    'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib',
                    '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                    '/tmp/cephtest/archive/coverage',
                    '/tmp/cephtest/binary/usr/local/bin/rbd',
                    '-c', '/tmp/cephtest/ceph.conf',
                    '-p', 'rbd',
                    'rm',
                    name,
                    ],
                )

@contextlib.contextmanager
def task(ctx, config):
    create_image(ctx, config)
<commit_msg>Add rbd modprobe method.<commit_after>import contextlib
import logging

log = logging.getLogger(__name__)

def default_image_name(role):
    return 'testimage.{role}'.format(role=role)

@contextlib.contextmanager
def create_image(ctx, config):
    """
    Create an rbd image.

    For example::

        tasks:
        - ceph:
        - rbd.create_image:
            client.0:
                image_name: testimage
                image_size: 100
            client.1:
    """
    assert isinstance(config, dict) or isinstance(config, list), \
        "task create_image only supports a list or dictionary for configuration"

    if isinstance(config, dict):
        images = config.items()
    else:
        images = [(role, None) for role in config]

    for role, properties in images:
        if properties is None:
            properties = {}
        name = properties.get('image_name', default_image_name(role))
        size = properties.get('image_size', 1024)
        (remote,) = ctx.cluster.only(role).remotes.keys()
        log.info('Creating image {name} with size {size}'.format(name=name,
                                                                 size=size))
        remote.run(
            args=[
                'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib',
                '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                '/tmp/cephtest/archive/coverage',
                '/tmp/cephtest/binary/usr/local/bin/rbd',
                '-c', '/tmp/cephtest/ceph.conf',
                '-p', 'rbd',
                'create',
                '-s', str(size),
                name,
                ],
            )
    try:
        yield
    finally:
        log.info('Deleting rbd images...')
        for role, properties in images:
            if properties is None:
                properties = {}
            name = properties.get('image_name', default_image_name(role))
            (remote,) = ctx.cluster.only(role).remotes.keys()
            remote.run(
                args=[
                    'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib',
                    '/tmp/cephtest/binary/usr/local/bin/ceph-coverage',
                    '/tmp/cephtest/archive/coverage',
                    '/tmp/cephtest/binary/usr/local/bin/rbd',
                    '-c', '/tmp/cephtest/ceph.conf',
                    '-p', 'rbd',
                    'rm',
                    name,
                    ],
                )

@contextlib.contextmanager
def modprobe(ctx, config):
    """
    Load the rbd kernel module..

    For example::

        tasks:
        - ceph:
        - rbd.create_image: [client.0]
        - rbd.modprobe: [client.0]
    """
    for role in config:
        (remote,) = ctx.cluster.only(role).remotes.keys()
        remote.run(
            args=[
                'sudo',
                'modprobe',
                'rbd',
                ],
            )
    try:
        yield
    finally:
        log.info('Unloading rbd kernel module...')
        for role in config:
            (remote,) = ctx.cluster.only(role).remotes.keys()
            remote.run(
                args=[
                    'sudo',
                    'modprobe',
                    '-r',
                    'rbd',
                    ],
                )

@contextlib.contextmanager
def task(ctx, config):
    create_image(ctx, config)
<|endoftext|>


<commit_before>from django.conf.urls.defaults import *
from django.views.generic.simple import direct_to_template
from django.conf import settings
from userprofile.views import get_profiles

urlpatterns = patterns('',

    # Demo FrontPage$
    (r'^$', direct_to_template, {'extra_context': { 'hola': 'adios', 'profiles': get_profiles }, 'template': 'front.html' }),

    # Profile application
    (r'^accounts/', include('userprofile.urls')),

    # Serves media content. WARNING!! Only for development uses. On production use lighthttpd for media content.
    (r'^site_media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': '%s/../media/' % settings.PROJECT_PATH}),

    # Admin (not really needed)
    (r'^admin/', include('django.contrib.admin.urls')),

)
<commit_msg>little cleanup<commit_after>from django.conf.urls.defaults import *
from django.views.generic.simple import direct_to_template
from django.conf import settings
from userprofile.views import get_profiles

urlpatterns = patterns('',

    # Demo FrontPage$
    (r'^$', direct_to_template, {'extra_context': { 'profiles': get_profiles }, 'template': 'front.html' }),

    # Profile application
    (r'^accounts/', include('userprofile.urls')),

    # Serves media content. WARNING!! Only for development uses. On production use lighthttpd for media content.
    (r'^site_media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': '%s/../media/' % settings.PROJECT_PATH}),

    # Admin (not really needed)
    (r'^admin/', include('django.contrib.admin.urls')),

)
<|endoftext|>