<commit_before>#!/usr/bin/env python3

from subprocess import Popen, PIPE
from tempfile import mkstemp
from io import StringIO
from warnings import warn
import numpy as np
import pytest
import cirq

import sys, os
sys.path.insert(
    1, os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + '/../../'))
from python import utils
from python import qflex

circuit_test = """16
0 h 0
0 h 1
0 h 2
0 h 3
0 h 4
0 h 5
0 h 6
0 h 7
0 h 8
0 h 9
0 h 10
0 h 11
0 h 12
0 h 13
0 h 14
0 h 15
1 cz 0 1
1 cz 6 7
1 cz 8 9
1 cz 14 15
1 t 2
1 t 3
1 t 4
1 t 5
1 t 10
1 t 11
1 t 12
1 t 13
2 cz 4 8
2 cz 6 10
2 y_1_2 0
2 y_1_2 1
2 x_1_2 7
2 x_1_2 9
2 y_1_2 14
2 x_1_2 15
3 cz 1 2
3 cz 9 10
3 t 0
3 y_1_2 4
3 x_1_2 6
3 t 7
3 x_1_2 8
3 t 14
3 t 15
4 cz 0 4
4 cz 9 13
4 cz 2 6
4 cz 11 15
4 y_1_2 1
4 t 8
4 y_1_2 10
5 cz 2 3
5 cz 4 5
5 cz 10 11
5 cz 12 13
5 y_1_2 0
5 t 1
5 x_1_2 6
5 y_1_2 9
5 y_1_2 15
6 cz 5 9
6 cz 7 11
6 t 0
6 x_1_2 2
6 y_1_2 3
6 y_1_2 4
6 t 6
6 y_1_2 10
6 y_1_2 12
6 x_1_2 13
6 t 15
7 cz 5 6
7 cz 13 14
7 t 2
7 t 3
7 t 4
7 y_1_2 7
7 y_1_2 9
7 t 10
7 y_1_2 11
7 t 12
8 cz 8 12
8 cz 1 5
8 cz 10 14
8 cz 3 7
8 x_1_2 6
8 t 9
8 t 11
8 x_1_2 13
9 cz 0 1
9 cz 6 7
9 cz 8 9
9 cz 14 15
9 x_1_2 3
9 y_1_2 5
9 y_1_2 10
9 y_1_2 12
9 t 13
10 cz 4 8
10 cz 6 10
10 x_1_2 0
10 y_1_2 1
10 t 3
10 t 5
10 x_1_2 7
10 y_1_2 9
10 t 12
10 y_1_2 14
10 x_1_2 15
11 cz 1 2
11 cz 9 10
11 t 0
11 y_1_2 4
11 x_1_2 6
11 t 7
11 y_1_2 8
11 t 14
11 t 15
12 cz 0 4
12 cz 9 13
12 cz 2 6
12 cz 11 15
12 y_1_2 1
12 t 8
12 x_1_2 10
13 cz 2 3
13 cz 4 5
13 cz 10 11
13 cz 12 13
13 x_1_2 0
13 t 1
13 y_1_2 6
13 x_1_2 9
13 x_1_2 15
14 cz 5 9
14 cz 7 11
14 t 0
14 y_1_2 2
14 x_1_2 3
14 y_1_2 4
14 t 6
14 x_1_2 10
14 y_1_2 12
14 y_1_2 13
14 t 15
15 cz 5 6
15 cz 13 14
15 t 2
15 t 3
15 t 4
15 x_1_2 7
15 x_1_2 9
15 t 10
15 x_1_2 11
15 t 12
16 cz 8 12
16 cz 1 5
16 cz 10 14
16 cz 3 7
16 y_1_2 6
16 t 9
16 t 11
16 y_1_2 13
17 h 0
17 h 1
17 h 2
17 h 3
17 h 4
17 h 5
17 h 6
17 h 7
17 h 8
17 h 9
17 h 10
17 h 11
17 h 12
17 h 13
17 h 14
17 h 15
"""

grid_test = """1111
1111
1111
1111
"""

ordering_test = """
cut () 4 8
cut () 5 9
cut () 6 10

expand A 0
expand A 1
expand A 2
expand A 3
expand A 4
expand A 5
expand A 6
expand A 7

expand B 8
expand B 9
expand B 10
expand B 11
expand B 12
expand B 13
expand B 14
expand B 15

merge A B
"""

ordering_with_cuts_test = """
cut () 4 8
cut () 5 9
cut () 6 10

expand A 0
expand A 1
expand A 2
expand A 3
expand A 4
expand A 5
expand A 6
expand A 7

expand B 8
expand B 9
expand B 10
expand B 11
expand B 12
expand B 13
expand B 14
expand B 15

merge A B
"""

# Simulate circuit
qubits = utils.GetGridQubits(StringIO(grid_test))
circuit = utils.GetCircuit(StringIO(circuit_test), qubits)
results = cirq.Simulator().simulate(circuit)

# Save circuit and grid on temporary files
circuit_filename = mkstemp()
grid_filename = mkstemp()
ordering_filename = mkstemp()
ordering_with_cuts_filename = mkstemp()

with open(circuit_filename[1], 'w') as f:
    print(circuit_test, file=f)
with open(grid_filename[1], 'w') as f:
    print(grid_test, file=f)
with open(ordering_filename[1], 'w') as f:
    print(ordering_test, file=f)
with open(ordering_with_cuts_filename[1], 'w') as f:
    print(ordering_with_cuts_test, file=f)

@pytest.mark.parametrize(
    'x', [np.random.randint(0, 2**len(qubits)) for _ in range(50)])
def test_simulation(x):

    # Get configuration as a string
    final_conf = bin(x)[2:].zfill(len(qubits))

    options = {
        'circuit_filename': circuit_filename[1],
        'ordering_filename': ordering_filename[1],
        'grid_filename': grid_filename[1],
        'final_state': final_conf
    }

    # Get output from qFlex
    qflex_amplitude = qflex.simulate(options)[0][1]

    # Compare the amplitudes
    assert (np.abs(results.final_state[x] - qflex_amplitude) < 1.e-6)

@pytest.mark.parametrize(
    'x', [np.random.randint(0, 2**len(qubits)) for _ in range(50)])
def test_simulation_with_cuts(x):

    # Get configuration as a string
    final_conf = bin(x)[2:].zfill(len(qubits))

    options = {
        'circuit_filename': circuit_filename[1],
        'ordering_filename': ordering_with_cuts_filename[1],
        'grid_filename': grid_filename[1],
        'final_state': final_conf
    }

    # Get output from qFlex
    qflex_amplitude = qflex.simulate(options)[0][1]

    # Compare the amplitudes
    assert (np.abs(results.final_state[x] - qflex_amplitude) < 1.e-6)
<commit_msg>Add test for auto-ordering.<commit_after>#!/usr/bin/env python3

from subprocess import Popen, PIPE
from tempfile import mkstemp
from io import StringIO
from warnings import warn
import numpy as np
import pytest
import cirq

import sys, os
sys.path.insert(
    1, os.path.realpath(os.path.dirname(os.path.realpath(__file__)) + '/../../'))
from python.ordering import order_circuit_simulation as auto_order
from python import utils
from python import qflex

num_runs = 50

circuit_test = """16
0 h 0
0 h 1
0 h 2
0 h 3
0 h 4
0 h 5
0 h 6
0 h 7
0 h 8
0 h 9
0 h 10
0 h 11
0 h 12
0 h 13
0 h 14
0 h 15
1 cz 0 1
1 cz 6 7
1 cz 8 9
1 cz 14 15
1 t 2
1 t 3
1 t 4
1 t 5
1 t 10
1 t 11
1 t 12
1 t 13
2 cz 4 8
2 cz 6 10
2 y_1_2 0
2 y_1_2 1
2 x_1_2 7
2 x_1_2 9
2 y_1_2 14
2 x_1_2 15
3 cz 1 2
3 cz 9 10
3 t 0
3 y_1_2 4
3 x_1_2 6
3 t 7
3 x_1_2 8
3 t 14
3 t 15
4 cz 0 4
4 cz 9 13
4 cz 2 6
4 cz 11 15
4 y_1_2 1
4 t 8
4 y_1_2 10
5 cz 2 3
5 cz 4 5
5 cz 10 11
5 cz 12 13
5 y_1_2 0
5 t 1
5 x_1_2 6
5 y_1_2 9
5 y_1_2 15
6 cz 5 9
6 cz 7 11
6 t 0
6 x_1_2 2
6 y_1_2 3
6 y_1_2 4
6 t 6
6 y_1_2 10
6 y_1_2 12
6 x_1_2 13
6 t 15
7 cz 5 6
7 cz 13 14
7 t 2
7 t 3
7 t 4
7 y_1_2 7
7 y_1_2 9
7 t 10
7 y_1_2 11
7 t 12
8 cz 8 12
8 cz 1 5
8 cz 10 14
8 cz 3 7
8 x_1_2 6
8 t 9
8 t 11
8 x_1_2 13
9 cz 0 1
9 cz 6 7
9 cz 8 9
9 cz 14 15
9 x_1_2 3
9 y_1_2 5
9 y_1_2 10
9 y_1_2 12
9 t 13
10 cz 4 8
10 cz 6 10
10 x_1_2 0
10 y_1_2 1
10 t 3
10 t 5
10 x_1_2 7
10 y_1_2 9
10 t 12
10 y_1_2 14
10 x_1_2 15
11 cz 1 2
11 cz 9 10
11 t 0
11 y_1_2 4
11 x_1_2 6
11 t 7
11 y_1_2 8
11 t 14
11 t 15
12 cz 0 4
12 cz 9 13
12 cz 2 6
12 cz 11 15
12 y_1_2 1
12 t 8
12 x_1_2 10
13 cz 2 3
13 cz 4 5
13 cz 10 11
13 cz 12 13
13 x_1_2 0
13 t 1
13 y_1_2 6
13 x_1_2 9
13 x_1_2 15
14 cz 5 9
14 cz 7 11
14 t 0
14 y_1_2 2
14 x_1_2 3
14 y_1_2 4
14 t 6
14 x_1_2 10
14 y_1_2 12
14 y_1_2 13
14 t 15
15 cz 5 6
15 cz 13 14
15 t 2
15 t 3
15 t 4
15 x_1_2 7
15 x_1_2 9
15 t 10
15 x_1_2 11
15 t 12
16 cz 8 12
16 cz 1 5
16 cz 10 14
16 cz 3 7
16 y_1_2 6
16 t 9
16 t 11
16 y_1_2 13
17 h 0
17 h 1
17 h 2
17 h 3
17 h 4
17 h 5
17 h 6
17 h 7
17 h 8
17 h 9
17 h 10
17 h 11
17 h 12
17 h 13
17 h 14
17 h 15
"""

grid_test = """1111
1111
1111
1111
"""

ordering_test = """
cut () 4 8
cut () 5 9
cut () 6 10

expand A 0
expand A 1
expand A 2
expand A 3
expand A 4
expand A 5
expand A 6
expand A 7

expand B 8
expand B 9
expand B 10
expand B 11
expand B 12
expand B 13
expand B 14
expand B 15

merge A B
"""

ordering_with_cuts_test = """
cut () 4 8
cut () 5 9
cut () 6 10

expand A 0
expand A 1
expand A 2
expand A 3
expand A 4
expand A 5
expand A 6
expand A 7

expand B 8
expand B 9
expand B 10
expand B 11
expand B 12
expand B 13
expand B 14
expand B 15

merge A B
"""

# Simulate circuit
qubits = utils.GetGridQubits(StringIO(grid_test))
circuit = utils.GetCircuit(StringIO(circuit_test), qubits)
auto_ordering = auto_order.circuit_to_ordering(circuit, qubit_names=sorted(qubits))
results = cirq.Simulator().simulate(circuit)

# Save circuit and grid on temporary files
circuit_filename = mkstemp()
grid_filename = mkstemp()
ordering_filename = mkstemp()
ordering_with_cuts_filename = mkstemp()
ordering_auto_filename = mkstemp()

with open(circuit_filename[1], 'w') as f:
    print(circuit_test, file=f)
with open(grid_filename[1], 'w') as f:
    print(grid_test, file=f)
with open(ordering_filename[1], 'w') as f:
    print(ordering_test, file=f)
with open(ordering_with_cuts_filename[1], 'w') as f:
    print(ordering_with_cuts_test, file=f)
with open(ordering_auto_filename[1], 'w') as f:
    print('\n'.join(auto_ordering), file=f)

@pytest.mark.parametrize(
    'x', [np.random.randint(0, 2**len(qubits)) for _ in range(num_runs)])
def test_simulation(x):

    # Get configuration as a string
    final_conf = bin(x)[2:].zfill(len(qubits))

    options = {
        'circuit_filename': circuit_filename[1],
        'ordering_filename': ordering_filename[1],
        'grid_filename': grid_filename[1],
        'final_state': final_conf
    }

    # Get output from qFlex
    qflex_amplitude = qflex.simulate(options)[0][1]

    # Compare the amplitudes
    assert (np.abs(results.final_state[x] - qflex_amplitude) < 1.e-6)

@pytest.mark.parametrize(
    'x', [np.random.randint(0, 2**len(qubits)) for _ in range(num_runs)])
def test_simulation_with_cuts(x):

    # Get configuration as a string
    final_conf = bin(x)[2:].zfill(len(qubits))

    options = {
        'circuit_filename': circuit_filename[1],
        'ordering_filename': ordering_with_cuts_filename[1],
        'grid_filename': grid_filename[1],
        'final_state': final_conf
    }

    # Get output from qFlex
    qflex_amplitude = qflex.simulate(options)[0][1]

    # Compare the amplitudes
    assert (np.abs(results.final_state[x] - qflex_amplitude) < 1.e-6)

@pytest.mark.parametrize(
    'x', [np.random.randint(0, 2**len(qubits)) for _ in range(num_runs)])
def test_simulation_with_auto_order(x):

    # Get configuration as a string
    final_conf = bin(x)[2:].zfill(len(qubits))

    options = {
        'circuit_filename': circuit_filename[1],
        'ordering_filename': ordering_auto_filename[1],
        'grid_filename': grid_filename[1],
        'final_state': final_conf
    }

    # Get output from qFlex
    qflex_amplitude = qflex.simulate(options)[0][1]

    # Compare the amplitudes
    assert (np.abs(results.final_state[x] - qflex_amplitude) < 1.e-6)
<|endoftext|>


<commit_before>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib.auth.models import User
from django.utils import simplejson
from django.utils import simplejson
from django.contrib.auth.decorators import login_required

from dajaxice.decorators import dajaxice_register
from dajax.core import Dajax

from survey.models import *
from survey.forms import SurveyQuestionForm, SurveyQuestionNewForm, SurveyResponseForm

def audio_msg_type(audio_message):
    if audio_message is None:
        return 2 # Text2Speech
    else:
        return 1 # Audio File

@login_required
@dajaxice_register
def survey_question_add_update(request, id, data, form_type, after_initial_save, record_id):
    """ Ajax method to update the question """
    dajax = Dajax()

    if form_type == 'old_form':
        form = SurveyQuestionForm(data)

        if after_initial_save == 1 and record_id:
            survey_question = SurveyQuestion.objects.get(pk=int(record_id))
        else:
            survey_question = SurveyQuestion.objects.get(pk=int(id))

        form = SurveyQuestionForm(data, instance=survey_question)

    if form_type == 'new_form':
        form = SurveyQuestionNewForm(request.user, data)

    if form.is_valid():
        if form_type == 'old_form':
            object = form.save(commit=False)
            audio_message = form.cleaned_data.get('audio_message')
            object.message_type = audio_msg_type(audio_message)
            object.save()

        if form_type == 'new_form':
            obj = form.save(commit=False)
            obj.user = User.objects.get(username=request.user)
            obj.surveyapp = form.cleaned_data.get('surveyapp')
            audio_message = form.cleaned_data.get('audio_message')
            obj.message_type = audio_msg_type(audio_message)
            obj.type = form.cleaned_data.get('type')
            obj.save()
            dajax.assign('#id', 'value', obj.id)
        #dajax.alert("%s is successfully saved !!" % form.cleaned_data.get('question'))
    else:
        if form_type == 'new_form':
            dajax.remove_css_class('#new_survey_que_form_'+id+' input','error')
        if form_type == 'old_form':
            dajax.remove_css_class('#que_form_'+id+' input','error')
        dajax.alert("error")
    return dajax.json()

@login_required
@dajaxice_register
def survey_response_add_update(request, id, que_id, data, form_type, new_response_id):
    """ Ajax method to update the response """
    dajax = Dajax()

    # 1st check survey question id
    if int(que_id) != 0:
        try:
            SurveyQuestion.objects.get(pk=que_id)
        except:
            #dajax.alert("error : First Save your survey question !!")
            return dajax.json()

    if int(que_id) == 0:
        #dajax.alert("error : First Save your survey question !!")
        return dajax.json()

    if form_type == 'old_form':
        form = SurveyResponseForm(data)
        survey_response = SurveyResponse.objects.get(pk=int(id))
        form = SurveyResponseForm(data, instance=survey_response)

    if form_type == 'new_form':
        form = SurveyResponseForm(data)

        if int(new_response_id) != 0:
            survey_response = SurveyResponse.objects.get(pk=int(new_response_id))
            form = SurveyResponseForm(data, instance=survey_response)

    if form.is_valid():
        key = form.cleaned_data.get("key")
        keyvalue = form.cleaned_data.get('keyvalue')

        duplicate_count = 0
        # start checking of duplicate key
        if form_type == 'old_form':
            resp_obj = SurveyResponse.objects.get(pk=id)
            if resp_obj.key != key:
                duplicate_count = SurveyResponse.objects.filter(key=key,
                                                    surveyquestion=resp_obj.surveyquestion).count()

        if form_type == 'new_form':
            if que_id == 0: # without que id
                # get last inserted record
                surveyquestion = SurveyQuestion.objects.all().order_by('-id')[0]
            else: # with que id
                surveyquestion = SurveyQuestion.objects.get(pk=que_id)
            duplicate_count = SurveyResponse.objects.filter(key=key,
                                                    surveyquestion=surveyquestion).count()

        if duplicate_count >= 1 and form_type == 'old_form':
            #dajax.alert("error : (%s) duplicate record key & previous key (%s) is not changed!" % (key,
            #                                                                                resp_obj.key))
            return dajax.json()

        if duplicate_count >= 1 and form_type == 'new_form' and int(new_response_id) == 0:
            #dajax.alert("error : (%s) duplicate record key !" % (key))
            return dajax.json()

        # end checking of duplicate key

        if form_type == 'old_form':
            form.save()
        if form_type == 'new_form':
            obj = form.save(commit=False)
            obj.surveyquestion = surveyquestion
            obj.save()
            dajax.assign('#new_response_id', 'value', obj.id)

        #dajax.alert("(%s - %s) is successfully saved !!" % (form.cleaned_data.get('key'),
        #                                                   form.cleaned_data.get('keyvalue')))
    else:
        if form_type == 'new_form':
            dajax.remove_css_class('#new_survey_que_form_'+id+' input','error')
        if form_type == 'old_form':
            dajax.remove_css_class('#que_form_'+id+' input','error')
        dajax.alert("error")
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error,'error')

    return dajax.json()

@login_required
@dajaxice_register
def survey_question_delete(request, id):
    """Method to delete the question & its responses"""
    dajax = Dajax()
    try:
        survey_question = SurveyQuestion.objects.get(pk=int(id))
        que = survey_question.question
        survey_response_list = SurveyResponse.objects.filter(surveyquestion=survey_question)
        for survey_resp in survey_response_list:
            survey_resp.delete()
        survey_question.delete()
        #dajax.alert("(%s) is successfully deleted !!" % (que))
    except:
        dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error,'error')
    return dajax.json()

@login_required
@dajaxice_register
def survey_response_delete(request, id):
    """Method to delete responses"""
    dajax = Dajax()
    try:
        survey_response = SurveyResponse.objects.get(pk=int(id))
        key = survey_response.key
        keyvalue = survey_response.keyvalue
        survey_response.delete()
        #dajax.alert("(%s - %s) is successfully deleted !!" % (key, keyvalue))
    except:
        dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error,'error')
    return dajax.json()

@login_required
@dajaxice_register
def survey_question_sort(request, id, sort_order):
    dajax = Dajax()

    try:
        survey_question = SurveyQuestion.objects.get(pk=int(id))
        survey_question.order = sort_order
        survey_question.save()
        #dajax.alert("(%s) has been successfully sorted !!" % (survey_question.question))
    except:
        #dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error,'error')
    return dajax.json()
<commit_msg>clean code<commit_after>#
# Newfies-Dialer License
# http://www.newfies-dialer.org
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Copyright (C) 2011-2012 Star2Billing S.L.
#
# The Initial Developer of the Original Code is
# Arezqui Belaid <info@star2billing.com>
#

from django.contrib.auth.models import User
from django.contrib.auth.decorators import login_required

from dajaxice.decorators import dajaxice_register
from dajax.core import Dajax

from survey.models import *
from survey.forms import SurveyQuestionForm, \
                        SurveyQuestionNewForm, \
                        SurveyResponseForm

def audio_msg_type(audio_message):
    if audio_message is None:
        return 2  # Text2Speech
    else:
        return 1  # Audio File

@login_required
@dajaxice_register
def survey_question_add_update(request, id, data, form_type,
    after_initial_save, record_id):
    """ Ajax method to update the question """
    dajax = Dajax()

    if form_type == 'old_form':
        form = SurveyQuestionForm(data)

        if after_initial_save == 1 and record_id:
            survey_question = SurveyQuestion.objects.get(pk=int(record_id))
        else:
            survey_question = SurveyQuestion.objects.get(pk=int(id))

        form = SurveyQuestionForm(data, instance=survey_question)

    if form_type == 'new_form':
        form = SurveyQuestionNewForm(request.user, data)

    if form.is_valid():
        if form_type == 'old_form':
            object = form.save(commit=False)
            audio_message = form.cleaned_data.get('audio_message')
            object.message_type = audio_msg_type(audio_message)
            object.save()

        if form_type == 'new_form':
            obj = form.save(commit=False)
            obj.user = User.objects.get(username=request.user)
            obj.surveyapp = form.cleaned_data.get('surveyapp')
            audio_message = form.cleaned_data.get('audio_message')
            obj.message_type = audio_msg_type(audio_message)
            obj.type = form.cleaned_data.get('type')
            obj.save()
            dajax.assign('#id', 'value', obj.id)
        #dajax.alert("%s is successfully saved !!" % \
        #    form.cleaned_data.get('question'))
    else:
        if form_type == 'new_form':
            dajax.remove_css_class('#new_survey_que_form_' + id + ' input', \
                                    'error')
        if form_type == 'old_form':
            dajax.remove_css_class('#que_form_' + id + ' input', 'error')
        dajax.alert("error")
    return dajax.json()

@login_required
@dajaxice_register
def survey_response_add_update(request, id, que_id, data,
    form_type, new_response_id):
    """ Ajax method to update the response """
    dajax = Dajax()

    # 1st check survey question id
    if int(que_id) != 0:
        try:
            SurveyQuestion.objects.get(pk=que_id)
        except:
            #dajax.alert("error : First Save your survey question !!")
            return dajax.json()

    if int(que_id) == 0:
        #dajax.alert("error : First Save your survey question !!")
        return dajax.json()

    if form_type == 'old_form':
        form = SurveyResponseForm(data)
        survey_response = SurveyResponse.objects.get(pk=int(id))
        form = SurveyResponseForm(data, instance=survey_response)

    if form_type == 'new_form':
        form = SurveyResponseForm(data)

        if int(new_response_id) != 0:
            survey_response = SurveyResponse.objects\
                                    .get(pk=int(new_response_id))
            form = SurveyResponseForm(data, instance=survey_response)

    if form.is_valid():
        key = form.cleaned_data.get("key")
        keyvalue = form.cleaned_data.get('keyvalue')
        #TODO: Check keyvalue, this vars is never used

        duplicate_count = 0
        # start checking of duplicate key
        if form_type == 'old_form':
            resp_obj = SurveyResponse.objects.get(pk=id)
            if resp_obj.key != key:
                duplicate_count = SurveyResponse.objects\
                                    .filter(key=key,
                                        surveyquestion=resp_obj.surveyquestion)\
                                    .count()

        if form_type == 'new_form':
            if que_id == 0:  # without que id
                # get last inserted record
                surveyquestion = SurveyQuestion.objects\
                                        .all().order_by('-id')[0]
            else:  # with que id
                surveyquestion = SurveyQuestion.objects.get(pk=que_id)
            duplicate_count = SurveyResponse.objects.filter(key=key,
                                    surveyquestion=surveyquestion).count()

        if duplicate_count >= 1 and form_type == 'old_form':
            #dajax.alert("error : (%s) duplicate record key & previous key (%s) is not changed!" % (key,
            #                                                                                resp_obj.key))
            return dajax.json()

        if duplicate_count >= 1 \
            and form_type == 'new_form' \
            and int(new_response_id) == 0:
            #dajax.alert("error : (%s) duplicate record key !" % (key))
            return dajax.json()

        # end checking of duplicate key

        if form_type == 'old_form':
            form.save()
        if form_type == 'new_form':
            obj = form.save(commit=False)
            obj.surveyquestion = surveyquestion
            obj.save()
            dajax.assign('#new_response_id', 'value', obj.id)

        # dajax.alert("(%s - %s) is successfully saved !!" % \
        #        (form.cleaned_data.get('key'),
        #        form.cleaned_data.get('keyvalue')))
    else:
        if form_type == 'new_form':
            dajax.remove_css_class('#new_survey_que_form_' + id + ' input', \
                                    'error')
        if form_type == 'old_form':
            dajax.remove_css_class('#que_form_' + id + ' input', 'error')
        dajax.alert("error")
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error, 'error')

    return dajax.json()

@login_required
@dajaxice_register
def survey_question_delete(request, id):
    """Method to delete the question & its responses"""
    dajax = Dajax()
    try:
        survey_question = SurveyQuestion.objects.get(pk=int(id))
        survey_response_list = SurveyResponse.objects\
                        .filter(surveyquestion=survey_question)
        for survey_resp in survey_response_list:
            survey_resp.delete()
        survey_question.delete()
        # dajax.alert("(%s) is successfully deleted !!" % \
        #    (survey_question.question))
    except:
        dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error, 'error')
    return dajax.json()

@login_required
@dajaxice_register
def survey_response_delete(request, id):
    """Method to delete responses"""
    dajax = Dajax()
    try:
        survey_response = SurveyResponse.objects.get(pk=int(id))
        survey_response.delete()
        # key = survey_response.key
        # keyvalue = survey_response.keyvalue
        # dajax.alert("(%s - %s) is successfully deleted !!" % (key, keyvalue))
    except:
        dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error, 'error')
    return dajax.json()

@login_required
@dajaxice_register
def survey_question_sort(request, id, sort_order):
    dajax = Dajax()

    try:
        survey_question = SurveyQuestion.objects.get(pk=int(id))
        survey_question.order = sort_order
        survey_question.save()
        # dajax.alert("(%s) has been successfully sorted !!" % \
        #    (survey_question.question))
    except:
        #dajax.alert("%s is not exist !!" % (id))
        for error in form.errors:
            dajax.add_css_class('#id_%s' % error, 'error')
    return dajax.json()
<|endoftext|>


<commit_before>#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#
import os
import gevent
import logging
import kazoo.client
import kazoo.exceptions
import kazoo.handlers.gevent
import kazoo.recipe.election
from kazoo.client import KazooState
from kazoo.retry import KazooRetry

from bitarray import bitarray
from cfgm_common.exceptions import ResourceExhaustionError, ResourceExistsError
from gevent.coros import BoundedSemaphore

import uuid

LOG_DIR = '/var/log/contrail/'

class IndexAllocator(object):

    def __init__(self, zookeeper_client, path, size=0, start_idx=0,
                 reverse=False,alloc_list=None, max_alloc=0):
        self._size = size
        self._start_idx = start_idx
        if alloc_list is None:
            self._alloc_list = [{'start':start_idx, 'end':start_idx+size}]
        else:
            sorted_alloc_list = sorted(alloc_list, key=lambda k: k['start'])
            self._alloc_list = sorted_alloc_list

        alloc_count = len(self._alloc_list)
        total_size = 0
        size = 0

        #check for overlap in alloc_list --TODO
        for alloc_idx in range (0, alloc_count -1):
            idx_start_addr = self._alloc_list[alloc_idx]['start']
            idx_end_addr = self._alloc_list[alloc_idx]['end']
            next_start_addr = self._alloc_list[alloc_idx+1]['start']
            if next_start_addr <= idx_end_addr:
                raise Exception(
                    'Allocation Lists Overlapping: %s' %(alloc_list))
            size += idx_end_addr - idx_start_addr + 1
        size += self._alloc_list[alloc_count-1]['end'] - self._alloc_list[alloc_count-1]['start'] + 1

        if max_alloc == 0:
            self._max_alloc = size
        else:
            self._max_alloc = max_alloc

        self._zookeeper_client = zookeeper_client
        self._path = path
        self._in_use = bitarray('0')
        self._reverse = reverse
        for idx in self._zookeeper_client.get_children(path):
            idx_int = self._get_bit_from_zk_index(int(idx))
            if idx_int >= 0:
                self._set_in_use(idx_int)
        # end for idx
    # end __init__

    def _get_zk_index_from_bit(self, idx):
        size = idx
        if self._reverse:
            for alloc in reversed(self._alloc_list):
                size -= alloc['end'] - alloc['start'] + 1
                if size < 0:
                   return alloc['start']-size - 1
        else:
            for alloc in self._alloc_list:
                size -= alloc['end'] - alloc['start'] + 1
                if size < 0:
                   return alloc['end']+size + 1

        raise ResourceExhaustionError(
            'Cannot get zk index from bit %s' %(idx))
    # end _get_zk_index

    def _get_bit_from_zk_index(self, idx):
        size = 0
        if self._reverse:
            for alloc in reversed(self._alloc_list):
                if alloc['start'] <= idx <= alloc['end']:
                    return alloc['end'] - idx + size
                size += alloc['end'] - alloc['start'] + 1
            pass
        else:
            for alloc in self._alloc_list:
                if alloc['start'] <= idx <= alloc['end']:
                    return idx - alloc['start'] + size
                size += alloc['end'] - alloc['start'] + 1
        return -1
    # end _get_bit_from_zk_index

    def _set_in_use(self, bitnum):
        # if the index is higher than _max_alloc, do not use the bitarray, in
        # order to reduce the size of the bitarray. Otherwise, set the bit
        # corresponding to idx to 1 and extend the _in_use bitarray if needed
        if bitnum > self._max_alloc:
            return
        if bitnum >= self._in_use.length():
            temp = bitarray(bitnum - self._in_use.length())
            temp.setall(0)
            temp.append('1')
            self._in_use.extend(temp)
        else:
            self._in_use[bitnum] = 1
    # end _set_in_use

    def _reset_in_use(self, bitnum):
        # if the index is higher than _max_alloc, do not use the bitarray, in
        # order to reduce the size of the bitarray. Otherwise, set the bit
        # corresponding to idx to 1 and extend the _in_use bitarray if needed
        if bitnum > self._max_alloc:
            return
        if bitnum >= self._in_use.length():
            return
        else:
            self._in_use[bitnum] = 0
    # end _reset_in_use

    def set_in_use(self, idx):
        bit_idx = self._get_bit_from_zk_index(idx)
        if bit_idx < 0:
            return
        self._set_in_use(bit_idx)
    # end set_in_use

    def reset_in_use(self, idx):
        bit_idx = self._get_bit_from_zk_index(idx)
        if bit_idx < 0:
            return
        self._reset_in_use(bit_idx)
    # end reset_in_use

    def get_alloc_count(self):
        return self._in_use.count()
    # end get_alloc_count

    def alloc(self, value=None):
        # Allocates a index from the allocation list
        if self._in_use.all():
            idx = self._in_use.length()
            if idx > self._max_alloc:
                raise ResourceExhaustionError()
            self._in_use.append(1)
        else:
            idx = self._in_use.index(0)
            self._in_use[idx] = 1

        idx = self._get_zk_index_from_bit(idx)
        try:
            # Create a node at path and return its integer value
            id_str = "%(#)010d" % {'#': idx}
            self._zookeeper_client.create_node(self._path + id_str, value)
            return idx
        except ResourceExistsError:
            return self.alloc(value)
    # end alloc

    def reserve(self, idx, value=None):
        # Reserves the requested index if available
        if not self._start_idx <= idx < self._start_idx + self._size:
            return None

        try:
            # Create a node at path and return its integer value
            id_str = "%(#)010d" % {'#': idx}
            self._zookeeper_client.create_node(self._path + id_str, value)
            self.set_in_use(idx)
            return idx
        except ResourceExistsError:
            self.set_in_use(idx)
            existing_value = self.read(idx)
            if (value == existing_value):
                # idempotent reserve
                return idx
            msg = 'For index %s reserve conflicts with existing value %s.' \
                  %(idx, existing_value)
            self._zookeeper_client.syslog(msg, level='notice')
            raise
    # end reserve

    def delete(self, idx):
        id_str = "%(#)010d" % {'#': idx}
        self._zookeeper_client.delete_node(self._path + id_str)
        bit_idx = self._get_bit_from_zk_index(idx)
        if 0 <= bit_idx < self._in_use.length():
            self._in_use[bit_idx] = 0
    # end delete

    def read(self, idx):
        id_str = "%(#)010d" % {'#': idx}
        id_val = self._zookeeper_client.read_node(self._path+id_str)
        if id_val is not None:
            bit_idx = self._get_bit_from_zk_index(idx)
            if bit_idx >= 0:
                self._set_in_use(bit_idx)
        return id_val
    # end read

    def empty(self):
        return not self._in_use.any()
    # end empty

    @classmethod
    def delete_all(cls, zookeeper_client, path):
        try:
            zookeeper_client.delete_node(path, recursive=True)
        except kazoo.exceptions.NotEmptyError:
            #TODO: Add retries for NotEmptyError
            zookeeper_client.syslog("NotEmptyError while deleting %s" % path)
    # end delete_all

#end class IndexAllocator

class ZookeeperClient(object):

    def __init__(self, module, server_list, logging_fn=None):
        # logging
        logger = logging.getLogger(module)
        logger.setLevel(logging.DEBUG)
        try:
            handler = logging.handlers.RotatingFileHandler(
                LOG_DIR + module + '-zk.log', maxBytes=10*1024*1024, backupCount=5)
        except IOError:
            print "Cannot open log file in %s" %(LOG_DIR)
        else:
            log_format = logging.Formatter('%(asctime)s [%(name)s]: %(message)s',
                                           datefmt='%m/%d/%Y %I:%M:%S %p')
            handler.setFormatter(log_format)
            logger.addHandler(handler)

        if logging_fn:
            self.log = logging_fn
        else:
            self.log = self.syslog

        # KazooRetry to retry keeper CRUD operations
        self._retry = KazooRetry(max_tries=None, max_delay=300,
                                 sleep_func=gevent.sleep)
        self._zk_client = kazoo.client.KazooClient(
                server_list,
                timeout=400,
                handler=kazoo.handlers.gevent.SequentialGeventHandler(),
                logger=logger,
                connection_retry=self._retry,
                command_retry=self._retry)

        self._zk_client.add_listener(self._zk_listener)
        self._logger = logger
        self._election = None
        self._server_list = server_list

        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')
        self._lost_cb = None
        self._suspend_cb = None

        self.connect()
    # end __init__

    # start
    def connect(self):
        while True:
            try:
                self._zk_client.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')

    # end

    def is_connected(self):
        return self._zk_client.state == KazooState.CONNECTED
    # end is_connected

    def syslog(self, msg, *args, **kwargs):
        if not self._logger:
            return
        level = kwargs.get('level', 'info')
        if isinstance(level, int):
            from pysandesh.sandesh_logger import SandeshLogger
            level = SandeshLogger.get_py_logger_level(level)

        log_method = getattr(self._logger, level, self._logger.info)
        log_method(msg)
    # end syslog

    def set_lost_cb(self, lost_cb=None):
        # set a callback to be called when kazoo state is lost
        # set to None for default action
        self._lost_cb = lost_cb
    # end set_lost_cb

    def set_suspend_cb(self, suspend_cb=None):
        # set a callback to be called when kazoo state is suspend
        # set to None for default action
        self._suspend_cb = suspend_cb
    # end set_suspend_cb

    def _zk_listener(self, state):
        if state == KazooState.CONNECTED:
            if self._election:
                self._election.cancel()
            # Update connection info
            self._sandesh_connection_info_update(status='UP', message='')
        elif state == KazooState.LOST:
            # Lost the session with ZooKeeper Server
            # Best of option we have is to exit the process and restart all
            # over again
            self._sandesh_connection_info_update(status='DOWN',
                                      message='Connection to Zookeeper lost')
            if self._lost_cb:
                self._lost_cb()
            else:
                os._exit(2)
        elif state == KazooState.SUSPENDED:
            # Update connection info
            self._sandesh_connection_info_update(status='INIT',
                message = 'Connection to zookeeper lost. Retrying')
            if self._suspend_cb:
                self._suspend_cb()

    # end

    def master_election(self, path, identifier, func, *args, **kwargs):
        self._election = self._zk_client.Election(path, identifier)
        self._election.run(func, *args, **kwargs)
    # end master_election

    def create_node(self, path, value=None):
        try:
            if value is None:
                value = uuid.uuid4()
            retry = self._retry.copy()
            retry(self._zk_client.create, path, str(value), makepath=True)
        except kazoo.exceptions.NodeExistsError:
            current_value = self.read_node(path)
            if current_value == value:
                return True;
            raise ResourceExistsError(path, str(current_value), 'zookeeper')
    # end create_node

    def delete_node(self, path, recursive=False):
        try:
            retry = self._retry.copy()
            retry(self._zk_client.delete, path, recursive=recursive)
        except kazoo.exceptions.NoNodeError:
            pass
        except Exception as e:
            raise e
    # end delete_node

    def read_node(self, path, include_timestamp=False):
        try:
            retry = self._retry.copy()
            value = retry(self._zk_client.get, path)
            if include_timestamp:
                return value
            return value[0]
        except Exception:
            return None
    # end read_node

    def get_children(self, path):
        try:
            retry = self._retry.copy()
            return retry(self._zk_client.get_children, path)
        except Exception:
            return []
    # end read_node

    def exists(self, path):
        try:
            retry = self._retry.copy()
            return retry(self._zk_client.exists, path)
        except Exception:
            return []
    # end exists

    def _sandesh_connection_info_update(self, status, message):
        from pysandesh.connection_info import ConnectionState
        from pysandesh.gen_py.process_info.ttypes import ConnectionStatus
        from pysandesh.gen_py.process_info.ttypes import ConnectionType as ConnType
        from pysandesh.gen_py.sandesh.ttypes import SandeshLevel

        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type = ConnType.ZOOKEEPER,
                name = 'Zookeeper', status = new_conn_state,
                message = message,
                server_addrs = self._server_list.split(','))

        if (self._conn_state and self._conn_state != ConnectionStatus.DOWN and
            new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' %(message)
            self.log(msg, level=SandeshLevel.SYS_ERR)
        if (self._conn_state and self._conn_state != new_conn_state and
            new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self.log(msg, level=SandeshLevel.SYS_NOTICE)

        self._conn_state = new_conn_state
    # end _sandesh_connection_info_update

# end class ZookeeperClient
<commit_msg>If integer level is passed, use logger.log api<commit_after>#
# Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
#
import os
import gevent
import logging
import kazoo.client
import kazoo.exceptions
import kazoo.handlers.gevent
import kazoo.recipe.election
from kazoo.client import KazooState
from kazoo.retry import KazooRetry

from bitarray import bitarray
from cfgm_common.exceptions import ResourceExhaustionError, ResourceExistsError
from gevent.coros import BoundedSemaphore

import uuid

LOG_DIR = '/var/log/contrail/'

class IndexAllocator(object):

    def __init__(self, zookeeper_client, path, size=0, start_idx=0,
                 reverse=False,alloc_list=None, max_alloc=0):
        self._size = size
        self._start_idx = start_idx
        if alloc_list is None:
            self._alloc_list = [{'start':start_idx, 'end':start_idx+size}]
        else:
            sorted_alloc_list = sorted(alloc_list, key=lambda k: k['start'])
            self._alloc_list = sorted_alloc_list

        alloc_count = len(self._alloc_list)
        total_size = 0
        size = 0

        #check for overlap in alloc_list --TODO
        for alloc_idx in range (0, alloc_count -1):
            idx_start_addr = self._alloc_list[alloc_idx]['start']
            idx_end_addr = self._alloc_list[alloc_idx]['end']
            next_start_addr = self._alloc_list[alloc_idx+1]['start']
            if next_start_addr <= idx_end_addr:
                raise Exception(
                    'Allocation Lists Overlapping: %s' %(alloc_list))
            size += idx_end_addr - idx_start_addr + 1
        size += self._alloc_list[alloc_count-1]['end'] - self._alloc_list[alloc_count-1]['start'] + 1

        if max_alloc == 0:
            self._max_alloc = size
        else:
            self._max_alloc = max_alloc

        self._zookeeper_client = zookeeper_client
        self._path = path
        self._in_use = bitarray('0')
        self._reverse = reverse
        for idx in self._zookeeper_client.get_children(path):
            idx_int = self._get_bit_from_zk_index(int(idx))
            if idx_int >= 0:
                self._set_in_use(idx_int)
        # end for idx
    # end __init__

    def _get_zk_index_from_bit(self, idx):
        size = idx
        if self._reverse:
            for alloc in reversed(self._alloc_list):
                size -= alloc['end'] - alloc['start'] + 1
                if size < 0:
                   return alloc['start']-size - 1
        else:
            for alloc in self._alloc_list:
                size -= alloc['end'] - alloc['start'] + 1
                if size < 0:
                   return alloc['end']+size + 1

        raise ResourceExhaustionError(
            'Cannot get zk index from bit %s' %(idx))
    # end _get_zk_index

    def _get_bit_from_zk_index(self, idx):
        size = 0
        if self._reverse:
            for alloc in reversed(self._alloc_list):
                if alloc['start'] <= idx <= alloc['end']:
                    return alloc['end'] - idx + size
                size += alloc['end'] - alloc['start'] + 1
            pass
        else:
            for alloc in self._alloc_list:
                if alloc['start'] <= idx <= alloc['end']:
                    return idx - alloc['start'] + size
                size += alloc['end'] - alloc['start'] + 1
        return -1
    # end _get_bit_from_zk_index

    def _set_in_use(self, bitnum):
        # if the index is higher than _max_alloc, do not use the bitarray, in
        # order to reduce the size of the bitarray. Otherwise, set the bit
        # corresponding to idx to 1 and extend the _in_use bitarray if needed
        if bitnum > self._max_alloc:
            return
        if bitnum >= self._in_use.length():
            temp = bitarray(bitnum - self._in_use.length())
            temp.setall(0)
            temp.append('1')
            self._in_use.extend(temp)
        else:
            self._in_use[bitnum] = 1
    # end _set_in_use

    def _reset_in_use(self, bitnum):
        # if the index is higher than _max_alloc, do not use the bitarray, in
        # order to reduce the size of the bitarray. Otherwise, set the bit
        # corresponding to idx to 1 and extend the _in_use bitarray if needed
        if bitnum > self._max_alloc:
            return
        if bitnum >= self._in_use.length():
            return
        else:
            self._in_use[bitnum] = 0
    # end _reset_in_use

    def set_in_use(self, idx):
        bit_idx = self._get_bit_from_zk_index(idx)
        if bit_idx < 0:
            return
        self._set_in_use(bit_idx)
    # end set_in_use

    def reset_in_use(self, idx):
        bit_idx = self._get_bit_from_zk_index(idx)
        if bit_idx < 0:
            return
        self._reset_in_use(bit_idx)
    # end reset_in_use

    def get_alloc_count(self):
        return self._in_use.count()
    # end get_alloc_count

    def alloc(self, value=None):
        # Allocates a index from the allocation list
        if self._in_use.all():
            idx = self._in_use.length()
            if idx > self._max_alloc:
                raise ResourceExhaustionError()
            self._in_use.append(1)
        else:
            idx = self._in_use.index(0)
            self._in_use[idx] = 1

        idx = self._get_zk_index_from_bit(idx)
        try:
            # Create a node at path and return its integer value
            id_str = "%(#)010d" % {'#': idx}
            self._zookeeper_client.create_node(self._path + id_str, value)
            return idx
        except ResourceExistsError:
            return self.alloc(value)
    # end alloc

    def reserve(self, idx, value=None):
        # Reserves the requested index if available
        if not self._start_idx <= idx < self._start_idx + self._size:
            return None

        try:
            # Create a node at path and return its integer value
            id_str = "%(#)010d" % {'#': idx}
            self._zookeeper_client.create_node(self._path + id_str, value)
            self.set_in_use(idx)
            return idx
        except ResourceExistsError:
            self.set_in_use(idx)
            existing_value = self.read(idx)
            if (value == existing_value):
                # idempotent reserve
                return idx
            msg = 'For index %s reserve conflicts with existing value %s.' \
                  %(idx, existing_value)
            self._zookeeper_client.syslog(msg, level='notice')
            raise
    # end reserve

    def delete(self, idx):
        id_str = "%(#)010d" % {'#': idx}
        self._zookeeper_client.delete_node(self._path + id_str)
        bit_idx = self._get_bit_from_zk_index(idx)
        if 0 <= bit_idx < self._in_use.length():
            self._in_use[bit_idx] = 0
    # end delete

    def read(self, idx):
        id_str = "%(#)010d" % {'#': idx}
        id_val = self._zookeeper_client.read_node(self._path+id_str)
        if id_val is not None:
            bit_idx = self._get_bit_from_zk_index(idx)
            if bit_idx >= 0:
                self._set_in_use(bit_idx)
        return id_val
    # end read

    def empty(self):
        return not self._in_use.any()
    # end empty

    @classmethod
    def delete_all(cls, zookeeper_client, path):
        try:
            zookeeper_client.delete_node(path, recursive=True)
        except kazoo.exceptions.NotEmptyError:
            #TODO: Add retries for NotEmptyError
            zookeeper_client.syslog("NotEmptyError while deleting %s" % path)
    # end delete_all

#end class IndexAllocator

class ZookeeperClient(object):

    def __init__(self, module, server_list, logging_fn=None):
        # logging
        logger = logging.getLogger(module)
        logger.setLevel(logging.DEBUG)
        try:
            handler = logging.handlers.RotatingFileHandler(
                LOG_DIR + module + '-zk.log', maxBytes=10*1024*1024, backupCount=5)
        except IOError:
            print "Cannot open log file in %s" %(LOG_DIR)
        else:
            log_format = logging.Formatter('%(asctime)s [%(name)s]: %(message)s',
                                           datefmt='%m/%d/%Y %I:%M:%S %p')
            handler.setFormatter(log_format)
            logger.addHandler(handler)

        if logging_fn:
            self.log = logging_fn
        else:
            self.log = self.syslog

        # KazooRetry to retry keeper CRUD operations
        self._retry = KazooRetry(max_tries=None, max_delay=300,
                                 sleep_func=gevent.sleep)
        self._zk_client = kazoo.client.KazooClient(
                server_list,
                timeout=400,
                handler=kazoo.handlers.gevent.SequentialGeventHandler(),
                logger=logger,
                connection_retry=self._retry,
                command_retry=self._retry)

        self._zk_client.add_listener(self._zk_listener)
        self._logger = logger
        self._election = None
        self._server_list = server_list

        self._conn_state = None
        self._sandesh_connection_info_update(status='INIT', message='')
        self._lost_cb = None
        self._suspend_cb = None

        self.connect()
    # end __init__

    # start
    def connect(self):
        while True:
            try:
                self._zk_client.start()
                break
            except gevent.event.Timeout as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
            # Zookeeper is also throwing exception due to delay in master election
            except Exception as e:
                # Update connection info
                self._sandesh_connection_info_update(status='DOWN',
                                                     message=str(e))
                gevent.sleep(1)
        # Update connection info
        self._sandesh_connection_info_update(status='UP', message='')

    # end

    def is_connected(self):
        return self._zk_client.state == KazooState.CONNECTED
    # end is_connected

    def syslog(self, msg, *args, **kwargs):
        if not self._logger:
            return
        level = kwargs.get('level', 'info')
        if isinstance(level, int):
            from pysandesh.sandesh_logger import SandeshLogger
            level = SandeshLogger.get_py_logger_level(level)
            self._logger.log(level, msg)
            return

        log_method = getattr(self._logger, level, self._logger.info)
        log_method(msg)
    # end syslog

    def set_lost_cb(self, lost_cb=None):
        # set a callback to be called when kazoo state is lost
        # set to None for default action
        self._lost_cb = lost_cb
    # end set_lost_cb

    def set_suspend_cb(self, suspend_cb=None):
        # set a callback to be called when kazoo state is suspend
        # set to None for default action
        self._suspend_cb = suspend_cb
    # end set_suspend_cb

    def _zk_listener(self, state):
        if state == KazooState.CONNECTED:
            if self._election:
                self._election.cancel()
            # Update connection info
            self._sandesh_connection_info_update(status='UP', message='')
        elif state == KazooState.LOST:
            # Lost the session with ZooKeeper Server
            # Best of option we have is to exit the process and restart all
            # over again
            self._sandesh_connection_info_update(status='DOWN',
                                      message='Connection to Zookeeper lost')
            if self._lost_cb:
                self._lost_cb()
            else:
                os._exit(2)
        elif state == KazooState.SUSPENDED:
            # Update connection info
            self._sandesh_connection_info_update(status='INIT',
                message = 'Connection to zookeeper lost. Retrying')
            if self._suspend_cb:
                self._suspend_cb()

    # end

    def master_election(self, path, identifier, func, *args, **kwargs):
        self._election = self._zk_client.Election(path, identifier)
        self._election.run(func, *args, **kwargs)
    # end master_election

    def create_node(self, path, value=None):
        try:
            if value is None:
                value = uuid.uuid4()
            retry = self._retry.copy()
            retry(self._zk_client.create, path, str(value), makepath=True)
        except kazoo.exceptions.NodeExistsError:
            current_value = self.read_node(path)
            if current_value == value:
                return True;
            raise ResourceExistsError(path, str(current_value), 'zookeeper')
    # end create_node

    def delete_node(self, path, recursive=False):
        try:
            retry = self._retry.copy()
            retry(self._zk_client.delete, path, recursive=recursive)
        except kazoo.exceptions.NoNodeError:
            pass
        except Exception as e:
            raise e
    # end delete_node

    def read_node(self, path, include_timestamp=False):
        try:
            retry = self._retry.copy()
            value = retry(self._zk_client.get, path)
            if include_timestamp:
                return value
            return value[0]
        except Exception:
            return None
    # end read_node

    def get_children(self, path):
        try:
            retry = self._retry.copy()
            return retry(self._zk_client.get_children, path)
        except Exception:
            return []
    # end read_node

    def exists(self, path):
        try:
            retry = self._retry.copy()
            return retry(self._zk_client.exists, path)
        except Exception:
            return []
    # end exists

    def _sandesh_connection_info_update(self, status, message):
        from pysandesh.connection_info import ConnectionState
        from pysandesh.gen_py.process_info.ttypes import ConnectionStatus
        from pysandesh.gen_py.process_info.ttypes import ConnectionType as ConnType
        from pysandesh.gen_py.sandesh.ttypes import SandeshLevel

        new_conn_state = getattr(ConnectionStatus, status)
        ConnectionState.update(conn_type = ConnType.ZOOKEEPER,
                name = 'Zookeeper', status = new_conn_state,
                message = message,
                server_addrs = self._server_list.split(','))

        if (self._conn_state and self._conn_state != ConnectionStatus.DOWN and
            new_conn_state == ConnectionStatus.DOWN):
            msg = 'Connection to Zookeeper down: %s' %(message)
            self.log(msg, level=SandeshLevel.SYS_ERR)
        if (self._conn_state and self._conn_state != new_conn_state and
            new_conn_state == ConnectionStatus.UP):
            msg = 'Connection to Zookeeper ESTABLISHED'
            self.log(msg, level=SandeshLevel.SYS_NOTICE)

        self._conn_state = new_conn_state
    # end _sandesh_connection_info_update

# end class ZookeeperClient
<|endoftext|>


<commit_before>import os
import re

from gcloud import storage
from gcloud.exceptions import Forbidden

from pocs.utils import error
from pocs.utils.logger import get_root_logger
from pocs.utils.config import load_config

class PanStorage(object):
    """ Class for interacting with Google Cloud Platform """

    def __init__(self, bucket_name, auth_key=None, project_id='panoptes-survey'):
        """Create an object that can interact easily with storage buckets.

        Args:
            bucket_name (str): Name of bucket to use.
            auth_key (str, optional): Path to valid json authorization token.
            project_id (str, optional): Project id hosting the bucket. Default 'panoptes-survey'

        Raises:
            error.GoogleCloudError: Error raised if valid connection cannot be formed for
                given project, bucket, and authorization.
        """
        self.logger = get_root_logger()
        assert auth_key is not None and os.path.exists(auth_key), self.logger.error(
            "Cannot use google storage without PANOPTES_CLOUD_KEY variable set.")

        super(PanStorage, self).__init__()

        self.unit_id = load_config()['pan_id']
        assert re.match('PAN\d\d\d', self.unit_id) is not None

        self.project_id = project_id
        self.bucket_name = bucket_name

        self.client = storage.Client.from_service_account_json(
            auth_key,
            project=self.project_id
        )

        try:
            self.bucket = self.client.get_bucket(bucket_name)
        except Forbidden as e:
            raise error.GoogleCloudError(
                "Storage bucket does not exist or no permissions. " +
                "Ensure that the PANOPTES_CLOUD_KEY variable is properly set"
            )

        self.logger.info("Connected to storage bucket {}", self.bucket_name)

    def upload(self, local_path, remote_path=None):
        """Upload the given file to the Google Cloud Storage bucket.

        Note:
            The name of the current unit will be prepended to the path
            so that all files will be placed in a "subdirectory" according
            to unit.

        Args:
            local_path (str): Path to local file to be uploaded
            remote_path (str, optional): Destination path in bucket.

        Returns:
            str: Remote path of uploaded object
        """
        assert os.path.exists(local_path), self.logger.warning(
            "Local path does not exist, can't upload: {}", local_path)

        if remote_path is None:
            remote_path = local_path

        if not remote_path.startswith(self.unit_id):
            remote_path = '{}/{}'.format(self.unit_id, remote_path)

        self.logger.debug('Uploading file: {} to bucket: {} object: {} ',
                          local_path, self.bucket.name, remote_path)

        try:
            self.bucket.blob(remote_path).upload_from_filename(
                filename=local_path)
            self.logger.debug('Upload complete')

        except Exception as err:
            self.logger.warning(
                'Problem uploading file {}: {}'.format(local_path, err))

        return remote_path
<commit_msg>Support both old and new Google Cloud libraries (#582)<commit_after>import os
import re

# Supporting a change in Google Cloud libraries by trying imports both ways:
try:  # pragma: no cover
    # New way:
    from google.cloud import storage
    from google.cloud.exceptions import Forbidden
except ImportError:  # pragma: no cover
    # Old way:
    from gcloud import storage
    from gcloud.exceptions import Forbidden

from pocs.utils import error
from pocs.utils.logger import get_root_logger
from pocs.utils.config import load_config

class PanStorage(object):
    """ Class for interacting with Google Cloud Platform """

    def __init__(self, bucket_name, auth_key=None, project_id='panoptes-survey'):
        """Create an object that can interact easily with storage buckets.

        Args:
            bucket_name (str): Name of bucket to use.
            auth_key (str, optional): Path to valid json authorization token.
            project_id (str, optional): Project id hosting the bucket. Default 'panoptes-survey'

        Raises:
            error.GoogleCloudError: Error raised if valid connection cannot be formed for
                given project, bucket, and authorization.
        """
        self.logger = get_root_logger()
        assert auth_key is not None and os.path.exists(auth_key), self.logger.error(
            "Cannot use google storage without PANOPTES_CLOUD_KEY variable set.")

        super(PanStorage, self).__init__()

        self.unit_id = load_config()['pan_id']
        assert re.match('PAN\d\d\d', self.unit_id) is not None

        self.project_id = project_id
        self.bucket_name = bucket_name

        self.client = storage.Client.from_service_account_json(
            auth_key,
            project=self.project_id
        )

        try:
            self.bucket = self.client.get_bucket(bucket_name)
        except Forbidden as e:
            raise error.GoogleCloudError(
                "Storage bucket does not exist or no permissions. " +
                "Ensure that the PANOPTES_CLOUD_KEY variable is properly set"
            )

        self.logger.info("Connected to storage bucket {}", self.bucket_name)

    def upload(self, local_path, remote_path=None):
        """Upload the given file to the Google Cloud Storage bucket.

        Note:
            The name of the current unit will be prepended to the path
            so that all files will be placed in a "subdirectory" according
            to unit.

        Args:
            local_path (str): Path to local file to be uploaded
            remote_path (str, optional): Destination path in bucket.

        Returns:
            str: Remote path of uploaded object
        """
        assert os.path.exists(local_path), self.logger.warning(
            "Local path does not exist, can't upload: {}", local_path)

        if remote_path is None:
            remote_path = local_path

        if not remote_path.startswith(self.unit_id):
            remote_path = '{}/{}'.format(self.unit_id, remote_path)

        self.logger.debug('Uploading file: {} to bucket: {} object: {} ',
                          local_path, self.bucket.name, remote_path)

        try:
            self.bucket.blob(remote_path).upload_from_filename(
                filename=local_path)
            self.logger.debug('Upload complete')

        except Exception as err:
            self.logger.warning(
                'Problem uploading file {}: {}'.format(local_path, err))

        return remote_path
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Simple interface between Slack Webhooks and Zotero API

   Intended to be used in a cron-like system that will periodically
   run this script with since=`last run's version` which can be
   obtained using the artifact parameter's functionality

   Example: slack-zotero.py --group 12345 \
                            --api "kjas734890fnlkjafFJnadsf" \
                            --webhook "https://hooks.slack.com/services/OUHAEFNF/OUIHAQEUN/uihsdf786SHBF6ebSF" \
                            --since 9043 \
                            --channel "#test" \
                            --username "Zotero Bot" \
                            --icon ":cow:" \
                            --artifact "slack-zotero-bot-previous.json" \
                            -v
"""

import urllib.request
import json
import requests
import re
import html
import time
import datetime

def retrieve_articles(group_id, api_key, limit=1, include='data,bib', since=0):
    """Retrieves articles from a Zotero group API feed"""
    zotero_template = "https://api.zotero.org/groups/{group_id}/items/top?start=0&limit={limit}&format=json&v=3&key={api_key}"
    zotero_url = zotero_template.format(group_id=group_id, api_key=api_key, limit=limit)

    zotero_url += "&include={0}".format(include) if include else ""
    zotero_url += "&since={0}".format(since) if since else ""

    print("Retrieving most recent {limit} articles since version: {version}".format(limit=limit, version=since))

    response = urllib.request.urlopen(zotero_url)
    articles = json.loads(response.readall().decode('utf-8'))

    print("Retrieved {0} articles".format(len(articles)))

    return articles

def send_article_to_slack(webhook_url, article, channel=None, username=None, icon_emoji=None, verbose=True, mock=False):
    """Sends a JSON article to the given Slack Webhooks URL"""
    payload = {"text": format_article(article)}
    if channel:
        payload['channel'] = channel

    if username:
        payload['username'] = username

    if icon_emoji:
        payload['icon_emoji'] = icon_emoji

    response = None
    if not mock:
        response = requests.post(webhook_url, json=payload)
        if response.status_code != 200:
            print(
                'Request to slack returned an error %s, the response is:\n%s'
                % (response.status_code, response.text)
            )

    if verbose:
        version = int(article['version'])
        print("{0} - {1}".format(version, article['data']['title']))

    return response

def format_article(article):
    """Feel free to overwrite me with your preferred format"""
    submitter = article['meta']['createdByUser']['username']

    data = article['data']
    title = data['title']
    bib = html.unescape(re.sub("</?(div|i).*?>|\n", " ", article['bib']).strip())
    abstract = data['abstractNote']
    url = data['url']

    return "<{url}|*{title}*>\n{bib}\nUploaded By: *{submitter}*\n\n>>>{abstract}".format(title=title, abstract=abstract,
                                                                                url=url, submitter=submitter,
                                                                                bib=bib)

def main(zotero_group, zotero_api_key, slack_webhook_url, since_version=0, channel=None, username=None, icon_emoji=None,
         mock=False, verbose=True):
    timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

    articles = retrieve_articles(zotero_group, zotero_api_key, limit=25 if since_version else 1, since=since_version)

    max_version = max([since_version] + [article['version'] for article in articles])

    for article in articles:
        send_article_to_slack(slack_webhook_url, article, channel=channel, username=username, icon_emoji=icon_emoji,
                              verbose=verbose, mock=mock)

    print("Current Version: {0}".format(max_version))

    run_info = {"time": timestamp, "version": max_version, "articles_cnt": len(articles)}

    return run_info

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description="Retrieve most recent articles from a Zotero group using Zotero's "
                                                 "API v3 and send them to a Slack channel using Webhooks")

    parser.add_argument('--group', dest='group', type=int, required=True,
                        help='Zotero group ID of the library to monitor')

    parser.add_argument('--api', dest='api_key', type=str, required=True,
                        help='Zotero API key with access to the library')

    parser.add_argument('--webhook', dest='webhook', type=str, required=True,
                        help='Slack webhook URL to send articles')

    parser.add_argument('--since', dest='version', type=int, required=False, default=0,
                        help='Retrieve only articles created after this version')
    parser.add_argument('--channel', dest='channel', type=str, required=False, default=None,
                        help='Override default Slack webhooks channel')
    parser.add_argument('--username', dest='username', type=str, required=False, default=None,
                        help='Override default Slack webhooks username')
    parser.add_argument('--icon', dest='icon_emoji', type=str, required=False, default=None,
                        help='Override default Slack webhooks icon')

    parser.add_argument('--artifact', dest='artifact', type=str, required=False, default=None,
                        help='File to write the most recent article version')

    parser.add_argument('--mock', dest='mock', action='store_true',
                        help='Mock run; will not write to Slack or filesystem')

    parser.add_argument('-v', dest='verbose', action='store_true',
                        help='More verbose logging')

    args = parser.parse_args()

    run_info = main(args.group, args.api_key, args.webhook, args.version, args.channel, args.username, args.icon_emoji,
                    args.mock, args.verbose)

    if not args.mock and args.artifact:
        print("Writing run version to {artifact}".format(artifact=args.artifact))
        with open(args.artifact, 'w') as outfile:
            json.dump(run_info, outfile)
<commit_msg>--artifact now reads in and overrides --since<commit_after>#!/usr/bin/env python

"""Simple interface between Slack Webhooks and Zotero API

   Intended to be used in a cron-like system that will periodically
   run this script with since=`last run's version` which can be
   obtained using the artifact parameter's functionality

   Example: slack-zotero.py --group 12345 \
                            --api "kjas734890fnlkjafFJnadsf" \
                            --webhook "https://hooks.slack.com/services/OUHAEFNF/OUIHAQEUN/uihsdf786SHBF6ebSF" \
                            --since 9043 \
                            --channel "#test" \
                            --username "Zotero Bot" \
                            --icon ":cow:" \
                            --artifact "slack-zotero-bot-previous.json" \
                            -v
"""

import urllib.request
import json
import requests
import re
import html
import time
import datetime

def retrieve_articles(group_id, api_key, limit=1, include='data,bib', since=0):
    """Retrieves articles from a Zotero group API feed"""
    zotero_template = "https://api.zotero.org/groups/{group_id}/items/top?start=0&limit={limit}&format=json&v=3&key={api_key}"
    zotero_url = zotero_template.format(group_id=group_id, api_key=api_key, limit=limit)

    zotero_url += "&include={0}".format(include) if include else ""
    zotero_url += "&since={0}".format(since) if since else ""

    print("Retrieving most recent {limit} articles since version: {version}".format(limit=limit, version=since))

    response = urllib.request.urlopen(zotero_url)
    articles = json.loads(response.readall().decode('utf-8'))

    print("Retrieved {0} articles".format(len(articles)))

    return articles

def send_article_to_slack(webhook_url, article, channel=None, username=None, icon_emoji=None, verbose=True, mock=False):
    """Sends a JSON article to the given Slack Webhooks URL"""
    payload = {"text": format_article(article)}
    if channel:
        payload['channel'] = channel

    if username:
        payload['username'] = username

    if icon_emoji:
        payload['icon_emoji'] = icon_emoji

    response = None
    if not mock:
        response = requests.post(webhook_url, json=payload)
        if response.status_code != 200:
            print(
                'Request to slack returned an error %s, the response is:\n%s'
                % (response.status_code, response.text)
            )

    if verbose:
        version = int(article['version'])
        print("{0} - {1}".format(version, article['data']['title']))

    return response

def format_article(article):
    """Feel free to overwrite me with your preferred format"""
    submitter = article['meta']['createdByUser']['username']

    data = article['data']
    title = data['title']
    bib = html.unescape(re.sub("</?(div|i).*?>|\n", " ", article['bib']).strip())
    abstract = data['abstractNote']
    url = data['url']

    return "<{url}|*{title}*>\n{bib}\nUploaded By: *{submitter}*\n\n>>>{abstract}".format(title=title, abstract=abstract,
                                                                                url=url, submitter=submitter,
                                                                                bib=bib)

def main(zotero_group, zotero_api_key, slack_webhook_url, since_version=0, channel=None, username=None, icon_emoji=None,
         mock=False, verbose=True):
    timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')

    articles = retrieve_articles(zotero_group, zotero_api_key, limit=25 if since_version else 1, since=since_version)

    max_version = max([since_version] + [article['version'] for article in articles])

    for article in articles:
        send_article_to_slack(slack_webhook_url, article, channel=channel, username=username, icon_emoji=icon_emoji,
                              verbose=verbose, mock=mock)

    print("Current Version: {0}".format(max_version))

    run_info = {"time": timestamp, "version": max_version, "articles_cnt": len(articles)}

    return run_info

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description="Retrieve most recent articles from a Zotero group using Zotero's "
                                                 "API v3 and send them to a Slack channel using Webhooks")

    parser.add_argument('--group', dest='group', type=int, required=True,
                        help='Zotero group ID of the library to monitor')

    parser.add_argument('--api', dest='api_key', type=str, required=True,
                        help='Zotero API key with access to the library')

    parser.add_argument('--webhook', dest='webhook', type=str, required=True,
                        help='Slack webhook URL to send articles')

    parser.add_argument('--since', dest='version', type=int, required=False, default=0,
                        help='Retrieve only articles created after this version')
    parser.add_argument('--channel', dest='channel', type=str, required=False, default=None,
                        help='Override default Slack webhooks channel')
    parser.add_argument('--username', dest='username', type=str, required=False, default=None,
                        help='Override default Slack webhooks username')
    parser.add_argument('--icon', dest='icon_emoji', type=str, required=False, default=None,
                        help='Override default Slack webhooks icon')

    parser.add_argument('--artifact', dest='artifact', type=str, required=False, default=None,
                        help='Retrieve --since from & write run info to this file. OVERRIDES --since')

    parser.add_argument('--mock', dest='mock', action='store_true',
                        help='Mock run; will not write to Slack or filesystem')

    parser.add_argument('-v', dest='verbose', action='store_true',
                        help='More verbose logging')

    args = parser.parse_args()

    since = args.version
    if args.artifact:
        try:
            with open(args.artifact) as data_file:
                data = json.load(data_file)
                since = data['version']
        except (FileNotFoundError, KeyError) as e:
            print("Error reading version info from artifact file.")
            raise

    run_info = main(args.group, args.api_key, args.webhook, since, args.channel, args.username, args.icon_emoji,
                    args.mock, args.verbose)

    if not args.mock and args.artifact:
        print("Writing run version to {artifact}".format(artifact=args.artifact))
        with open(args.artifact, 'w') as outfile:
            json.dump(run_info, outfile)
<|endoftext|>


<commit_before>import os
import logging
import json
import requests
import urllib

from googleapiclient.discovery import build

logger = logging.getLogger('translation')

MSTRANSLATE_LANGS = ['hi']
GOOGLE_LANGS = ['hi']

def translateGoogle(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started google translation. %s", jobInfo);

    service = build('translate', 'v2',
            developerKey = os.environ['GOOGLE_DEV_KEY'])
    result = service.translations().list(
      source=fromLang,
      target=toLang,
      q=text).execute()['translations'][0]['translatedText']

    logger.info("Completed google translation. %s", jobInfo)
    return result
  except:
    logger.info("Google translation failed. %s", jobInfo)
    return ""

def translateMicrosoft(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started microsoft translation. %s", jobInfo);

    # get the access token
    args = {
      'client_id': os.environ['MSTRANSLATE_CLIENT_ID'],
      'client_secret': os.environ['MSTRANSLATE_CLIENT_SECRET'],
      'scope': 'http://api.microsofttranslator.com',
      'grant_type': 'client_credentials'
    }
    oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13'
    oauth_response = json.loads(
      requests.post(oauth_url, data=urllib.urlencode(args)).content)

    # make the translate api call
    strText = text
    if isinstance(text, unicode):
      strText = text.encode('utf-8')

    translation_args = {
      'text': strText,
      'to': toLang,
      'from': fromLang
    }

    headers={'Authorization': 'Bearer '+ oauth_response['access_token']}
    translate_url = 'https://api.microsofttranslator.com/V2/Ajax.svc/Translate?'
    translation_result = requests.get(
      translate_url + urllib.urlencode(translation_args),
      headers=headers)

    if translation_result.status_code == 200:
      logger.info("Completed microsoft translation. %s", jobInfo)
      return translation_result.content
    else:
      logger.info(
        "Microsoft translation call returned with status code %i",
        translation_result.status_code)
      return ""
  except Exception as e:
    logging.exception("Microsoft translation failed. %s", jobInfo)
    return ""

def translate(jobId, text, fromLang, toLang = 'en'):
  jobInfo = "fromLang: " + fromLang + " toLang: " + toLang \
     + " Job id: " + jobId

  # clip text if too long to save costs
  if len(text) > 1200:
    text = text[:1200]

  if fromLang in MSTRANSLATE_LANGS and fromLang in GOOGLE_LANGS:
    msResult = translateMicrosoft(jobInfo, text, fromLang, toLang)
    if len(msResult) > 0:
      return msResult
    else:
      return translateGoogle(jobInfo, text, fromLang, toLang)
<commit_msg>Microsoft translstion error sometimes returned with status code 200<commit_after>import os
import logging
import json
import requests
import urllib

from googleapiclient.discovery import build

logger = logging.getLogger('translation')

MSTRANSLATE_LANGS = ['hi']
GOOGLE_LANGS = ['hi']

def translateGoogle(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started google translation. %s", jobInfo);

    service = build('translate', 'v2',
            developerKey = os.environ['GOOGLE_DEV_KEY'])
    result = service.translations().list(
      source=fromLang,
      target=toLang,
      q=text).execute()['translations'][0]['translatedText']

    logger.info("Completed google translation. %s", jobInfo)
    return result
  except:
    logger.info("Google translation failed. %s", jobInfo)
    return ""

def translateMicrosoft(jobInfo, text, fromLang, toLang = 'en'):
  try:
    logger.info("Started microsoft translation. %s", jobInfo);

    # get the access token
    args = {
      'client_id': os.environ['MSTRANSLATE_CLIENT_ID'],
      'client_secret': os.environ['MSTRANSLATE_CLIENT_SECRET'],
      'scope': 'http://api.microsofttranslator.com',
      'grant_type': 'client_credentials'
    }
    oauth_url = 'https://datamarket.accesscontrol.windows.net/v2/OAuth2-13'
    oauth_response = json.loads(
      requests.post(oauth_url, data=urllib.urlencode(args)).content)

    # make the translate api call
    strText = text
    if isinstance(text, unicode):
      strText = text.encode('utf-8')

    translation_args = {
      'text': strText,
      'to': toLang,
      'from': fromLang
    }

    headers={'Authorization': 'Bearer '+ oauth_response['access_token']}
    translate_url = 'https://api.microsofttranslator.com/V2/Ajax.svc/Translate?'
    translation_result = requests.get(
      translate_url + urllib.urlencode(translation_args),
      headers=headers)

    if translation_result.status_code == 200 and \
      'TranslateApiException' not in translation_result.content:
      logger.info("Completed microsoft translation. %s", jobInfo)
      return translation_result.content
    else:
      logger.info(
        "Microsoft translation call returned with status code %i",
        translation_result.status_code)
      return ""
  except Exception as e:
    logging.exception("Microsoft translation failed. %s", jobInfo)
    return ""

def translate(jobId, text, fromLang, toLang = 'en'):
  jobInfo = "fromLang: " + fromLang + " toLang: " + toLang \
     + " Job id: " + jobId

  # clip text if too long to save costs
  if len(text) > 1200:
    text = text[:1200]

  if fromLang in MSTRANSLATE_LANGS and fromLang in GOOGLE_LANGS:
    msResult = translateMicrosoft(jobInfo, text, fromLang, toLang)
    if len(msResult) > 0:
      return msResult
    else:
      return translateGoogle(jobInfo, text, fromLang, toLang)
<|endoftext|>