<commit_before># -*- coding: utf-8 -*-
from decimal import Decimal
import logging
import datetime
from django.core.urlresolvers import reverse
from django.db.models import get_model
from django.utils.timezone import utc
import requests
import time
from django.utils.translation import ugettext_lazy as _
from getpaid.signals import user_data_query
from getpaid.backends import PaymentProcessorBase
from pagseguro import validar_dados
import urllib2
import urllib
from xml.dom.minidom import parseString

logger = logging.getLogger('getpaid.backends.pagseguro')

# see: https://pagseguro.uol.com.br/v2/guia-de-integracao/api-de-notificacoes.html#v2-item-api-de-notificacoes-status-da-transacao
class PagseguroTransactionStatus:
    IN_PROGRESS = 1     # payment transaction started and waiting be paid
    IN_ANALISE = 2
    PAID    = 3
    AVAILABLE = 4
    IN_DISPUTE = 5
    REFUNDED = 6
    CANCELED = 7

class PaymentProcessor(PaymentProcessorBase):
    BACKEND = 'getpaid.backends.pagseguro'
    BACKEND_NAME = _('PagSeguro')
    BACKEND_ACCEPTED_CURRENCY = ('BRL', )
    BACKEND_LOGO_URL = 'getpaid/backends/pagseguro/pagseguro_logo.png'

    _SEND_INSTRUCTION_PAGE = '/checkout.php?'
    _CHECKOUT_INSTRUCTION_PAGE = "checkout/?"
    _TRANSACTION_INSTRUCTION_PAGE = "checkout/payment.html?code="
    _ITEM_DATA_REQUIRED_FIELDS = ('id', 'description', 'quantity', 'value')

    _USER_DATA_TO_PAGSEGURO = {
        'name': 'Nome',  # Nome completo do cliente.
        'address': 'Logradouro',  # Logradouro do cliente (ex: Rua, Av, etc.)
        'address_number': 'Numero',  # Nº do imóvel do cliente (ex: 12)
        'address_complement': 'Complemento',  # Complemento (ex: Sala 109 ou Casa 1)
        'address_zip_code': 'CEP',  # O CEP de 8 dígitos do cliente. Somente números (ex: 22345678)
        'address_quarter': 'Bairro',  # Bairro do cliente
        'address_city': 'Cidade',  # Cidade do cliente (ex: São Paulo)
        'address_state': 'Estado',  # Estado do cliente (SP) no formato duas letras
        'phone': 'TelefoneFixo',  # Telefone fixo do cliente
        'cliente_email': 'Email',  # E-mail do cliente
    }

    _ACOUNT_DATA = {
        "email": "",
        "token":""
    }

    _PRODUCT_DATA = {
        "currency": "BRL",
        "itemId1":"",
        "itemDescription1":"",
        "itemQuantity1":"1",
        "itemAmount1":"",
        "reference":"",
    }

    def get_gateway_url(self, request):
        """  """
        # To active the pagseguro sandbox, will be need run the pagseguro server
        import uuid
        is_dev = PaymentProcessor.get_backend_setting('testing', False)
        if is_dev:
            gateway_url = "https://ws.sandbox.pagseguro.uol.com.br/v2/"
            gateway_url2 = "https://sandbox.pagseguro.uol.com.br/v2/"
        else:
            gateway_url = "https://ws.pagseguro.uol.com.br/v2/"
            gateway_url2 = "https://pagseguro.uol.com.br/v2/"

        order = self.payment.order

        token = PaymentProcessor.get_backend_setting('token')
        email = PaymentProcessor.get_backend_setting('email')
        self._ACOUNT_DATA["token"] = token
        self._ACOUNT_DATA["email"] = email

        reference = uuid.uuid4().hex

        product_description = self.get_order_description(self.payment, self.payment.order)
        redirectURL = PaymentProcessor._get_view_full_url(request, 'getpaid-pagseguro-success', args=(self.payment.id,))

        self._PRODUCT_DATA.update(itemId1=self.payment.id,
                                  itemDescription1= product_description,
                                  currency=self.payment.currency,
                                  itemAmount1=self.payment.amount,
                                  reference=reference,
                                  redirectURL=redirectURL,
                                  )

        full_data = dict(self._ACOUNT_DATA, **self._PRODUCT_DATA)

        params = urllib.urlencode(full_data)

        payment_full_url = "%s%s%s" % (gateway_url, self._CHECKOUT_INSTRUCTION_PAGE , params)
        request.encoding = 'ISO-8859-1'

        dados = {}

        logger.info(payment_full_url)

        headers = {"Content-type": "application/x-www-form-urlencoded"}
        response = requests.post(payment_full_url, headers=headers).text

        code = ""
        logger.warning("xml response from pagseguro: "+str(response))
        dom = parseString(response)

        checkout = dom.getElementsByTagName("checkout")
        if checkout:
            childNodes = checkout[0].childNodes
            code = childNodes[0].firstChild.nodeValue
            date = childNodes[1].firstChild.nodeValue

        self.payment.external_id = reference
        self.payment.description = code
        self.payment.save();

        return "%s%s" % (gateway_url2, self._TRANSACTION_INSTRUCTION_PAGE+code), 'GET', {}

    @staticmethod
    def process_notification(params):
        _TRANSACTION_CONSULT_URL = "https://ws.pagseguro.uol.com.br/v2/transactions/notifications/%s?email=%s&token=%s"

        is_dev = PaymentProcessor.get_backend_setting('testing', False)
        if is_dev:
            _TRANSACTION_CONSULT_URL = "https://ws.sandbox.pagseguro.uol.com.br/v2/transactions/"+\
             "notifications.php?notificationCode=%s&email=%s&token=%s"

        Payment = get_model('getpaid', 'Payment')

        token = PaymentProcessor.get_backend_setting('token')
        email = PaymentProcessor.get_backend_setting('email')

        url_consult = _TRANSACTION_CONSULT_URL % (params["notificationCode"], email, token)
        resp = requests.get(url_consult)

        logger.info("pagseguro notification: " + resp.text)

        dom = parseString(resp.text.encode('utf-8'))
        transactionNode = dom.getElementsByTagName("transaction")
        code = dom.getElementsByTagName("code")[0].firstChild.nodeValue
        status_code = int(dom.getElementsByTagName("status")[0].firstChild.nodeValue)

        reference = ""
        reference_node = dom.getElementsByTagName("reference")
        if reference_node:
            reference = reference_node[0].firstChild.nodeValue

        try:
            payment = Payment.objects.get(external_id__exact=reference)
        except Payment.DoesNotExist:
            logger.error('Payment does not exist with external_id=%s' % reference)
            return

        if status_code in (PagseguroTransactionStatus.AVAILABLE, PagseguroTransactionStatus.PAID):
            logger.info("Updating payment" + str(payment.id))
            amount = dom.getElementsByTagName("grossAmount")[0].firstChild.nodeValue
            payment.amount_paid = Decimal(amount)
            payment.paid_on = datetime.datetime.utcnow().replace(tzinfo=utc)
            payment.change_status('paid')

        elif status_code in (PagseguroTransactionStatus.CANCELED,
                             PagseguroTransactionStatus.REFUNDED,
                             PagseguroTransactionStatus.IN_DISPUTE):
            payment.change_status('failed')

    @staticmethod
    def _get_view_full_url(request, view_name, args=None):
        url = reverse(view_name, args=args)
        return 'http://%s%s' % (request.get_host(), url)
<commit_msg>debuging process_notification<commit_after># -*- coding: utf-8 -*-
from decimal import Decimal
import logging
import datetime
from django.core.urlresolvers import reverse
from django.db.models import get_model
from django.utils.timezone import utc
import requests
import time
from django.utils.translation import ugettext_lazy as _
from getpaid.signals import user_data_query
from getpaid.backends import PaymentProcessorBase
from pagseguro import validar_dados
import urllib2
import urllib
from xml.dom.minidom import parseString

logger = logging.getLogger('getpaid.backends.pagseguro')

# see: https://pagseguro.uol.com.br/v2/guia-de-integracao/api-de-notificacoes.html#v2-item-api-de-notificacoes-status-da-transacao
class PagseguroTransactionStatus:
    IN_PROGRESS = 1     # payment transaction started and waiting be paid
    IN_ANALISE = 2
    PAID    = 3
    AVAILABLE = 4
    IN_DISPUTE = 5
    REFUNDED = 6
    CANCELED = 7

class PaymentProcessor(PaymentProcessorBase):
    BACKEND = 'getpaid.backends.pagseguro'
    BACKEND_NAME = _('PagSeguro')
    BACKEND_ACCEPTED_CURRENCY = ('BRL', )
    BACKEND_LOGO_URL = 'getpaid/backends/pagseguro/pagseguro_logo.png'

    _SEND_INSTRUCTION_PAGE = '/checkout.php?'
    _CHECKOUT_INSTRUCTION_PAGE = "checkout/?"
    _TRANSACTION_INSTRUCTION_PAGE = "checkout/payment.html?code="
    _ITEM_DATA_REQUIRED_FIELDS = ('id', 'description', 'quantity', 'value')

    _USER_DATA_TO_PAGSEGURO = {
        'name': 'Nome',  # Nome completo do cliente.
        'address': 'Logradouro',  # Logradouro do cliente (ex: Rua, Av, etc.)
        'address_number': 'Numero',  # Nº do imóvel do cliente (ex: 12)
        'address_complement': 'Complemento',  # Complemento (ex: Sala 109 ou Casa 1)
        'address_zip_code': 'CEP',  # O CEP de 8 dígitos do cliente. Somente números (ex: 22345678)
        'address_quarter': 'Bairro',  # Bairro do cliente
        'address_city': 'Cidade',  # Cidade do cliente (ex: São Paulo)
        'address_state': 'Estado',  # Estado do cliente (SP) no formato duas letras
        'phone': 'TelefoneFixo',  # Telefone fixo do cliente
        'cliente_email': 'Email',  # E-mail do cliente
    }

    _ACOUNT_DATA = {
        "email": "",
        "token":""
    }

    _PRODUCT_DATA = {
        "currency": "BRL",
        "itemId1":"",
        "itemDescription1":"",
        "itemQuantity1":"1",
        "itemAmount1":"",
        "reference":"",
    }

    def get_gateway_url(self, request):
        """  """
        # To active the pagseguro sandbox, will be need run the pagseguro server
        import uuid
        is_dev = PaymentProcessor.get_backend_setting('testing', False)
        if is_dev:
            gateway_url = "https://ws.sandbox.pagseguro.uol.com.br/v2/"
            gateway_url2 = "https://sandbox.pagseguro.uol.com.br/v2/"
        else:
            gateway_url = "https://ws.pagseguro.uol.com.br/v2/"
            gateway_url2 = "https://pagseguro.uol.com.br/v2/"

        order = self.payment.order

        token = PaymentProcessor.get_backend_setting('token')
        email = PaymentProcessor.get_backend_setting('email')
        self._ACOUNT_DATA["token"] = token
        self._ACOUNT_DATA["email"] = email

        reference = uuid.uuid4().hex

        product_description = self.get_order_description(self.payment, self.payment.order)
        redirectURL = PaymentProcessor._get_view_full_url(request, 'getpaid-pagseguro-success', args=(self.payment.id,))

        self._PRODUCT_DATA.update(itemId1=self.payment.id,
                                  itemDescription1= product_description,
                                  currency=self.payment.currency,
                                  itemAmount1=self.payment.amount,
                                  reference=reference,
                                  redirectURL=redirectURL,
                                  )

        full_data = dict(self._ACOUNT_DATA, **self._PRODUCT_DATA)

        params = urllib.urlencode(full_data)

        payment_full_url = "%s%s%s" % (gateway_url, self._CHECKOUT_INSTRUCTION_PAGE , params)
        request.encoding = 'ISO-8859-1'

        dados = {}

        logger.info(payment_full_url)

        headers = {"Content-type": "application/x-www-form-urlencoded"}
        response = requests.post(payment_full_url, headers=headers).text

        code = ""
        logger.warning("xml response from pagseguro: "+str(response))
        dom = parseString(response)

        checkout = dom.getElementsByTagName("checkout")
        if checkout:
            childNodes = checkout[0].childNodes
            code = childNodes[0].firstChild.nodeValue
            date = childNodes[1].firstChild.nodeValue

        self.payment.external_id = reference
        self.payment.description = code
        self.payment.save();

        return "%s%s" % (gateway_url2, self._TRANSACTION_INSTRUCTION_PAGE+code), 'GET', {}

    @staticmethod
    def process_notification(params):
        _TRANSACTION_CONSULT_URL = "https://ws.pagseguro.uol.com.br/v2/transactions/notifications/%s?email=%s&token=%s"

        is_dev = PaymentProcessor.get_backend_setting('testing', False)
        if is_dev:
            _TRANSACTION_CONSULT_URL = "https://ws.sandbox.pagseguro.uol.com.br/v2/transactions/"+\
             "notifications.php?notificationCode=%s&email=%s&token=%s"

        Payment = get_model('getpaid', 'Payment')

        token = PaymentProcessor.get_backend_setting('token')
        email = PaymentProcessor.get_backend_setting('email')

        url_consult = _TRANSACTION_CONSULT_URL % (params["notificationCode"], email, token)
        resp = requests.get(url_consult)
        print resp.text
        logger.info("pagseguro notification: " + resp.text)

        dom = parseString(resp.text.encode('utf-8'))
        transactionNode = dom.getElementsByTagName("transaction")
        code = dom.getElementsByTagName("code")[0].firstChild.nodeValue
        if dom.getElementsByTagName("status"):
            status_code = int(dom.getElementsByTagName("status")[0].firstChild.nodeValue)

        reference = ""
        reference_node = dom.getElementsByTagName("reference")
        if reference_node:
            reference = reference_node[0].firstChild.nodeValue

        try:
            payment = Payment.objects.get(external_id__exact=reference)
        except Payment.DoesNotExist:
            logger.error('Payment does not exist with external_id=%s' % reference)
            return

        if status_code in (PagseguroTransactionStatus.AVAILABLE, PagseguroTransactionStatus.PAID):
            logger.info("Updating payment" + str(payment.id))
            amount = dom.getElementsByTagName("grossAmount")[0].firstChild.nodeValue
            payment.amount_paid = Decimal(amount)
            payment.paid_on = datetime.datetime.utcnow().replace(tzinfo=utc)
            payment.change_status('paid')

        elif status_code in (PagseguroTransactionStatus.CANCELED,
                             PagseguroTransactionStatus.REFUNDED,
                             PagseguroTransactionStatus.IN_DISPUTE):
            payment.change_status('failed')

    @staticmethod
    def _get_view_full_url(request, view_name, args=None):
        url = reverse(view_name, args=args)
        return 'http://%s%s' % (request.get_host(), url)
<|endoftext|>


<commit_before>from django import forms
from django.forms import Widget
from django.utils.encoding import force_unicode
from django.utils.safestring import mark_safe
from django import utils
import copy
try:
    import simplejson as json
except ImportError:
    import json

class SplitJSONWidget(forms.Widget):

    def __init__(self, attrs=None, newline='
\n', sep='__', debug=False):
        self.newline = newline
        self.separator = sep
        self.debug = debug
        Widget.__init__(self, attrs)

    def _as_text_field(self, name, key, value, is_sub=False):
        attrs = self.build_attrs(self.attrs, type='text',
                                 name="%s%s%s" % (name, self.separator, key))
        attrs['value'] = force_unicode(value)
        attrs['id'] = attrs.get('name', None)
        return u""" %s:
        <input%s />""" % (attrs['id'], key, forms.util.flatatt(attrs))

    def _to_build(self, name, json_obj):
        inputs = []
        if isinstance(json_obj, list):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in enumerate(json_obj):
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, dict):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in json_obj.items():
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, (str, int)):
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, json_obj))
        elif json_obj is None:
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, ''))
        return inputs

    def _prepare_as_ul(self, l):
        if l:
            result = ''
            for el in l:
                if isinstance(el, list) and len(l) is 1:
                    result += '%s' % self._prepare_as_ul(el)
                elif isinstance(el, list):
                    result += '
'
                    result += '%s' % self._prepare_as_ul(el)
                    result += '
'
                else:
                    result += '
%s
' % el
            return result
        return ''

    def _to_pack_up(self, root_node, raw_data):

        copy_raw_data = copy.deepcopy(raw_data)
        result = []

        def _to_parse_key(k, v):
            if k.find(self.separator) is not -1:
                apx, _, nk = k.rpartition(self.separator)
                try:
                    # parse list
                    int(nk)
                    l = []
                    obj = {}
                    index = None
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            head, _, t = key.rpartition(self.separator)
                            _, _, index = head.rpartition(self.separator)
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                try:
                                    int(t)
                                    l.append(val)
                                except ValueError:
                                    if index in obj:
                                        obj[index].update({t: val})
                                    else:
                                        obj[index] = {t: val}
                                del copy_raw_data[key]
                        if obj:
                            for i in obj:
                                l.append(obj[i])
                    l.append(v)
                    return _to_parse_key(apx, l)
                except ValueError:
                    # parse dict
                    d = {}
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            _, _, t = key.rpartition(self.separator)
                            try:
                                int(t)
                                continue
                            except ValueError:
                                pass
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                d.update({t: val})
                                del copy_raw_data[key]
                    v = {nk: v}
                    if d:
                        v.update(d)
                    return _to_parse_key(apx, v)
            else:
                return v

        for k, v in raw_data.iteritems():
            if k in copy_raw_data:
                # to transform value from list to string
                v = v[0] if isinstance(v, list) and len(v) is 1 else v
                if k.find(self.separator) is not -1:
                    d = _to_parse_key(k, v)
                    # set type result
                    if not len(result):
                        result = type(d)()
                    try:
                        result.extend(d)
                    except:
                        result.update(d)
        return result

    def value_from_datadict(self, data, files, name):
        data_copy = copy.deepcopy(data)
        result = self._to_pack_up(name, data_copy)
        return json.dumps(result)

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except TypeError:
            pass
        inputs = self._to_build(name, value or {})
        result = self._prepare_as_ul(inputs)
        if self.debug:
            # render json as well
            source_data = u'
Source data:
%s
' % str(value)
            result = '%s%s' % (result, source_data)
        return mark_safe(result)
<commit_msg>changes<commit_after>from django import forms
from django.forms import Widget
from django.utils.encoding import force_text
from django.utils.safestring import mark_safe
from django import utils
import copy
try:
    import simplejson as json
except ImportError:
    import json

class SplitJSONWidget(forms.Widget):

    def __init__(self, attrs=None, newline='
\n', sep='__', debug=False):
        self.newline = newline
        self.separator = sep
        self.debug = debug
        Widget.__init__(self, attrs)

    def _as_text_field(self, name, key, value, is_sub=False):
        attrs = self.build_attrs(self.attrs, type='text',
                                 name="%s%s%s" % (name, self.separator, key))
        attrs['value'] = force_text(value)
        attrs['id'] = attrs.get('name', None)
        return u""" %s:
        <input%s />""" % (attrs['id'], key, forms.util.flatatt(attrs))

    def _to_build(self, name, json_obj):
        inputs = []
        if isinstance(json_obj, list):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in enumerate(json_obj):
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, dict):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in json_obj.items():
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, (str, int)):
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, json_obj))
        elif json_obj is None:
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, ''))
        return inputs

    def _prepare_as_ul(self, l):
        if l:
            result = ''
            for el in l:
                if isinstance(el, list) and len(l) is 1:
                    result += '%s' % self._prepare_as_ul(el)
                elif isinstance(el, list):
                    result += '
'
                    result += '%s' % self._prepare_as_ul(el)
                    result += '
'
                else:
                    result += '
%s
' % el
            return result
        return ''

    def _to_pack_up(self, root_node, raw_data):

        copy_raw_data = copy.deepcopy(raw_data)
        result = []

        def _to_parse_key(k, v):
            if k.find(self.separator) is not -1:
                apx, _, nk = k.rpartition(self.separator)
                try:
                    # parse list
                    int(nk)
                    l = []
                    obj = {}
                    index = None
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            head, _, t = key.rpartition(self.separator)
                            _, _, index = head.rpartition(self.separator)
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                try:
                                    int(t)
                                    l.append(val)
                                except ValueError:
                                    if index in obj:
                                        obj[index].update({t: val})
                                    else:
                                        obj[index] = {t: val}
                                del copy_raw_data[key]
                        if obj:
                            for i in obj:
                                l.append(obj[i])
                    l.append(v)
                    return _to_parse_key(apx, l)
                except ValueError:
                    # parse dict
                    d = {}
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            _, _, t = key.rpartition(self.separator)
                            try:
                                int(t)
                                continue
                            except ValueError:
                                pass
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                d.update({t: val})
                                del copy_raw_data[key]
                    v = {nk: v}
                    if d:
                        v.update(d)
                    return _to_parse_key(apx, v)
            else:
                return v

        for k, v in raw_data.iteritems():
            if k in copy_raw_data:
                # to transform value from list to string
                v = v[0] if isinstance(v, list) and len(v) is 1 else v
                if k.find(self.separator) is not -1:
                    d = _to_parse_key(k, v)
                    # set type result
                    if not len(result):
                        result = type(d)()
                    try:
                        result.extend(d)
                    except:
                        result.update(d)
        return result

    def value_from_datadict(self, data, files, name):
        data_copy = copy.deepcopy(data)
        result = self._to_pack_up(name, data_copy)
        return json.dumps(result)

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except TypeError:
            pass
        inputs = self._to_build(name, value or {})
        result = self._prepare_as_ul(inputs)
        if self.debug:
            # render json as well
            source_data = u'
Source data:
%s
' % str(value)
            result = '%s%s' % (result, source_data)
        return mark_safe(result)
<|endoftext|>


<commit_before>#!/usr/bin/python
"""better_twill is a small wrapper around twill to set some sane defaults and
monkey-patch some better versions of some of twill's methods.
It also handles twill's absense.
"""

import os
from os.path import abspath, dirname, join
import sys
from pkg_resources import parse_version as pv
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

# On OSX lxml needs to be imported before twill to avoid Resolver issues
# somehow caused by the mac specific 'ic' module
try:
    from lxml import etree
except ImportError:
    pass

try:
    import twill
except ImportError:
    twill = None

# When twill tries to connect to a site before the site is up, it raises an
# exception.  In 0.9b1, it's urlib2.URLError, but in -latest, it's
# _mechanize_dist._mechanize.BrowserStateError.
try:
    from _mechanize_dist._mechanize import BrowserStateError as ConnectError
except ImportError:
    from urllib2 import URLError as ConnectError

if twill:
    # We want Trac to generate valid html, and therefore want to test against
    # the html as generated by Trac.  "tidy" tries to clean up broken html, and
    # is responsible for one difficult to track down testcase failure (for
    # #5497).  Therefore we turn it off here.
    twill.commands.config('use_tidy', '0')

    # setup short names to reduce typing
    # This twill browser (and the tc commands that use it) are essentially
    # global, and not tied to our test fixture.
    tc = twill.commands
    b = twill.get_browser()

    # Setup XHTML validation for all retrieved pages
    try:
        from lxml import etree
    except ImportError:
        print "SKIP: validation of XHTML output in functional tests " \
              "(no lxml installed)"
        etree = None

    if etree and pv(etree.__version__) < pv('2.0.0'):
        # 2.0.7 and 2.1.x are known to work.
        print "SKIP: validation of XHTML output in functional tests " \
              "(lxml < 2.0, api incompatibility)"
        etree = None

    if etree:
        class _Resolver(etree.Resolver):
            base_dir = dirname(abspath(__file__))

            def resolve(self, system_url, public_id, context):
                return self.resolve_filename(join(self.base_dir,
                                                  system_url.split("/")[-1]),
                                             context)

        _parser = etree.XMLParser(dtd_validation=True)
        _parser.resolvers.add(_Resolver())
        etree.set_default_parser(_parser)

        def _format_error_log(data, log):
            msg = []
            for entry in log:
                context = data.splitlines()[max(0, entry.line - 5):
                                            entry.line + 6]
                msg.append("\n# %s\n# URL: %s\n# Line %d, column %d\n\n%s\n"
                    % (entry.message, entry.filename,
                       entry.line, entry.column,
                       "\n".join([each.decode('utf-8') for each in context])))
            return "\n".join(msg).encode('ascii', 'xmlcharrefreplace')

        def _validate_xhtml(func_name, *args, **kwargs):
            page = b.get_html()
            if "xhtml1-strict.dtd" not in page:
                return
            etree.clear_error_log()
            try:
                doc = etree.parse(StringIO(page), base_url=b.get_url())
            except etree.XMLSyntaxError, e:
                raise twill.errors.TwillAssertionError(
                    _format_error_log(page, e.error_log))

        b._post_load_hooks.append(_validate_xhtml)

    # When we can't find something we expected, or find something we didn't
    # expect, it helps the debugging effort to have a copy of the html to
    # analyze.
    def twill_write_html():
        """Write the current html to a file.  Name the file based on the
        current testcase.
        """
        frame = sys._getframe()
        while frame:
            if frame.f_code.co_name in ('runTest', 'setUp', 'tearDown'):
                testcase = frame.f_locals['self']
                testname = testcase.__class__.__name__
                tracdir = testcase._testenv.tracdir
                break
            frame = frame.f_back
        else:
            # We didn't find a testcase in the stack, so we have no clue what's
            # going on.
            raise Exception("No testcase was found on the stack.  This was "
                "really not expected, and I don't know how to handle it.")

        filename = os.path.join(tracdir, 'log', "%s.html" % testname)
        html_file = open(filename, 'w')
        html_file.write(b.get_html())
        html_file.close()

        return filename

    # Twill isn't as helpful with errors as I'd like it to be, so we replace
    # the formvalue function.  This would be better done as a patch to Twill.
    def better_formvalue(form, field, value, fv=tc.formvalue):
        try:
            fv(form, field, value)
        except (twill.errors.TwillAssertionError,
                twill.utils.ClientForm.ItemNotFoundError), e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.formvalue = better_formvalue

    # Twill's formfile function leaves a filehandle open which prevents the
    # file from being deleted on Windows.  Since we would just assume use a
    # StringIO object in the first place, allow the file-like object to be
    # provided directly.
    def better_formfile(formname, fieldname, filename, content_type=None,
                        fp=None):
        if not fp:
            filename = filename.replace('/', os.path.sep)
            temp_fp = open(filename, 'rb')
            data = temp_fp.read()
            temp_fp.close()
            fp = StringIO(data)

        form = b.get_form(formname)
        control = b.get_form_field(form, fieldname)

        if not control.is_of_kind('file'):
            raise twill.errors.TwillException('ERROR: field is not a file '
                                              'upload field!')

        b.clicked(form, control)
        control.add_file(fp, content_type, filename)
    tc.formfile = better_formfile

    # Twill's tc.find() does not provide any guidance on what we got instead of
    # what was expected.
    def better_find(what, flags='', tcfind=tc.find):
        try:
            tcfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.find = better_find
    def better_notfind(what, flags='', tcnotfind=tc.notfind):
        try:
            tcnotfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.notfind = better_notfind
else:
    b = tc = None
<commit_msg>0.12dev: Wrap the global twill browser object in a transparent proxy that accesses the object via `twill.get_browser()`, as the object can change after a `browser_reset()` call.<commit_after>#!/usr/bin/python
"""better_twill is a small wrapper around twill to set some sane defaults and
monkey-patch some better versions of some of twill's methods.
It also handles twill's absense.
"""

import os
from os.path import abspath, dirname, join
import sys
from pkg_resources import parse_version as pv
try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

# On OSX lxml needs to be imported before twill to avoid Resolver issues
# somehow caused by the mac specific 'ic' module
try:
    from lxml import etree
except ImportError:
    pass

try:
    import twill
except ImportError:
    twill = None

# When twill tries to connect to a site before the site is up, it raises an
# exception.  In 0.9b1, it's urlib2.URLError, but in -latest, it's
# _mechanize_dist._mechanize.BrowserStateError.
try:
    from _mechanize_dist._mechanize import BrowserStateError as ConnectError
except ImportError:
    from urllib2 import URLError as ConnectError

if twill:
    # We want Trac to generate valid html, and therefore want to test against
    # the html as generated by Trac.  "tidy" tries to clean up broken html, and
    # is responsible for one difficult to track down testcase failure (for
    # #5497).  Therefore we turn it off here.
    twill.commands.config('use_tidy', '0')

    # We use a transparent proxy to access the global browser object through
    # twill.get_browser(), as the browser can be destroyed by browser_reset()
    # (see #7472).
    class _BrowserProxy(object):
        def __getattribute__(self, name):
            return getattr(twill.get_browser(), name)

        def __setattr(self, name, value):
            setattr(twill.get_browser(), name, value)

    # setup short names to reduce typing
    # This twill browser (and the tc commands that use it) are essentially
    # global, and not tied to our test fixture.
    tc = twill.commands
    b = _BrowserProxy()

    # Setup XHTML validation for all retrieved pages
    try:
        from lxml import etree
    except ImportError:
        print "SKIP: validation of XHTML output in functional tests " \
              "(no lxml installed)"
        etree = None

    if etree and pv(etree.__version__) < pv('2.0.0'):
        # 2.0.7 and 2.1.x are known to work.
        print "SKIP: validation of XHTML output in functional tests " \
              "(lxml < 2.0, api incompatibility)"
        etree = None

    if etree:
        class _Resolver(etree.Resolver):
            base_dir = dirname(abspath(__file__))

            def resolve(self, system_url, public_id, context):
                return self.resolve_filename(join(self.base_dir,
                                                  system_url.split("/")[-1]),
                                             context)

        _parser = etree.XMLParser(dtd_validation=True)
        _parser.resolvers.add(_Resolver())
        etree.set_default_parser(_parser)

        def _format_error_log(data, log):
            msg = []
            for entry in log:
                context = data.splitlines()[max(0, entry.line - 5):
                                            entry.line + 6]
                msg.append("\n# %s\n# URL: %s\n# Line %d, column %d\n\n%s\n"
                    % (entry.message, entry.filename,
                       entry.line, entry.column,
                       "\n".join([each.decode('utf-8') for each in context])))
            return "\n".join(msg).encode('ascii', 'xmlcharrefreplace')

        def _validate_xhtml(func_name, *args, **kwargs):
            page = b.get_html()
            if "xhtml1-strict.dtd" not in page:
                return
            etree.clear_error_log()
            try:
                doc = etree.parse(StringIO(page), base_url=b.get_url())
            except etree.XMLSyntaxError, e:
                raise twill.errors.TwillAssertionError(
                    _format_error_log(page, e.error_log))

        b._post_load_hooks.append(_validate_xhtml)

    # When we can't find something we expected, or find something we didn't
    # expect, it helps the debugging effort to have a copy of the html to
    # analyze.
    def twill_write_html():
        """Write the current html to a file.  Name the file based on the
        current testcase.
        """
        frame = sys._getframe()
        while frame:
            if frame.f_code.co_name in ('runTest', 'setUp', 'tearDown'):
                testcase = frame.f_locals['self']
                testname = testcase.__class__.__name__
                tracdir = testcase._testenv.tracdir
                break
            frame = frame.f_back
        else:
            # We didn't find a testcase in the stack, so we have no clue what's
            # going on.
            raise Exception("No testcase was found on the stack.  This was "
                "really not expected, and I don't know how to handle it.")

        filename = os.path.join(tracdir, 'log', "%s.html" % testname)
        html_file = open(filename, 'w')
        html_file.write(b.get_html())
        html_file.close()

        return filename

    # Twill isn't as helpful with errors as I'd like it to be, so we replace
    # the formvalue function.  This would be better done as a patch to Twill.
    def better_formvalue(form, field, value, fv=tc.formvalue):
        try:
            fv(form, field, value)
        except (twill.errors.TwillAssertionError,
                twill.utils.ClientForm.ItemNotFoundError), e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.formvalue = better_formvalue

    # Twill's formfile function leaves a filehandle open which prevents the
    # file from being deleted on Windows.  Since we would just assume use a
    # StringIO object in the first place, allow the file-like object to be
    # provided directly.
    def better_formfile(formname, fieldname, filename, content_type=None,
                        fp=None):
        if not fp:
            filename = filename.replace('/', os.path.sep)
            temp_fp = open(filename, 'rb')
            data = temp_fp.read()
            temp_fp.close()
            fp = StringIO(data)

        form = b.get_form(formname)
        control = b.get_form_field(form, fieldname)

        if not control.is_of_kind('file'):
            raise twill.errors.TwillException('ERROR: field is not a file '
                                              'upload field!')

        b.clicked(form, control)
        control.add_file(fp, content_type, filename)
    tc.formfile = better_formfile

    # Twill's tc.find() does not provide any guidance on what we got instead of
    # what was expected.
    def better_find(what, flags='', tcfind=tc.find):
        try:
            tcfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.find = better_find
    def better_notfind(what, flags='', tcnotfind=tc.notfind):
        try:
            tcnotfind(what, flags)
        except twill.errors.TwillAssertionError, e:
            filename = twill_write_html()
            args = e.args + (filename,)
            raise twill.errors.TwillAssertionError(*args)
    tc.notfind = better_notfind
else:
    b = tc = None
<|endoftext|>


<commit_before>try:
    import collections.abc as collectionsabc
except ImportError:
    import collections as collectionsabc
import io
import json
import threading

__version__ = '1.1.0'

try:
    import builtins
    import pathlib
except ImportError:
    pass
else:
    def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
        if isinstance(file, pathlib.Path):
            return file.open(mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline)
        else:
            return builtins.open(file, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd, opener=opener)

class Node:
    def __deepcopy__(self, memodict={}):
        return self.value()

    def __init__(self, root, key_path=None):
        if not isinstance(root, File):
            root = File(root)
        self.root = root
        self.key_path = [] if key_path is None else key_path[:]

    def __str__(self):
        return str(self.value())

    def __repr__(self):
        return 'lazyjson.Node(' + repr(self.root.file_info) + ', ' + repr(self.key_path) + ')'

    def value(self):
        return self.root.value_at_key_path(self.key_path)

class Dict(Node, collectionsabc.MutableMapping):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __iter__(self):
        for key in self.value():
            yield key

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

class List(Node, collectionsabc.MutableSequence):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

    def insert(self, index, value):
        self.root.insert_value_at_key_path(self.key_path + [index], value)

class File(Dict):
    def __init__(self, file_info, file_is_open=None):
        self.file_is_open = isinstance(file_info, io.IOBase) if file_is_open is None else file_is_open
        self.file_info = file_info
        self.lock = threading.Lock()
        super().__init__(self)

    def delete_value_at_key_path(self, key_path):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = {}
        else:
            for key in key_path[:-1]:
                item = item[key]
            del item[key_path[-1]]
        self.set(json_dict)

    def insert_value_at_key_path(self, key_path, value):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item.insert(key_path[-1], value)
        self.set(json_dict)

    def set(self, value):
        if isinstance(value, Node):
            value = value.value()
        with self.lock:
            if self.file_is_open:
                json.dump(value, self.file_info, sort_keys=True, indent=4, separators=(',', ': '))
                print(file=self.file_info) # json.dump doesn't end the file in a newline, so add it manually
            else:
                with open(self.file_info, 'w') as json_file:
                    json.dump(value, json_file, sort_keys=True, indent=4, separators=(',', ': '))
                    print(file=json_file) # json.dump doesn't end the file in a newline, so add it manually

    def set_value_at_key_path(self, key_path, value):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item[key_path[-1]] = value
        self.set(json_dict)

    def value_at_key_path(self, key_path):
        if self.file_is_open:
            ret = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                ret = json.load(json_file)
        for key in key_path:
            ret = ret[key]
        return ret
<commit_msg>Add failsafe to prevent writing illegal values to file<commit_after>try:
    import collections.abc as collectionsabc
except ImportError:
    import collections as collectionsabc
import io
import json
import threading

__version__ = '1.1.0'

try:
    import builtins
    import pathlib
except ImportError:
    pass
else:
    def open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None):
        if isinstance(file, pathlib.Path):
            return file.open(mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline)
        else:
            return builtins.open(file, mode=mode, buffering=buffering, encoding=encoding, errors=errors, newline=newline, closefd=closefd, opener=opener)

class Node:
    def __deepcopy__(self, memodict={}):
        return self.value()

    def __init__(self, root, key_path=None):
        if not isinstance(root, File):
            root = File(root)
        self.root = root
        self.key_path = [] if key_path is None else key_path[:]

    def __str__(self):
        return str(self.value())

    def __repr__(self):
        return 'lazyjson.Node(' + repr(self.root.file_info) + ', ' + repr(self.key_path) + ')'

    def value(self):
        return self.root.value_at_key_path(self.key_path)

class Dict(Node, collectionsabc.MutableMapping):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __iter__(self):
        for key in self.value():
            yield key

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

class List(Node, collectionsabc.MutableSequence):
    def __delitem__(self, key):
        self.root.delete_value_at_key_path(self.key_path + [key])

    def __getitem__(self, key):
        if isinstance(self.value()[key], dict):
            return Dict(self.root, self.key_path + [key])
        elif isinstance(self.value()[key], list):
            return List(self.root, self.key_path + [key])
        else:
            return self.value()[key]

    def __len__(self):
        return len(self.value())

    def __setitem__(self, key, value):
        if isinstance(value, Node):
            value = value.value()
        self.root.set_value_at_key_path(self.key_path + [key], value)

    def insert(self, index, value):
        self.root.insert_value_at_key_path(self.key_path + [index], value)

class File(Dict):
    def __init__(self, file_info, file_is_open=None):
        self.file_is_open = isinstance(file_info, io.IOBase) if file_is_open is None else file_is_open
        self.file_info = file_info
        self.lock = threading.Lock()
        super().__init__(self)

    def delete_value_at_key_path(self, key_path):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = {}
        else:
            for key in key_path[:-1]:
                item = item[key]
            del item[key_path[-1]]
        self.set(json_dict)

    def insert_value_at_key_path(self, key_path, value):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item.insert(key_path[-1], value)
        self.set(json_dict)

    def set(self, value):
        if isinstance(value, Node):
            value = value.value()
        json.dumps(value) # try writing the value to a string first to prevent corrupting the file if the value is not JSON serializable
        with self.lock:
            if self.file_is_open:
                json.dump(value, self.file_info, sort_keys=True, indent=4, separators=(',', ': '))
                print(file=self.file_info) # json.dump doesn't end the file in a newline, so add it manually
            else:
                with open(self.file_info, 'w') as json_file:
                    json.dump(value, json_file, sort_keys=True, indent=4, separators=(',', ': '))
                    print(file=json_file) # json.dump doesn't end the file in a newline, so add it manually

    def set_value_at_key_path(self, key_path, value):
        if self.file_is_open:
            json_dict = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                json_dict = json.load(json_file)
        item = json_dict
        if len(key_path) == 0:
            json_dict = value
        else:
            for key in key_path[:-1]:
                item = item[key]
            item[key_path[-1]] = value
        self.set(json_dict)

    def value_at_key_path(self, key_path):
        if self.file_is_open:
            ret = json.load(self.file_info)
        else:
            with open(self.file_info) as json_file:
                ret = json.load(json_file)
        for key in key_path:
            ret = ret[key]
        return ret
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (c) 2018 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

import time
from datetime import datetime
from uuid import uuid4

import requests
from bson import ObjectId
from flask import render_template
from flask_babel import gettext as _

from eduid_userdb import MailAddress
from eduid_userdb.logs import MailAddressProofing
from eduid_userdb.proofing import EmailProofingElement
from eduid_userdb.signup import SignupUser

from eduid_webapp.signup.app import current_signup_app as current_app
from eduid_webapp.signup.helpers import generate_eppn

def verify_recaptcha(secret_key, captcha_response, user_ip, retries=3):
    """
    Verify the recaptcha response received from the client
    against the recaptcha API.

    :param secret_key: Recaptcha secret key
    :param captcha_response: User recaptcha response
    :param user_ip: User ip address
    :param retries: Number of times to retry sending recaptcha response

    :type secret_key: str
    :type captcha_response: str
    :type user_ip: str
    :type retries: int

    :return: True|False
    :rtype: bool
    """
    url = 'https://www.google.com/recaptcha/api/siteverify'
    params = {'secret': secret_key, 'response': captcha_response, 'remoteip': user_ip}
    while retries:
        retries -= 1
        try:
            current_app.logger.debug('Sending the CAPTCHA response')
            verify_rs = requests.get(url, params=params, verify=True)
            current_app.logger.debug("CAPTCHA response: {}".format(verify_rs))
            verify_rs = verify_rs.json()
            if verify_rs.get('success', False) is True:
                current_app.logger.info("Valid CAPTCHA response from " "{}".format(user_ip))
                return True
        except requests.exceptions.RequestException as e:
            if not retries:
                current_app.logger.error('Caught RequestException while ' 'sending CAPTCHA, giving up.')
                raise e
            current_app.logger.warning('Caught RequestException while ' 'sending CAPTCHA, trying again.')
            current_app.logger.warning(e)
            time.sleep(0.5)

    current_app.logger.info(
        "Invalid CAPTCHA response from {}: {}".format(user_ip, verify_rs.get('error-codes', 'Unspecified error'))
    )
    return False

def generate_verification_link():
    """
    Generate a verification code and build a verification link with it.

    :return: code and link
    :rtype: pair of str
    """
    code = str(uuid4())
    link = '{}code/{}'.format(current_app.config.signup_url, code)
    return link, code

def send_verification_mail(email):
    """
    Render and send an email with a verification code/link
    for the provided email.

    :param email: Email address to verify
    :type email: str | unicode
    """
    verification_link, code = generate_verification_link()

    signup_user = current_app.private_userdb.get_user_by_pending_mail_address(email)
    if not signup_user:
        mailaddress = EmailProofingElement(email=email, created_by='signup', is_verified=False, verification_code=code)
        # TODO: Let User create a user_id if it is None (as default)
        signup_user = SignupUser(user_id=ObjectId(), eppn=generate_eppn())
        signup_user.pending_mail_address = mailaddress
        current_app.logger.info("New user {}/{} created. e-mail is pending confirmation".format(signup_user, email))
    else:
        # update mailaddress on existing user with new code
        signup_user.pending_mail_address.verification_code = code
        current_app.logger.info("User {}/{} updated with new e-mail confirmation code".format(signup_user, email))

    # Send verification mail
    subject = _("eduid-signup verification email")

    context = {
        "email": email,
        "verification_link": verification_link,
        "site_name": current_app.config.eduid_site_name,
        "site_url": current_app.config.eduid_site_url,
    }

    text = render_template("verification_email.txt.jinja2", **context)
    html = render_template("verification_email.html.jinja2", **context)

    current_app.mail_relay.sendmail(subject, [email], text, html, reference=signup_user.proofing_reference)
    current_app.logger.info("Sent email address verification mail for user {} to address {}".format(signup_user, email))
    current_app.stats.count(name='mail_sent')
    current_app.private_userdb.save(signup_user)
    current_app.logger.info("Saved user {} to private db".format(signup_user))

class AlreadyVerifiedException(Exception):
    pass

class CodeDoesNotExist(Exception):
    pass

class ProofingLogFailure(Exception):
    pass

def verify_email_code(code):
    """
    Look up a user in the signup userdb using an e-mail verification code.

    Mark the e-mail address as confirmed, save the user and return the user object.

    :param code: Code as received from user
    :type code: str | unicode

    :return: Signup user object
    :rtype: SignupUser
    """
    current_app.logger.info("Trying to verify code {}".format(code))

    signup_db = current_app.private_userdb
    signup_user = signup_db.get_user_by_mail_verification_code(code)

    if not signup_user:
        current_app.logger.debug("Code {} not found in database".format(code))
        raise CodeDoesNotExist()

    email = signup_user.pending_mail_address.email
    user = current_app.central_userdb.get_user_by_mail(email, raise_on_missing=False)

    if user:
        current_app.logger.debug("Email {} already present in central db".format(email))
        raise AlreadyVerifiedException()

    mail_address = MailAddress(signup_user.pending_mail_address)
    if mail_address.is_verified:
        # There really should be no way to get here, is_verified is set to False when
        # the EmailProofingElement is created.
        current_app.logger.debug("Code {} already verified ({})".format(code, mail_address))
        raise AlreadyVerifiedException()

    mail_address_proofing = MailAddressProofing(
        eppn=signup_user.eppn,
        created_by='signup',
        mail_address=mail_address.email,
        reference=signup_user.proofing_reference,
        proofing_version='2013v1',
    )
    if current_app.proofing_log.save(mail_address_proofing):
        mail_address.is_verified = True
        mail_address.verified_ts = datetime.utcnow()
        mail_address.verified_by = 'signup'
        mail_address.is_primary = True
        signup_user.pending_mail_address = None
        signup_user.mail_addresses.add(mail_address)
        result = signup_db.save(signup_user)
        current_app.logger.info("Code {} verified and user {} saved: {!r}".format(code, signup_user, result))
        current_app.stats.count(name='mail_verified')
        return signup_user
    else:
        current_app.logger.error('Failed to save proofing log, aborting')
        raise ProofingLogFailure()
<commit_msg>Instantiate MailAddress correctly<commit_after># -*- coding: utf-8 -*-
#
# Copyright (c) 2018 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

import time
from datetime import datetime
from uuid import uuid4

import requests
from bson import ObjectId
from flask import render_template
from flask_babel import gettext as _

from eduid_userdb import MailAddress
from eduid_userdb.logs import MailAddressProofing
from eduid_userdb.proofing import EmailProofingElement
from eduid_userdb.signup import SignupUser

from eduid_webapp.signup.app import current_signup_app as current_app
from eduid_webapp.signup.helpers import generate_eppn

def verify_recaptcha(secret_key, captcha_response, user_ip, retries=3):
    """
    Verify the recaptcha response received from the client
    against the recaptcha API.

    :param secret_key: Recaptcha secret key
    :param captcha_response: User recaptcha response
    :param user_ip: User ip address
    :param retries: Number of times to retry sending recaptcha response

    :type secret_key: str
    :type captcha_response: str
    :type user_ip: str
    :type retries: int

    :return: True|False
    :rtype: bool
    """
    url = 'https://www.google.com/recaptcha/api/siteverify'
    params = {'secret': secret_key, 'response': captcha_response, 'remoteip': user_ip}
    while retries:
        retries -= 1
        try:
            current_app.logger.debug('Sending the CAPTCHA response')
            verify_rs = requests.get(url, params=params, verify=True)
            current_app.logger.debug("CAPTCHA response: {}".format(verify_rs))
            verify_rs = verify_rs.json()
            if verify_rs.get('success', False) is True:
                current_app.logger.info("Valid CAPTCHA response from " "{}".format(user_ip))
                return True
        except requests.exceptions.RequestException as e:
            if not retries:
                current_app.logger.error('Caught RequestException while ' 'sending CAPTCHA, giving up.')
                raise e
            current_app.logger.warning('Caught RequestException while ' 'sending CAPTCHA, trying again.')
            current_app.logger.warning(e)
            time.sleep(0.5)

    current_app.logger.info(
        "Invalid CAPTCHA response from {}: {}".format(user_ip, verify_rs.get('error-codes', 'Unspecified error'))
    )
    return False

def generate_verification_link():
    """
    Generate a verification code and build a verification link with it.

    :return: code and link
    :rtype: pair of str
    """
    code = str(uuid4())
    link = '{}code/{}'.format(current_app.config.signup_url, code)
    return link, code

def send_verification_mail(email):
    """
    Render and send an email with a verification code/link
    for the provided email.

    :param email: Email address to verify
    :type email: str | unicode
    """
    verification_link, code = generate_verification_link()

    signup_user = current_app.private_userdb.get_user_by_pending_mail_address(email)
    if not signup_user:
        mailaddress = EmailProofingElement(email=email, created_by='signup', is_verified=False, verification_code=code)
        # TODO: Let User create a user_id if it is None (as default)
        signup_user = SignupUser(user_id=ObjectId(), eppn=generate_eppn())
        signup_user.pending_mail_address = mailaddress
        current_app.logger.info("New user {}/{} created. e-mail is pending confirmation".format(signup_user, email))
    else:
        # update mailaddress on existing user with new code
        signup_user.pending_mail_address.verification_code = code
        current_app.logger.info("User {}/{} updated with new e-mail confirmation code".format(signup_user, email))

    # Send verification mail
    subject = _("eduid-signup verification email")

    context = {
        "email": email,
        "verification_link": verification_link,
        "site_name": current_app.config.eduid_site_name,
        "site_url": current_app.config.eduid_site_url,
    }

    text = render_template("verification_email.txt.jinja2", **context)
    html = render_template("verification_email.html.jinja2", **context)

    current_app.mail_relay.sendmail(subject, [email], text, html, reference=signup_user.proofing_reference)
    current_app.logger.info("Sent email address verification mail for user {} to address {}".format(signup_user, email))
    current_app.stats.count(name='mail_sent')
    current_app.private_userdb.save(signup_user)
    current_app.logger.info("Saved user {} to private db".format(signup_user))

class AlreadyVerifiedException(Exception):
    pass

class CodeDoesNotExist(Exception):
    pass

class ProofingLogFailure(Exception):
    pass

def verify_email_code(code):
    """
    Look up a user in the signup userdb using an e-mail verification code.

    Mark the e-mail address as confirmed, save the user and return the user object.

    :param code: Code as received from user
    :type code: str | unicode

    :return: Signup user object
    :rtype: SignupUser
    """
    current_app.logger.info("Trying to verify code {}".format(code))

    signup_db = current_app.private_userdb
    signup_user = signup_db.get_user_by_mail_verification_code(code)

    if not signup_user:
        current_app.logger.debug("Code {} not found in database".format(code))
        raise CodeDoesNotExist()

    email = signup_user.pending_mail_address.email
    user = current_app.central_userdb.get_user_by_mail(email, raise_on_missing=False)

    if user:
        current_app.logger.debug("Email {} already present in central db".format(email))
        raise AlreadyVerifiedException()

    pending_mail_address: EmailProofingElement = signup_user.pending_mail_address
    mail_address = MailAddress(
        email=pending_mail_address.email,
        created_by=pending_mail_address.created_by,
        created_ts=pending_mail_address.created_ts,
        modified_ts=pending_mail_address.modified_ts,
        is_verified=pending_mail_address.is_verified,
        is_primary=False,
    )
    if mail_address.is_verified:
        # There really should be no way to get here, is_verified is set to False when
        # the EmailProofingElement is created.
        current_app.logger.debug("Code {} already verified ({})".format(code, mail_address))
        raise AlreadyVerifiedException()

    mail_address_proofing = MailAddressProofing(
        eppn=signup_user.eppn,
        created_by='signup',
        mail_address=mail_address.email,
        reference=signup_user.proofing_reference,
        proofing_version='2013v1',
    )
    if current_app.proofing_log.save(mail_address_proofing):
        mail_address.is_verified = True
        mail_address.verified_ts = datetime.utcnow()
        mail_address.verified_by = 'signup'
        mail_address.is_primary = True
        signup_user.pending_mail_address = None
        signup_user.mail_addresses.add(mail_address)
        result = signup_db.save(signup_user)
        current_app.logger.info("Code {} verified and user {} saved: {!r}".format(code, signup_user, result))
        current_app.stats.count(name='mail_verified')
        return signup_user
    else:
        current_app.logger.error('Failed to save proofing log, aborting')
        raise ProofingLogFailure()
<|endoftext|>


<commit_before>import subprocess, socket, math

import diamond.collector

def parse_line(line):
    metric_name, rhs = line.strip().split(':', 1)
    rhs = rhs.strip()
    if ' ' in rhs:
        str_value, units = rhs.split(' ', 1)
        if units not in ('ms', 'ms.'):
            raise ValueError("Cannot parse " + repr(line))
    else:
        str_value = rhs

    try:
        value = float(str_value)
    except:
        value = str_value

    return metric_name, value

class Keyspace(object):
    def __init__(self, name, stats, tables):
        self.name = name
        self.stats = stats
        self.tables = tables

class Table(object):
    def __init__(self, name, stats):
        self.name = name
        self.stats = stats

def clean_key(key):
    return key.replace(' ', '_').replace(',', '_').replace('(', '').replace(')', '')

bad_keyspaces = ('system', 'system_traces')

class ColumnFamilyStatsCollector(diamond.collector.Collector):

    last_read = {}
    last_write = {}

    def collect(self):
        for keyspace in self.cfstats():
            if keyspace.name not in bad_keyspaces:
                for (key, value) in keyspace.stats:
                    name = 'cassandra.cfstats.{}.{}'.format(
                        keyspace.name, key)
                    self.publish(name, value)
                for table in keyspace.tables:
                    for (key, value) in table.stats:
                        name = 'cassandra.cfstats.{}.{}.{}'.format(
                            keyspace.name, table.name, key)
                        self.publish(name, value)

    def get_periodic_rw(self, history_dict, key, value):
        if history_dict.get(key, 0) == 0:
            history_dict[key] = value

        periodic_value = value - history_dict.get(key, 0)
        history_dict[key] = value
        return periodic_value

    def cfstats(self):
        output = subprocess.check_output(['nodetool', 'cfstats'])
        lines = [line for line in output.splitlines()
                 if line and (line != '----------------')]

        # cfstats output is structured in a very specific way: all lines are
        # key: value pairs prefixed by tabs. everything indented belongs to the

        keyspaces = []
        ks_name = ""
        table_name = ""
        for line in lines:
            try:

                tab_count = len(line) - len(line.lstrip('\t'))
                if tab_count == 0:
                    key, value = parse_line(line)
                    assert key == 'Keyspace'
                    ks_name = value
                    keyspaces.append(Keyspace(value, [], []))
                elif tab_count == 1:
                    key, value = parse_line(line)
                    if not math.isnan(value):

                        if key == "Read Count":
                            value = self.get_periodic_rw(
                                        ColumnFamilyStatsCollector.last_read,
                                        ks_name, value)

                        elif key == "Write Count":
                            value = self.get_periodic_rw(
                                        ColumnFamilyStatsCollector.last_write,
                                        ks_name, value)

                        keyspaces[-1].stats.append((clean_key(key), value))

                elif tab_count == 2:
                    key, value = parse_line(line)
                    if key == 'Table':
                        table_name = value
                        keyspaces[-1].tables.append(Table(value, []))
                    else:
                        if not math.isnan(value):
                            key_name = ks_name + table_name
                            if key == "Local read count":
                                value = self.get_periodic_rw(
                                            ColumnFamilyStatsCollector.last_read,
                                            key_name, value)

                            elif key == "Local write count":
                                value = self.get_periodic_rw(
                                            ColumnFamilyStatsCollector.last_write,
                                            key_name, value)

                            keyspaces[-1].tables[-1].stats.append((clean_key(key), value))
                else:
                    raise ValueError
            except ValueError:
                self.log.error("Unable to parse line: %s" % line)

        return keyspaces
<commit_msg>SONIC-5930 add local read/write rate key value pair<commit_after>import subprocess, socket, math

import diamond.collector

def parse_line(line):
    metric_name, rhs = line.strip().split(':', 1)
    rhs = rhs.strip()
    if ' ' in rhs:
        str_value, units = rhs.split(' ', 1)
        if units not in ('ms', 'ms.'):
            raise ValueError("Cannot parse " + repr(line))
    else:
        str_value = rhs

    try:
        value = float(str_value)
    except:
        value = str_value

    return metric_name, value

class Keyspace(object):
    def __init__(self, name, stats, tables):
        self.name = name
        self.stats = stats
        self.tables = tables

class Table(object):
    def __init__(self, name, stats):
        self.name = name
        self.stats = stats

def clean_key(key):
    return key.replace(' ', '_').replace(',', '_').replace('(', '').replace(')', '')

bad_keyspaces = ('system', 'system_traces')

class ColumnFamilyStatsCollector(diamond.collector.Collector):

    last_read = {}
    last_write = {}

    def collect(self):
        for keyspace in self.cfstats():
            if keyspace.name not in bad_keyspaces:
                for (key, value) in keyspace.stats:
                    name = 'cassandra.cfstats.{}.{}'.format(
                        keyspace.name, key)
                    self.publish(name, value)
                for table in keyspace.tables:
                    for (key, value) in table.stats:
                        name = 'cassandra.cfstats.{}.{}.{}'.format(
                            keyspace.name, table.name, key)
                        self.publish(name, value)

    def get_periodic_rw(self, history_dict, key, value):
        if history_dict.get(key, 0) == 0:
            history_dict[key] = value

        periodic_value = value - history_dict.get(key, 0)
        history_dict[key] = value
        return periodic_value

    def cfstats(self):
        output = subprocess.check_output(['nodetool', 'cfstats'])
        lines = [line for line in output.splitlines()
                 if line and (line != '----------------')]

        # cfstats output is structured in a very specific way: all lines are
        # key: value pairs prefixed by tabs. everything indented belongs to the

        keyspaces = []
        ks_name = ""
        table_name = ""
        for line in lines:
            try:

                tab_count = len(line) - len(line.lstrip('\t'))
                if tab_count == 0:
                    key, value = parse_line(line)
                    assert key == 'Keyspace'
                    ks_name = value
                    keyspaces.append(Keyspace(value, [], []))
                elif tab_count == 1:
                    key, value = parse_line(line)
                    if not math.isnan(value):

                        if key == "Read Count":
                            value = self.get_periodic_rw(
                                        ColumnFamilyStatsCollector.last_read,
                                        ks_name, value)

                        elif key == "Write Count":
                            value = self.get_periodic_rw(
                                        ColumnFamilyStatsCollector.last_write,
                                        ks_name, value)

                        keyspaces[-1].stats.append((clean_key(key), value))

                elif tab_count == 2:
                    key, value = parse_line(line)
                    if key == 'Table':
                        table_name = value
                        keyspaces[-1].tables.append(Table(value, []))
                    else:
                        if not math.isnan(value):
                            key_name = ks_name + table_name
                            if key == "Local read count":
                                rate_value = self.get_periodic_rw(
                                            ColumnFamilyStatsCollector.last_read,
                                            key_name, value)

                                keyspaces[-1].tables[-1].stats.append(
                                    (clean_key("Local_read_rate"), rate_value))

                            elif key == "Local write count":
                                rate_value = self.get_periodic_rw(
                                            ColumnFamilyStatsCollector.last_write,
                                            key_name, value)

                                keyspaces[-1].tables[-1].stats.append(
                                    (clean_key("Local_write_rate"), rate_value))

                            keyspaces[-1].tables[-1].stats.append((clean_key(key), value))
                else:
                    raise ValueError
            except ValueError:
                self.log.error("Unable to parse line: %s" % line)

        return keyspaces
<|endoftext|>


<commit_before>from django import get_version, forms
from django.forms import Widget
from django import utils
import copy
from distutils.version import StrictVersion
try:
    import simplejson as json
except ImportError:
    import json
if StrictVersion(get_version()) < StrictVersion('1.9.0'):
    from django.forms.util import flatatt
else:
    from django.forms.utils import flatatt

class SplitJSONWidget(forms.Widget):

    def __init__(self, attrs=None, newline='
\n', sep='__', debug=False):
        self.newline = newline
        self.separator = sep
        self.debug = debug
        Widget.__init__(self, attrs)

    def _as_text_field(self, name, key, value, is_sub=False):
        attrs = self.build_attrs(self.attrs, type='text',
                                 name="%s%s%s" % (name, self.separator, key))
        attrs['value'] = utils.encoding.force_unicode(value)
        attrs['id'] = attrs.get('name', None)
        return u""" %s:
        <input%s />""" % (attrs['id'], key, flatatt(attrs))

    def _to_build(self, name, json_obj):
        inputs = []
        if isinstance(json_obj, list):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in enumerate(json_obj):
                _l.append(self._to_build("%s%s%s" % (name,
                                         self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, dict):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in json_obj.items():
                _l.append(self._to_build("%s%s%s" % (name,
                                                     self.separator, key),
                                         value))
            inputs.extend([_l])
        elif isinstance(json_obj, (basestring, int, float)):
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, json_obj))
        elif json_obj is None:
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, ''))
        return inputs

    def _prepare_as_ul(self, l):
        if l:
            result = ''
            for el in l:
                if isinstance(el, list) and len(l) == 1:
                    result += '%s' % self._prepare_as_ul(el)
                elif isinstance(el, list):
                    result += '
'
                    result += '%s' % self._prepare_as_ul(el)
                    result += '
'
                else:
                    result += '
%s
' % el
            return result
        return ''

    def _to_pack_up(self, root_node, raw_data):

        copy_raw_data = copy.deepcopy(raw_data)
        result = []

        def _to_parse_key(k, v):
            if k.find(self.separator) != -1:
                apx, _, nk = k.rpartition(self.separator)
                try:
                    # parse list
                    int(nk)
                    l = []
                    obj = {}
                    index = None
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            head, _, t = key.rpartition(self.separator)
                            _, _, index = head.rpartition(self.separator)
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                try:
                                    int(t)
                                    l.append(val)
                                except ValueError:
                                    if index in obj:
                                        obj[index].update({t: val})
                                    else:
                                        obj[index] = {t: val}
                                del copy_raw_data[key]
                        if obj:
                            for i in obj:
                                l.append(obj[i])
                    l.append(v)
                    return _to_parse_key(apx, l)
                except ValueError:
                    # parse dict
                    d = {}
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            _, _, t = key.rpartition(self.separator)
                            try:
                                int(t)
                                continue
                            except ValueError:
                                pass
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                d.update({t: val})
                                del copy_raw_data[key]
                    v = {nk: v}
                    if d:
                        v.update(d)
                    return _to_parse_key(apx, v)
            else:
                return v

        for k, v in raw_data.iteritems():
            if k in copy_raw_data:
                # to transform value from list to string
                v = v[0] if isinstance(v, list) and len(v) is 1 else v
                if k.find(self.separator) != -1:
                    d = _to_parse_key(k, v)
                    # set type result
                    if not len(result):
                        result = type(d)()
                    try:
                        result.extend(d)
                    except:
                        result.update(d)
        return result

    def value_from_datadict(self, data, files, name):
        data_copy = copy.deepcopy(data)
        result = self._to_pack_up(name, data_copy)
        return json.dumps(result)

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except (TypeError, KeyError):
            pass
        inputs = self._to_build(name, value or {})
        result = self._prepare_as_ul(inputs)
        if self.debug:
            # render json as well
            source_data = u'
Source data:
%s
' % str(value)
            result = '%s%s' % (result, source_data)
            print result
        return utils.safestring.mark_safe(result)<commit_msg>update<commit_after>from django import get_version, forms
from django.forms import Widget
from django import utils
from django.utils.encoding import force_text
import copy
from distutils.version import StrictVersion
try:
    import simplejson as json
except ImportError:
    import json
if StrictVersion(get_version()) < StrictVersion('1.9.0'):
    from django.forms.util import flatatt
else:
    from django.forms.utils import flatatt

class SplitJSONWidget(forms.Widget):

    def __init__(self, attrs=None, newline='
\n', sep='__', debug=False):
        self.newline = newline
        self.separator = sep
        self.debug = debug
        Widget.__init__(self, attrs)

    def _as_text_field(self, name, key, value, is_sub=False):
        attrs = self.build_attrs(self.attrs, type='text',
                                 name="%s%s%s" % (name, self.separator, key))
        attrs['value'] = force_text(value)
        attrs['id'] = attrs.get('name', None)
        return u""" %s:
        <input%s />""" % (attrs['id'], key, flatatt(attrs))

    def _to_build(self, name, json_obj):
        inputs = []
        if isinstance(json_obj, list):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in enumerate(json_obj):
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, dict):
            title = name.rpartition(self.separator)[2]
            _l = ['%s:%s' % (title, self.newline)]
            for key, value in json_obj.items():
                _l.append(self._to_build("%s%s%s" % (name, self.separator, key), value))
            inputs.extend([_l])
        elif isinstance(json_obj, (str, int, float)):
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, json_obj))
        elif json_obj is None:
            name, _, key = name.rpartition(self.separator)
            inputs.append(self._as_text_field(name, key, ''))
        return inputs

    def _prepare_as_ul(self, l):
        if l:
            result = ''
            for el in l:
                if isinstance(el, list) and len(l) == 1:
                    result += '%s' % self._prepare_as_ul(el)
                elif isinstance(el, list):
                    result += '
'
                    result += '%s' % self._prepare_as_ul(el)
                    result += '
'
                else:
                    result += '
%s
' % el
            return result
        return ''

    def _to_pack_up(self, root_node, raw_data):

        copy_raw_data = copy.deepcopy(raw_data)
        result = []

        def _to_parse_key(k, v):
            if k.find(self.separator) != -1:
                apx, _, nk = k.rpartition(self.separator)
                try:
                    # parse list
                    int(nk)
                    l = []
                    obj = {}
                    index = None
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            head, _, t = key.rpartition(self.separator)
                            _, _, index = head.rpartition(self.separator)
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                try:
                                    int(t)
                                    l.append(val)
                                except ValueError:
                                    if index in obj:
                                        obj[index].update({t: val})
                                    else:
                                        obj[index] = {t: val}
                                del copy_raw_data[key]
                        if obj:
                            for i in obj:
                                l.append(obj[i])
                    l.append(v)
                    return _to_parse_key(apx, l)
                except ValueError:
                    # parse dict
                    d = {}
                    if apx != root_node:
                        for key, val in copy_raw_data.items():
                            _, _, t = key.rpartition(self.separator)
                            try:
                                int(t)
                                continue
                            except ValueError:
                                pass
                            if key is k:
                                del copy_raw_data[key]
                            elif key.startswith(apx):
                                d.update({t: val})
                                del copy_raw_data[key]
                    v = {nk: v}
                    if d:
                        v.update(d)
                    return _to_parse_key(apx, v)
            else:
                return v

        for k, v in raw_data.iteritems():
            if k in copy_raw_data:
                # to transform value from list to string
                v = v[0] if isinstance(v, list) and len(v) is 1 else v
                if k.find(self.separator) != -1:
                    d = _to_parse_key(k, v)
                    # set type result
                    if not len(result):
                        result = type(d)()
                    try:
                        result.extend(d)
                    except:
                        result.update(d)
        return result

    def value_from_datadict(self, data, files, name):
        data_copy = copy.deepcopy(data)
        result = self._to_pack_up(name, data_copy)
        return json.dumps(result)

    def render(self, name, value, attrs=None):
        try:
            value = json.loads(value)
        except (TypeError, KeyError):
            pass
        inputs = self._to_build(name, value or {})
        result = self._prepare_as_ul(inputs)
        if self.debug:
            # render json as well
            source_data = u'
Source data:
%s
' % str(value)
            result = '%s%s' % (result, source_data)
        return utils.safestring.mark_safe(result)<|endoftext|>