<commit_before>import logging

from django.http import HttpResponse
from django.shortcuts import redirect
from django.utils.http import urlquote_plus
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.views.generic import FormView
from django.views.generic import TemplateView
from django.views.generic import View
from django.contrib import auth

from mama_cas.forms import LoginForm
from mama_cas.models import ServiceTicket
from mama_cas.utils import add_query_params
from mama_cas.mixins import NeverCacheMixin

LOG = logging.getLogger('mama_cas')

class LoginView(NeverCacheMixin, FormView):
    """
    (2.1 and 2.2) Credential requestor and acceptor.

    This URI operates in two modes: a credential requestor when a GET request
    is received, and a credential acceptor for POST requests.

    """
    template_name = 'mama_cas/login.html'
    form_class = LoginForm

    def get(self, *args, **kwargs):
        """
        As a credential requestor, /login takes up to three optional parameters:

        1. service - the identifier of the application the client is accessing.
           In most cases this will be a URL.
        2. renew - if set, a client must present credentials regardless of any
           existing single sign-on session. If set, its value should be "true".
        3. gateway - if set, the client will not be prompted for credentials. If
           set, its value should be "true".

        """
        service = self.request.GET.get('service')
        renew = self.request.GET.get('renew')
        gateway = self.request.GET.get('gateway')

        if renew:
            LOG.debug("Renew request received by credential requestor")
            auth.logout(self.request)
            login = add_query_params(reverse('cas_login'), { 'service': service })
            return redirect(login)
        elif gateway and service:
            LOG.debug("Gateway request received by credential requestor")
            if self.request.user.is_authenticated():
                st = ServiceTicket.objects.create_ticket(service=service,
                                                         user=self.request.user)
                service = add_query_params(service, { 'ticket': st.ticket })
            return redirect(service)
        elif self.request.user.is_authenticated():
            if service:
                LOG.debug("Service ticket request received by credential requestor")
                st = ServiceTicket.objects.create_ticket(service=service,
                                                         user=self.request.user)
                service = add_query_params(service, {'ticket': st.ticket })
                return redirect(service)
            else:
                messages.success(self.request, "You are logged in as %s" % self.request.user)
        return super(LoginView, self).get(*args, **kwargs)

    def form_valid(self, form):
        """
        As a credential acceptor, /login takes two required parameters:

        1. username - the username provided by the client
        2. password - the password provided by the client

        If authentication is successful, the user is logged in which creates
        the single sign-on session. If a service is provided, a corresponding
        ``ServiceTicket`` is created, and the user is redirected to the
        service URL. If no service is provided, the user is redirected back
        to the login page with a message indicating a successful login.

        If authentication fails, the login form is redisplayed with an appropriate
        error message displayed indicating the reason for failure.

        """
        auth.login(self.request, form.user)
        LOG.info("User logged in as '%s'" % self.request.user)
        service = form.cleaned_data.get('service')
        if service:
            st = ServiceTicket.objects.create_ticket(service=service,
                                                     user=self.request.user,
                                                     primary=True)
            service = add_query_params(service, { 'ticket': st.ticket })
            return redirect(service)
        return redirect(reverse('cas_login'))

    def get_initial(self):
        service = self.request.GET.get('service')
        if service:
            return { 'service': urlquote_plus(service) }

class LogoutView(NeverCacheMixin, TemplateView):
    """
    (2.3) End a client's single sign-on CAS session.

    When this URI is accessed, any current single sign-on session is
    ended, requiring a new single sign-on session to be established
    for future authentication attempts.

    If a URL is specified, it will be displayed on the page as a suggested
    link to follow.
    """
    template_name = 'mama_cas/login.html'

    def get(self, *args, **kwargs):
        if self.request.user.is_authenticated():
            auth.logout(self.request)
        messages.success(self.request, "You have been successfully logged out.")
        url = self.request.GET.get('url', None)
        if url:
            messages.success(self.request, "The application has provided this link to follow: " \
                "%s" % (url, url), extra_tags='safe')
        return redirect(reverse('cas_login'))

class ValidateView(NeverCacheMixin, View):
    """
    (2.4) Check the validity of a service ticket. [CAS 1.0]

    When both ``service`` and ``ticket`` are provided, this URI responds with
    a ``ServiceTicket`` validation success or failure. Whether or not the
    validation succeeds, the ``ServiceTicket`` is consumed, rendering it
    invalid for future authentication attempts.

    If ``renew`` is specified, validation will only succeed if the
    ``ServiceTicket`` was issued from the presentation of the user's primary
    credentials (i.e. not from an existing single sign-on session).
    """
    def get(self, *args, **kwargs):
        service = self.request.GET.get('service')
        ticket = self.request.GET.get('ticket')
        renew = self.request.GET.get('renew')

        LOG.debug("Service ticket validation request received for %s" % ticket)
        if service and ticket:
            st = ServiceTicket.objects.validate_ticket(ticket, service=service, renew=renew)
            if st:
                return self.validation_success(st.user.username)
        return self.validation_failure()

    def validation_success(self, username):
        LOG.debug("Service ticket validation request successful")
        response = HttpResponse(content="yes\n%s\n" % username)
        response.content_type = 'text/plain'
        return response

    def validation_failure(self):
        LOG.debug("Service ticket validation request failed")
        response = HttpResponse(content="no\n\n")
        response.content_type = 'text/plain'
        return response

def service_validate(request):
    """
    (2.5) Check the validity of a service ticket. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)

def proxy_validate(request):
    """
    (2.6) Check the validity of a service ticket, and additionally
    validate proxy tickets. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)

def proxy(request):
    """
    (2.7) Provide proxy tickets to services that have acquired proxy-
    granting tickets. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)
<commit_msg>Improve debug output from views<commit_after>import logging

from django.http import HttpResponse
from django.shortcuts import redirect
from django.utils.http import urlquote_plus
from django.core.urlresolvers import reverse
from django.contrib import messages
from django.views.generic import FormView
from django.views.generic import TemplateView
from django.views.generic import View
from django.contrib import auth

from mama_cas.forms import LoginForm
from mama_cas.models import ServiceTicket
from mama_cas.utils import add_query_params
from mama_cas.mixins import NeverCacheMixin

LOG = logging.getLogger('mama_cas')

class LoginView(NeverCacheMixin, FormView):
    """
    (2.1 and 2.2) Credential requestor and acceptor.

    This URI operates in two modes: a credential requestor when a GET request
    is received, and a credential acceptor for POST requests.

    """
    template_name = 'mama_cas/login.html'
    form_class = LoginForm

    def get(self, *args, **kwargs):
        """
        As a credential requestor, /login takes up to three optional parameters:

        1. service - the identifier of the application the client is accessing.
           In most cases this will be a URL.
        2. renew - if set, a client must present credentials regardless of any
           existing single sign-on session. If set, its value should be "true".
        3. gateway - if set, the client will not be prompted for credentials. If
           set, its value should be "true".

        """
        service = self.request.GET.get('service')
        renew = self.request.GET.get('renew')
        gateway = self.request.GET.get('gateway')

        if renew:
            LOG.debug("Renew request received by credential requestor")
            auth.logout(self.request)
            login = add_query_params(reverse('cas_login'), { 'service': service })
            return redirect(login)
        elif gateway and service:
            LOG.debug("Gateway request received by credential requestor")
            if self.request.user.is_authenticated():
                st = ServiceTicket.objects.create_ticket(service=service,
                                                         user=self.request.user)
                service = add_query_params(service, { 'ticket': st.ticket })
            return redirect(service)
        elif self.request.user.is_authenticated():
            if service:
                LOG.debug("Service ticket request received by credential requestor")
                st = ServiceTicket.objects.create_ticket(service=service,
                                                         user=self.request.user)
                service = add_query_params(service, {'ticket': st.ticket })
                return redirect(service)
            else:
                messages.success(self.request, "You are logged in as %s" % self.request.user)
        return super(LoginView, self).get(*args, **kwargs)

    def form_valid(self, form):
        """
        As a credential acceptor, /login takes two required parameters:

        1. username - the username provided by the client
        2. password - the password provided by the client

        If authentication is successful, the user is logged in which creates
        the single sign-on session. If a service is provided, a corresponding
        ``ServiceTicket`` is created, and the user is redirected to the
        service URL. If no service is provided, the user is redirected back
        to the login page with a message indicating a successful login.

        If authentication fails, the login form is redisplayed with an appropriate
        error message displayed indicating the reason for failure.

        """
        auth.login(self.request, form.user)
        LOG.info("User logged in as '%s'" % self.request.user)
        service = form.cleaned_data.get('service')
        if service:
            st = ServiceTicket.objects.create_ticket(service=service,
                                                     user=self.request.user,
                                                     primary=True)
            service = add_query_params(service, { 'ticket': st.ticket })
            return redirect(service)
        return redirect(reverse('cas_login'))

    def get_initial(self):
        service = self.request.GET.get('service')
        if service:
            return { 'service': urlquote_plus(service) }

class LogoutView(NeverCacheMixin, TemplateView):
    """
    (2.3) End a client's single sign-on CAS session.

    When this URI is accessed, any current single sign-on session is
    ended, requiring a new single sign-on session to be established
    for future authentication attempts.

    If a URL is specified, it will be displayed on the page as a suggested
    link to follow.
    """
    template_name = 'mama_cas/login.html'

    def get(self, *args, **kwargs):
        LOG.debug("Logout request received for user '%s'" % self.request.user)
        if self.request.user.is_authenticated():
            auth.logout(self.request)
        messages.success(self.request, "You have been successfully logged out.")
        url = self.request.GET.get('url', None)
        if url:
            messages.success(self.request, "The application has provided this link to follow: " \
                "%s" % (url, url), extra_tags='safe')
        return redirect(reverse('cas_login'))

class ValidateView(NeverCacheMixin, View):
    """
    (2.4) Check the validity of a service ticket. [CAS 1.0]

    When both ``service`` and ``ticket`` are provided, this URI responds with
    a ``ServiceTicket`` validation success or failure. Whether or not the
    validation succeeds, the ``ServiceTicket`` is consumed, rendering it
    invalid for future authentication attempts.

    If ``renew`` is specified, validation will only succeed if the
    ``ServiceTicket`` was issued from the presentation of the user's primary
    credentials (i.e. not from an existing single sign-on session).
    """
    def get(self, *args, **kwargs):
        service = self.request.GET.get('service')
        ticket = self.request.GET.get('ticket')
        renew = self.request.GET.get('renew')

        LOG.debug("Service ticket validation request received for %s" % ticket)
        if service and ticket:
            st = ServiceTicket.objects.validate_ticket(ticket, service=service, renew=renew)
            if st:
                return self.validation_success(st.user.username)
        return self.validation_failure()

    def validation_success(self, username):
        response = HttpResponse(content="yes\n%s\n" % username)
        response.content_type = 'text/plain'
        return response

    def validation_failure(self):
        response = HttpResponse(content="no\n\n")
        response.content_type = 'text/plain'
        return response

def service_validate(request):
    """
    (2.5) Check the validity of a service ticket. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)

def proxy_validate(request):
    """
    (2.6) Check the validity of a service ticket, and additionally
    validate proxy tickets. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)

def proxy(request):
    """
    (2.7) Provide proxy tickets to services that have acquired proxy-
    granting tickets. [CAS 2.0]
    """
    return HttpResponse(content='Not Implemented', content_type='text/plain', status=501)
<|endoftext|>


<commit_before># encoding: utf-8
'''
Created on Nov 26, 2015

@author: tal

Based in part on:
Learn math - https://github.com/fchollet/keras/blob/master/examples/addition_rnn.py

See https://medium.com/@majortal/deep-spelling-9ffef96a24f6#.2c9pu8nlm
"""

'''

from __future__ import print_function, division, unicode_literals

import os
from collections import Counter
import re
import numpy as np
from numpy.random import choice as random_choice, randint as random_randint, shuffle as random_shuffle, seed as random_seed, rand
from numpy import zeros as np_zeros # pylint:disable=no-name-in-module

from keras.models import Sequential
from keras.engine.training import slice_X
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, Dropout
from keras.layers import recurrent

random_seed(123) # Reproducibility

# Parameters for the model and dataset
NUMBER_OF_ITERATIONS = 20000
EPOCHS_PER_ITERATION = 5
RNN = recurrent.LSTM
INPUT_LAYERS = 2
OUTPUT_LAYERS = 2
AMOUNT_OF_DROPOUT = 0.3
BATCH_SIZE = 500
HIDDEN_SIZE = 700
INITIALIZATION = "he_normal" # : Gaussian initialization scaled by fan_in (He et al., 2014)
MAX_INPUT_LEN = 40
MIN_INPUT_LEN = 3
INVERTED = True
AMOUNT_OF_NOISE = 0.2 / MAX_INPUT_LEN
NUMBER_OF_CHARS = 100 # 75
CHARS = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ .")

# Some cleanup:
NORMALIZE_WHITESPACE_REGEX = re.compile(r'[^\S\n]+', re.UNICODE) # match all whitespace except newlines
RE_DASH_FILTER = re.compile(r'[\-\˗\֊\‐\‑\‒\–\—\⁻\₋\−\﹣\－]', re.UNICODE)
RE_APOSTROPHE_FILTER = re.compile(r''|[ʼ՚＇‘’‛❛❜ߴߵ`‵´ˊˋ{}{}{}{}{}{}{}{}{}]'.format(unichr(768), unichr(769), unichr(832),
                                                                                unichr(833), unichr(2387), unichr(5151),
                                                                                unichr(5152), unichr(65344), unichr(8242)),
                                  re.UNICODE)
RE_LEFT_PARENTH_FILTER = re.compile(r'[\(\[\{\⁽\₍\❨\❪\﹙\（]', re.UNICODE)
RE_RIGHT_PARENTH_FILTER = re.compile(r'[\)\]\}\⁾\₎\❩\❫\﹚\）]', re.UNICODE)
ALLOWED_CURRENCIES = """¥£₪$€฿₨"""
ALLOWED_PUNCTUATION = """-!?/;"'%&<>.()[]{}@#:,|=*"""
RE_BASIC_CLEANER = re.compile(r'[^\w\s{}{}]'.format(re.escape(ALLOWED_CURRENCIES), re.escape(ALLOWED_PUNCTUATION)), re.UNICODE)

# pylint:disable=invalid-name

def add_noise_to_string(a_string, amount_of_noise):
    """Add some artificial spelling mistakes to the string"""
    if rand() < amount_of_noise * len(a_string):
        # Replace a character with a random character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + random_choice(CHARS[:-1]) + a_string[random_char_position + 1:]
    if rand() < amount_of_noise * len(a_string):
        # Delete a character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + a_string[random_char_position + 1:]
    if len(a_string) < MAX_INPUT_LEN and rand() < amount_of_noise * len(a_string):
        # Add a random character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + random_choice(CHARS[:-1]) + a_string[random_char_position:]
    if rand() < amount_of_noise * len(a_string):
        # Transpose 2 characters
        random_char_position = random_randint(len(a_string) - 1)
        a_string = (a_string[:random_char_position] + a_string[random_char_position + 1] + a_string[random_char_position] +
                    a_string[random_char_position + 2:])
    return a_string

def vectorize(questions, answers, chars=None):
    """Vectorize the questions and expected answers"""
    print('Vectorization...')
    chars = chars or CHARS
    x_maxlen = max(len(question) for question in questions)
    y_maxlen = max(len(answer) for answer in answers)
#     print (len(questions), x_maxlen, len(chars))
    len_of_questions = len(questions)
    ctable = CharacterTable(chars)
    print("X = np_zeros")
    X = np_zeros((len_of_questions, x_maxlen, len(chars)), dtype=np.bool)
    print("for i, sentence in enumerate(questions):")
    for i in xrange(len(questions)):
        sentence = questions.pop()
        for j, c in enumerate(sentence):
            X[i, j, ctable.char_indices[c]] = 1
    print("y = np_zeros")
    y = np_zeros((len_of_questions, y_maxlen, len(chars)), dtype=np.bool)
    print("for i, sentence in enumerate(answers):")
    for i in xrange(len(answers)):
        sentence = answers.pop()
        for j, c in enumerate(sentence):
            y[i, j, ctable.char_indices[c]] = 1

    # Explicitly set apart 10% for validation data that we never train over
    split_at = int(len(X) - len(X) / 10)
    (X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
    (y_train, y_val) = (y[:split_at], y[split_at:])

    print(X_train.shape)
    print(y_train.shape)

    return X_train, X_val, y_train, y_val, y_maxlen, ctable

def generate_model(output_len, chars=None):
    """Generate the model"""
    print('Build model...')
    chars = chars or CHARS
    model = Sequential()
    # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
    # note: in a situation where your input sequences have a variable length,
    # use input_shape=(None, nb_feature).
    for layer_number in range(INPUT_LAYERS):
        model.add(recurrent.LSTM(HIDDEN_SIZE, input_shape=(None, len(chars)), init=INITIALIZATION,
                                 return_sequences=layer_number + 1 < INPUT_LAYERS))
        model.add(Dropout(AMOUNT_OF_DROPOUT))
    # For the decoder's input, we repeat the encoded input for each time step
    model.add(RepeatVector(output_len))
    # The decoder RNN could be multiple layers stacked or a single layer
    for _ in range(OUTPUT_LAYERS):
        model.add(recurrent.LSTM(HIDDEN_SIZE, return_sequences=True, init=INITIALIZATION))
        model.add(Dropout(AMOUNT_OF_DROPOUT))

    # For each of step of the output sequence, decide which character should be chosen
    model.add(TimeDistributed(Dense(len(chars), init=INITIALIZATION)))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

class Colors(object):
    """For nicer printouts"""
    ok = '\033[92m'
    fail = '\033[91m'
    close = '\033[0m'

class CharacterTable(object):
    """
    Given a set of characters:
    + Encode them to a one hot integer representation
    + Decode the one hot integer representation to their character output
    + Decode a vector of probabilities to their character output
    """
    def __init__(self, chars):
        self.chars = sorted(set(chars))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))

    def encode(self, C, maxlen):
        """Encode as one-hot"""
        X = np_zeros((maxlen, len(self.chars)), dtype=np.bool) # pylint:disable=no-member
        for i, c in enumerate(C):
            X[i, self.char_indices[c]] = 1
        return X

    def decode(self, X, calc_argmax=True):
        """Decode from one-hot"""
        if calc_argmax:
            X = X.argmax(axis=-1)
        return ''.join(self.indices_char[x] for x in X)

def iterate_training(model, X_train, y_train, X_val, y_val, ctable):
    """Iterative Training"""
    # Train the model each generation and show predictions against the validation dataset
    for iteration in range(1, NUMBER_OF_ITERATIONS):
        print()
        print('-' * 50)
        print('Iteration', iteration)
        model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=EPOCHS_PER_ITERATION, validation_data=(X_val, y_val))
        # Select 10 samples from the validation set at random so we can visualize errors
        for _ in range(10):
            ind = random_randint(0, len(X_val))
            rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])] # pylint:disable=no-member
            preds = model.predict_classes(rowX, verbose=0)
            q = ctable.decode(rowX[0])
            correct = ctable.decode(rowy[0])
            guess = ctable.decode(preds[0], calc_argmax=False)
            if INVERTED:
                print('Q', q[::-1]) # inverted back!
            else:
                print('Q', q)
            print('A', correct)
            print(Colors.ok + '☑' + Colors.close if correct == guess else Colors.fail + '☒' + Colors.close, guess)
            print('---')

def clean_text(text):
    """Clean the text - remove unwanted chars, fold punctuation etc."""
    from time import time
    start_time = time()
    result = NORMALIZE_WHITESPACE_REGEX.sub(' ', text.strip())
    print("NORMALIZE_WHITESPACE_REGEX", time() - start_time)
    result = RE_DASH_FILTER.sub('-', result)
    print("RE_DASH_FILTER", time() - start_time)
    result = RE_APOSTROPHE_FILTER.sub("'", result)
    print("RE_APOSTROPHE_FILTER", time() - start_time)
    result = RE_LEFT_PARENTH_FILTER.sub("(", result)
    print("RE_LEFT_PARENTH_FILTER")
    result = RE_RIGHT_PARENTH_FILTER.sub(")", result)
    print("RE_RIGHT_PARENTH_FILTER")
    result = RE_BASIC_CLEANER.sub('', result)
    print("RE_BASIC_CLEANER")
    return result

def read_news(_most_popular_chars_are=""):
    """Read the news corpus"""
    print("reading news")
    news = open(os.path.expanduser("~/downloads/training-monolingual/news.2011.en.shuffled")).read().decode('utf-8')
    print("read news")
    news = clean_text(news)
    print("cleaned text")
    counter = Counter(news)
    most_popular_chars = {key for key, _value in counter.most_common(NUMBER_OF_CHARS)}
    print("".join(sorted(most_popular_chars)))
    lines = [line.strip() for line in news.split('\n')]
    print("Read {} lines of input corpus".format(len(lines)))
    lines = [line for line in lines if line and not bool(set(line) - most_popular_chars)]
    print("Left with {} lines of input corpus".format(len(lines)))
    return lines

def generate_news_data(corpus):
    """Generate some news data"""
    print ("Generating Data")
    questions, answers, seen_answers = [], [], set()
    while corpus:
        line = corpus.pop()
        while len(line) > MIN_INPUT_LEN:
            if len(line) <= MAX_INPUT_LEN:
                answer = line
                line = ""
            else:
                space_location = line.rfind(" ", MIN_INPUT_LEN, MAX_INPUT_LEN - 1)
                if space_location > -1:
                    answer = line[:space_location]
                    line = line[len(answer) + 1:]
                else:
                    space_location = line.rfind(" ") # no limits this time
                    if space_location == -1:
                        break # we are done with this line
                    else:
                        line = line[space_location + 1:]
                        continue
            if answer and answer in seen_answers:
                continue
            seen_answers.add(answer)
            answers.append(answer)
        if random_randint(100000) == 8: # Show some progress
            print('.', end="")
    print('suffle', end=" ")
    random_shuffle(answers)
    print("Done")
    for answer_index, answer in enumerate(answers):
        question = add_noise_to_string(answer, AMOUNT_OF_NOISE)
        question += '.' * (MAX_INPUT_LEN - len(question))
        answer += "." * (MAX_INPUT_LEN - len(answer))
        answers[answer_index] = answer
        assert len(answer) == MAX_INPUT_LEN
        if random_randint(100000) == 8: # Show some progress
            print (len(seen_answers))
            print ("answer:   '{}'".format(answer))
            print ("question: '{}'".format(question))
            print ()
        question = question[::-1] if INVERTED else question
        questions.append(question)

    return questions, answers

def main_news():
    """Main"""
    questions, answers = generate_news_data(read_news())
    chars_answer = set.union(*(set(answer) for answer in answers))
    chars_question = set.union(*(set(question) for question in questions))
    chars = list(set.union(chars_answer, chars_question))
    X_train, X_val, y_train, y_val, y_maxlen, ctable = vectorize(questions, answers, chars)
    print ("y_maxlen, chars", y_maxlen, "".join(chars))
    model = generate_model(y_maxlen, chars)
    iterate_training(model, X_train, y_train, X_val, y_val, ctable)

if __name__ == '__main__':
    main_news()
<commit_msg>Added pre-processing steps that can be run separately.<commit_after># encoding: utf-8
'''
Created on Nov 26, 2015

@author: tal

Based in part on:
Learn math - https://github.com/fchollet/keras/blob/master/examples/addition_rnn.py

See https://medium.com/@majortal/deep-spelling-9ffef96a24f6#.2c9pu8nlm
'''

from __future__ import print_function, division, unicode_literals

import os
from collections import Counter
import re
import json
import numpy as np
from numpy.random import choice as random_choice, randint as random_randint, shuffle as random_shuffle, seed as random_seed, rand
from numpy import zeros as np_zeros # pylint:disable=no-name-in-module

from keras.models import Sequential
from keras.engine.training import slice_X
from keras.layers import Activation, TimeDistributed, Dense, RepeatVector, Dropout
from keras.layers import recurrent

random_seed(123) # Reproducibility

# Parameters for the model and dataset
NUMBER_OF_ITERATIONS = 20000
EPOCHS_PER_ITERATION = 5
RNN = recurrent.LSTM
INPUT_LAYERS = 2
OUTPUT_LAYERS = 2
AMOUNT_OF_DROPOUT = 0.3
BATCH_SIZE = 500
HIDDEN_SIZE = 700
INITIALIZATION = "he_normal" # : Gaussian initialization scaled by fan_in (He et al., 2014)
MAX_INPUT_LEN = 40
MIN_INPUT_LEN = 3
INVERTED = True
AMOUNT_OF_NOISE = 0.2 / MAX_INPUT_LEN
NUMBER_OF_CHARS = 100 # 75
CHARS = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ .")

DATA_FILES_PATH = "~/Downloads"
NEWS_FILE_NAME = "news.2011.en.shuffled"
CLEAN_NEWS_FILE_NAME = "clean_news.2011.en.shuffled"
FILTERED_NEWS_FILE_NAME = "filtered_news.2011.en.shuffled"
MOST_POPULAR_CHARS_FILE_NAME = "most_popular_chars.json"

# Some cleanup:
NORMALIZE_WHITESPACE_REGEX = re.compile(r'[^\S\n]+', re.UNICODE) # match all whitespace except newlines
RE_DASH_FILTER = re.compile(r'[\-\˗\֊\‐\‑\‒\–\—\⁻\₋\−\﹣\－]', re.UNICODE)
RE_APOSTROPHE_FILTER = re.compile(r''|[ʼ՚＇‘’‛❛❜ߴߵ`‵´ˊˋ{}{}{}{}{}{}{}{}{}]'.format(unichr(768), unichr(769), unichr(832),
                                                                                unichr(833), unichr(2387), unichr(5151),
                                                                                unichr(5152), unichr(65344), unichr(8242)),
                                  re.UNICODE)
RE_LEFT_PARENTH_FILTER = re.compile(r'[\(\[\{\⁽\₍\❨\❪\﹙\（]', re.UNICODE)
RE_RIGHT_PARENTH_FILTER = re.compile(r'[\)\]\}\⁾\₎\❩\❫\﹚\）]', re.UNICODE)
ALLOWED_CURRENCIES = """¥£₪$€฿₨"""
ALLOWED_PUNCTUATION = """-!?/;"'%&<>.()[]{}@#:,|=*"""
RE_BASIC_CLEANER = re.compile(r'[^\w\s{}{}]'.format(re.escape(ALLOWED_CURRENCIES), re.escape(ALLOWED_PUNCTUATION)), re.UNICODE)

# pylint:disable=invalid-name

def add_noise_to_string(a_string, amount_of_noise):
    """Add some artificial spelling mistakes to the string"""
    if rand() < amount_of_noise * len(a_string):
        # Replace a character with a random character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + random_choice(CHARS[:-1]) + a_string[random_char_position + 1:]
    if rand() < amount_of_noise * len(a_string):
        # Delete a character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + a_string[random_char_position + 1:]
    if len(a_string) < MAX_INPUT_LEN and rand() < amount_of_noise * len(a_string):
        # Add a random character
        random_char_position = random_randint(len(a_string))
        a_string = a_string[:random_char_position] + random_choice(CHARS[:-1]) + a_string[random_char_position:]
    if rand() < amount_of_noise * len(a_string):
        # Transpose 2 characters
        random_char_position = random_randint(len(a_string) - 1)
        a_string = (a_string[:random_char_position] + a_string[random_char_position + 1] + a_string[random_char_position] +
                    a_string[random_char_position + 2:])
    return a_string

def vectorize(questions, answers, chars=None):
    """Vectorize the questions and expected answers"""
    print('Vectorization...')
    chars = chars or CHARS
    x_maxlen = max(len(question) for question in questions)
    y_maxlen = max(len(answer) for answer in answers)
#     print (len(questions), x_maxlen, len(chars))
    len_of_questions = len(questions)
    ctable = CharacterTable(chars)
    print("X = np_zeros")
    X = np_zeros((len_of_questions, x_maxlen, len(chars)), dtype=np.bool)
    print("for i, sentence in enumerate(questions):")
    for i in xrange(len(questions)):
        sentence = questions.pop()
        for j, c in enumerate(sentence):
            X[i, j, ctable.char_indices[c]] = 1
    print("y = np_zeros")
    y = np_zeros((len_of_questions, y_maxlen, len(chars)), dtype=np.bool)
    print("for i, sentence in enumerate(answers):")
    for i in xrange(len(answers)):
        sentence = answers.pop()
        for j, c in enumerate(sentence):
            y[i, j, ctable.char_indices[c]] = 1

    # Explicitly set apart 10% for validation data that we never train over
    split_at = int(len(X) - len(X) / 10)
    (X_train, X_val) = (slice_X(X, 0, split_at), slice_X(X, split_at))
    (y_train, y_val) = (y[:split_at], y[split_at:])

    print(X_train.shape)
    print(y_train.shape)

    return X_train, X_val, y_train, y_val, y_maxlen, ctable

def generate_model(output_len, chars=None):
    """Generate the model"""
    print('Build model...')
    chars = chars or CHARS
    model = Sequential()
    # "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE
    # note: in a situation where your input sequences have a variable length,
    # use input_shape=(None, nb_feature).
    for layer_number in range(INPUT_LAYERS):
        model.add(recurrent.LSTM(HIDDEN_SIZE, input_shape=(None, len(chars)), init=INITIALIZATION,
                                 return_sequences=layer_number + 1 < INPUT_LAYERS))
        model.add(Dropout(AMOUNT_OF_DROPOUT))
    # For the decoder's input, we repeat the encoded input for each time step
    model.add(RepeatVector(output_len))
    # The decoder RNN could be multiple layers stacked or a single layer
    for _ in range(OUTPUT_LAYERS):
        model.add(recurrent.LSTM(HIDDEN_SIZE, return_sequences=True, init=INITIALIZATION))
        model.add(Dropout(AMOUNT_OF_DROPOUT))

    # For each of step of the output sequence, decide which character should be chosen
    model.add(TimeDistributed(Dense(len(chars), init=INITIALIZATION)))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

class Colors(object):
    """For nicer printouts"""
    ok = '\033[92m'
    fail = '\033[91m'
    close = '\033[0m'

class CharacterTable(object):
    """
    Given a set of characters:
    + Encode them to a one hot integer representation
    + Decode the one hot integer representation to their character output
    + Decode a vector of probabilities to their character output
    """
    def __init__(self, chars):
        self.chars = sorted(set(chars))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))

    def encode(self, C, maxlen):
        """Encode as one-hot"""
        X = np_zeros((maxlen, len(self.chars)), dtype=np.bool) # pylint:disable=no-member
        for i, c in enumerate(C):
            X[i, self.char_indices[c]] = 1
        return X

    def decode(self, X, calc_argmax=True):
        """Decode from one-hot"""
        if calc_argmax:
            X = X.argmax(axis=-1)
        return ''.join(self.indices_char[x] for x in X)

def iterate_training(model, X_train, y_train, X_val, y_val, ctable):
    """Iterative Training"""
    # Train the model each generation and show predictions against the validation dataset
    for iteration in range(1, NUMBER_OF_ITERATIONS):
        print()
        print('-' * 50)
        print('Iteration', iteration)
        model.fit(X_train, y_train, batch_size=BATCH_SIZE, nb_epoch=EPOCHS_PER_ITERATION, validation_data=(X_val, y_val))
        # Select 10 samples from the validation set at random so we can visualize errors
        for _ in range(10):
            ind = random_randint(0, len(X_val))
            rowX, rowy = X_val[np.array([ind])], y_val[np.array([ind])] # pylint:disable=no-member
            preds = model.predict_classes(rowX, verbose=0)
            q = ctable.decode(rowX[0])
            correct = ctable.decode(rowy[0])
            guess = ctable.decode(preds[0], calc_argmax=False)
            if INVERTED:
                print('Q', q[::-1]) # inverted back!
            else:
                print('Q', q)
            print('A', correct)
            print(Colors.ok + '☑' + Colors.close if correct == guess else Colors.fail + '☒' + Colors.close, guess)
            print('---')

def clean_text(text):
    """Clean the text - remove unwanted chars, fold punctuation etc."""
    from time import time
    start_time = time()
    result = NORMALIZE_WHITESPACE_REGEX.sub(' ', text.strip())
    print("NORMALIZE_WHITESPACE_REGEX", time() - start_time)
    result = RE_DASH_FILTER.sub('-', result)
    print("RE_DASH_FILTER", time() - start_time)
    result = RE_APOSTROPHE_FILTER.sub("'", result)
    print("RE_APOSTROPHE_FILTER", time() - start_time)
    result = RE_LEFT_PARENTH_FILTER.sub("(", result)
    print("RE_LEFT_PARENTH_FILTER")
    result = RE_RIGHT_PARENTH_FILTER.sub(")", result)
    print("RE_RIGHT_PARENTH_FILTER")
    result = RE_BASIC_CLEANER.sub('', result)
    print("RE_BASIC_CLEANER")
    return result

def preprocesses_data1():
    """Pre-process the data - step 1"""
    print("Reading data:")
    news = open(os.path.expanduser(os.path.join(DATA_FILES_PATH, NEWS_FILE_NAME))).read().decode('utf-8')
    print("Read the data\nCleaning data:")
    news = clean_text(news)
    print("Cleaned the data\nWriting to file:")
    with open(os.path.expanduser(os.path.join(DATA_FILES_PATH, CLEAN_NEWS_FILE_NAME)), "wb") as clean_news:
        clean_news.write(news.encode("utf-8"))
    print("Written to file")

def preprocesses_data2():
    """Pre-process the data - step 2"""
    print("Reading data:")
    data = open(os.path.expanduser(os.path.join(DATA_FILES_PATH, CLEAN_NEWS_FILE_NAME))).read().decode('utf-8')
    print("Read.\nCounting characters:")
    counter = Counter(data.replace("\n", ""))
    print("Done.\nWriting to file:")
    with open(os.path.expanduser(os.path.join(DATA_FILES_PATH, MOST_POPULAR_CHARS_FILE_NAME)), 'wb') as output_file:
        output_file.write(json.dumps(counter))
    print("Done")

def preprocesses_data3():
    """Pre-process the data - step 3"""
    chars = json.loads(open(os.path.expanduser(os.path.join(DATA_FILES_PATH, MOST_POPULAR_CHARS_FILE_NAME))).read())
    counter = Counter(chars)
    most_popular_chars = {key for key, _value in counter.most_common(NUMBER_OF_CHARS)}
    print("The top {} chars are:".format(NUMBER_OF_CHARS))
    print("".join(sorted(most_popular_chars)))
    print("Reading data:")
    data = open(os.path.expanduser(os.path.join(DATA_FILES_PATH, CLEAN_NEWS_FILE_NAME))).read().decode('utf-8')
    print("Read.\nFiltering:")
    lines = [line.strip() for line in data.split('\n')]
    print("Read {} lines of input corpus".format(len(lines)))
    lines = [line for line in lines if line and not bool(set(line) - most_popular_chars)]
    print("Left with {} lines of input corpus".format(len(lines)))
    with open(os.path.expanduser(os.path.join(DATA_FILES_PATH, FILTERED_NEWS_FILE_NAME)), "wb") as output_file:
        output_file.write("\n".join(lines).encode('utf-8'))

def read_news():
    """Read the news corpus"""
    print("reading news")
    lines = open(os.path.expanduser(os.path.join(DATA_FILES_PATH, FILTERED_NEWS_FILE_NAME))).read().decode('utf-8').split("\n")
    print("read news")
    return lines

def generate_news_data(corpus):
    """Generate some news data"""
    print ("Generating Data")
    questions, answers, seen_answers = [], [], set()
    while corpus:
        line = corpus.pop()
        while len(line) > MIN_INPUT_LEN:
            if len(line) <= MAX_INPUT_LEN:
                answer = line
                line = ""
            else:
                space_location = line.rfind(" ", MIN_INPUT_LEN, MAX_INPUT_LEN - 1)
                if space_location > -1:
                    answer = line[:space_location]
                    line = line[len(answer) + 1:]
                else:
                    space_location = line.rfind(" ") # no limits this time
                    if space_location == -1:
                        break # we are done with this line
                    else:
                        line = line[space_location + 1:]
                        continue
            if answer and answer in seen_answers:
                continue
            seen_answers.add(answer)
            answers.append(answer)
        if random_randint(100000) == 8: # Show some progress
            print('.', end="")
    print('shuffle', end=" ")
    random_shuffle(answers)
    print("Done")
    for answer_index, answer in enumerate(answers):
        question = add_noise_to_string(answer, AMOUNT_OF_NOISE)
        question += '.' * (MAX_INPUT_LEN - len(question))
        answer += "." * (MAX_INPUT_LEN - len(answer))
        answers[answer_index] = answer
        assert len(answer) == MAX_INPUT_LEN
        if random_randint(100000) == 8: # Show some progress
            print (len(seen_answers))
            print ("answer:   '{}'".format(answer))
            print ("question: '{}'".format(question))
            print ()
        question = question[::-1] if INVERTED else question
        questions.append(question)

    return questions, answers

def main_news():
    """Main"""
    questions, answers = generate_news_data(read_news())
    chars_answer = set.union(*(set(answer) for answer in answers))
    chars_question = set.union(*(set(question) for question in questions))
    chars = list(set.union(chars_answer, chars_question))
    X_train, X_val, y_train, y_val, y_maxlen, ctable = vectorize(questions, answers, chars)
    print ("y_maxlen, chars", y_maxlen, "".join(chars))
    model = generate_model(y_maxlen, chars)
    iterate_training(model, X_train, y_train, X_val, y_val, ctable)

if __name__ == '__main__':
    main_news()
<|endoftext|>


<commit_before>import json
import os

import git

from common import file, console

current_data_version = 1
if os.path.exists("./upgrade/current_version.json"):
    current_data_version = json.loads(file.read_file("./upgrade/current_version.json"))["current_data_version"]
repo = git.Repo("./")
remote = repo.remote()
def upgrade_check():
    info = remote.fetch("master")
    if info[0] == "FETCH_HEAD":
        return True
    return False
def upgrade_pull():
    if repo.is_dirty():
        console.log("Error","The current warehouse is modified and can not be upgraded automatically. Please re-store the warehouse and try again.")
        exit(1)
    remote.pull()
    if not repo.is_dirty() and os.path.exists("./upgrade/upgrade_from_{}.py".format(current_data_version)):
        eval(file.read_file("./upgrade/upgrade_from_{}.py".format(current_data_version)))
    console.log("Success", "Upgrade Successful!")
    console.log("Info", "Please restart your service process to ensure that the program is up and running.")
    exit(0)
<commit_msg>修改判断方法<commit_after>import json
import os

import git

from common import file, console

current_data_version = 1
if os.path.exists("./upgrade/current_version.json"):
    current_data_version = json.loads(file.read_file("./upgrade/current_version.json"))["current_data_version"]
repo = git.Repo("./")
remote = repo.remote()
def upgrade_check():
    if remote.fetch("master")[0] == "FETCH_HEAD":
        return True
    return False
def upgrade_pull():
    if repo.is_dirty():
        console.log("Error","The current warehouse is modified and can not be upgraded automatically. Please re-store the warehouse and try again.")
        exit(1)
    remote.pull()
    if not repo.is_dirty() and os.path.exists("./upgrade/upgrade_from_{}.py".format(current_data_version)):
        eval(file.read_file("./upgrade/upgrade_from_{}.py".format(current_data_version)))
    console.log("Success", "Upgrade Successful!")
    console.log("Info", "Please restart your service process to ensure that the program is up and running.")
    exit(0)
<|endoftext|>


<commit_before>import sublime, sublime_plugin

class SumCommand(sublime_plugin.TextCommand):
  def run(self, edit):
    new_view = self.view.window().new_file()
    new_view.set_name('Sum')
    new_view.insert(edit, 0, '42')
<commit_msg>Don't warn about new file changes when closing<commit_after>import sublime, sublime_plugin

class SumCommand(sublime_plugin.TextCommand):
  def run(self, edit):
    new_view = self.view.window().new_file()
    new_view.set_name('Sum')
    new_view.insert(edit, 0, '42')
    new_view.set_scratch(True)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# - coding: utf-8 -
# Copyright (C) 2010 Toms Bauģis <toms.baugis at gmail.com>

"""Base template"""

import gtk
from lib import graphics
import math

class Scene(graphics.Scene):
    def __init__(self):
        graphics.Scene.__init__(self)
        self.connect("on-finish-frame", self.on_enter_frame)
        self.start_angle = 0
        #self.framerate = 30

    def on_enter_frame(self, scene, context):
        self.start_angle += 0.006

        g = graphics.Graphics(context)

        g.fill_area(0, 0, self.width, self.height, "#f00")
        g.set_line_style(width = 0.5)
        g.set_color("#fff331")

        x, y = self.width / 2, self.height / 2

        x = x + math.sin(self.start_angle * 0.3) * self.width / 4

        center_distance = math.cos(self.start_angle) * self.width / 8

        angle = self.start_angle
        step = math.pi * 2 / 64

        distance = max(self.width, self.height)

        while angle < self.start_angle + math.pi * 2:
            g.move_to(x + math.cos(angle) * center_distance, y + math.sin(angle) * center_distance)
            g.line_to(x + math.cos(angle) * distance, y + math.sin(angle) * distance)
            g.line_to(x + math.cos(angle+step) * distance, y + math.sin(angle+step) * distance)
            g.line_to(x + math.cos(angle) * center_distance,y + math.sin(angle) * center_distance)

            angle += step * 2

        g.fill()

        self.redraw()

class BasicWindow:
    def __init__(self):
        window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        window.set_size_request(600, 500)
        window.connect("delete_event", lambda *args: gtk.main_quit())
        window.add(Scene())
        window.show_all()

example = BasicWindow()
gtk.main()
<commit_msg>went for white<commit_after>#!/usr/bin/env python
# - coding: utf-8 -
# Copyright (C) 2010 Toms Bauģis <toms.baugis at gmail.com>

"""Base template"""

import gtk
from lib import graphics
import math

class Scene(graphics.Scene):
    def __init__(self):
        graphics.Scene.__init__(self)
        self.connect("on-finish-frame", self.on_enter_frame)
        self.start_angle = 0
        self.framerate = 120

    def on_enter_frame(self, scene, context):
        self.start_angle += 0.01 * 60 / self.framerate # good way to keep the speed constant when overriding frame rate

        g = graphics.Graphics(context)

        g.fill_area(0, 0, self.width, self.height, "#f00")
        g.set_line_style(width = 0.5)
        g.set_color("#fff")

        x, y = self.width / 2, self.height / 2

        x = x + math.sin(self.start_angle * 0.3) * self.width / 4

        center_distance = math.cos(self.start_angle) * self.width / 8

        angle = self.start_angle
        step = math.pi * 2 / 64

        distance = max(self.width, self.height)

        while angle < self.start_angle + math.pi * 2:
            g.move_to(x + math.cos(angle) * center_distance, y + math.sin(angle) * center_distance)
            g.line_to(x + math.cos(angle) * distance, y + math.sin(angle) * distance)
            g.line_to(x + math.cos(angle+step) * distance, y + math.sin(angle+step) * distance)
            g.line_to(x + math.cos(angle) * center_distance,y + math.sin(angle) * center_distance)

            angle += step * 2

        g.fill()
        self.redraw()

class BasicWindow:
    def __init__(self):
        window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        window.set_size_request(600, 500)
        window.connect("delete_event", lambda *args: gtk.main_quit())
        window.add(Scene())
        window.show_all()

example = BasicWindow()
gtk.main()
<|endoftext|>


<commit_before># Licensed to Modin Development Team under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information regarding
# copyright ownership.  The Modin Development Team licenses this file to you under the
# Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

from .base import BaseQueryCompiler
from .pandas import PandasQueryCompiler
from .pyarrow import PyarrowQueryCompiler

__all__ = ["BaseQueryCompiler", "PandasQueryCompiler", "PyarrowQueryCompiler"]
<commit_msg>FIX-#2090: Do not always require pyarrow (#2126)<commit_after># Licensed to Modin Development Team under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information regarding
# copyright ownership.  The Modin Development Team licenses this file to you under the
# Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

from .base import BaseQueryCompiler
from .pandas import PandasQueryCompiler

__all__ = ["BaseQueryCompiler", "PandasQueryCompiler"]
try:
    from .pyarrow import PyarrowQueryCompiler  # noqa: F401
except ImportError:
    pass
else:
    __all__.append("PyarrowQueryCompiler")
<|endoftext|>


<commit_before>from __future__ import print_function, absolute_import, division

from six import itervalues
from lxml import etree as ET
from sphinx.ext.autodoc import Documenter, AutoDirective, members_option, ALL
from sphinx.errors import ExtensionError

from . import get_doxygen_root
from .xmlutils import format_xml_paragraph

class DoxygenDocumenter(Documenter):
    # Variables to store the names of the object being documented. modname and fullname are redundant,
    # and objpath is always the empty list. This is inelegant, but we need to work with the superclass.

    fullname = None  # example: "OpenMM::NonbondedForce" or "OpenMM::NonbondedForce::methodName""
    modname = None   # example: "OpenMM::NonbondedForce" or "OpenMM::NonbondedForce::methodName""
    objname = None   # example: "NonbondedForce"  or "methodName"
    objpath = []     # always the empty list
    object = None    # the xml node for the object
    titles_allowed = True

    option_spec = {
        'members': members_option,
    }

    def __init__(self, directive, name, indent=u'', id=None, brief=False, parent=None):
        super().__init__(directive, name, indent)

        self.parent = parent
        if id is not None:
            self.parse_id(id)
        self.brief = brief

    def parse_id(self, id):
        return False

    def parse_name(self):
        """Determine what module to import and what attribute to document.
        Returns True and sets *self.modname*, *self.objname*, *self.fullname*,
        if parsing and resolving was successful.
        """
        # To view the context and order in which all of these methods get called,
        # See, Documenter.generate(). That's the main "entry point" that first
        # calls parse_name(), follwed by import_object(), format_signature(),
        # add_directive_header(), and then add_content() (which calls get_doc())

        # methods in the superclass sometimes use '.' to join namespace/class
        # names with method names, and we don't want that.
        self.name = self.name.replace('.', '::')
        self.fullname = self.name
        self.modname = self.fullname
        self.objpath = []

        if '::' in self.name:
            parts = self.name.split('::')
            self.objname = parts[-1]
        else:
            self.objname = self.name

        return True

    def document_members(self, all_members=False):
        """Generate reST for member documentation.
        If *all_members* is True, do all members, else those given by
        *self.options.members*.
        """
        want_all = all_members or self.options.inherited_members or \
            self.options.members is ALL
        # find out which members are documentable
        members_check_module, members = self.get_object_members(want_all)

        # remove members given by exclude-members
        if self.options.exclude_members:
            members = [(membername, member) for (membername, member) in members
                       if membername not in self.options.exclude_members]

        # document non-skipped members
        memberdocumenters = []
        for (mname, member, isattr) in self.filter_members(members, want_all):
            classes = [cls for cls in itervalues(AutoDirective._registry)
                       if cls.can_document_member(member, mname, isattr, self)]
            if not classes:
                # don't know how to document this member
                continue

            # prefer the documenter with the highest priority
            classes.sort(key=lambda cls: cls.priority)

            documenter = classes[-1](self.directive, mname, indent=self.indent,
                                     id=member.get('id'), brief=self.brief,
                                     parent=self.object)
            memberdocumenters.append((documenter, isattr))

        for documenter, isattr in memberdocumenters:
            documenter.generate(
                all_members=True, real_modname=self.real_modname,
                check_module=members_check_module and not isattr)

        # reset current objects
        self.env.temp_data['autodoc:module'] = None
        self.env.temp_data['autodoc:class'] = None

class DoxygenClassDocumenter(DoxygenDocumenter):
    objtype = 'doxyclass'
    directivetype = 'class'
    domain = 'cpp'
    priority = 100

    option_spec = {
        'members': members_option,
    }

    @classmethod
    def can_document_member(cls, member, membername, isattr, parent):
        # this method is only called from Documenter.document_members
        # when a higher level documenter (module or namespace) is trying
        # to choose the appropriate documenter for each of its lower-level
        # members. Currently not implemented since we don't have a higher-level
        # doumenter like a DoxygenNamespaceDocumenter.
        return False

    def import_object(self):
        """Import the object and set it as *self.object*.  In the call sequence, this
        is executed right after parse_name(), so it can use *self.fullname*, *self.objname*,
        and *self.modname*.

        Returns True if successful, False if an error occurred.
        """
        xpath_query = './compounddef/compoundname[text()="%s"]/..' % self.fullname
        match = get_doxygen_root().xpath(xpath_query)
        if len(match) != 1:
            raise ExtensionError('[autodoc_doxygen] could not find class (fullname="%s"). I tried'
                                 'the following xpath: "%s"' % (self.fullname, xpath_query))

        self.object = match[0]
        return True

    def format_signaure(self):
        return ''

    def format_name(self):
        return self.fullname

    def get_doc(self, encoding):
        if self.brief:
            description = self.object.find('briefdescription')
        else:
            description = self.object.find('detaileddescription')
        doc = [format_xml_paragraph(description)]
        return doc

    def get_object_members(self, want_all):
        all_members = self.object.xpath('.//sectiondef[@kind="func" '
            'or @kind="public-static-func"]/memberdef[@kind="function"]')

        if want_all:
            return False, ((m.find('name').text, m) for m in all_members)
        else:
            if not self.options.members:
                return False, []
            else:
                return False, ((m.find('name').text, m) for m in all_members
                               if m.find('name').text in self.options.members)

    def filter_members(self, members, want_all):
        ret = []
        for (membername, member) in members:
            ret.append((membername, member, False))
        return ret

    def document_members(self, all_members=False):
        super().document_members(all_members=all_members)
        # Uncomment to view the generated rst for the class.
        # print('\n'.join(self.directive.result))

    def generate(self, more_content=None, real_modname=None,
                 check_module=False, all_members=False):
        if not self.parse_name():
            self.directive.warn("don't know which module to import for autodocumenting %r" % self.name)
            return

        if not self.import_object():
            return

        self.real_modname = real_modname or self.get_real_modname()

        # we can't import anything, since we're not Python
        self.analyzer = None

        if check_module and not self.check_module():
            return

        sourcename = self.get_sourcename()

        # start with blank line
        self.add_line(u'', sourcename)
        title = '%s module reference' % self.format_name()

        # add title
        self.add_line(u'='*len(title), sourcename)
        self.add_line(title, sourcename)
        self.add_line(u'='*len(title), sourcename)
        self.add_line(u'', sourcename)

        # brief description
        self.brief = True
        self.add_content(None)

        # we want a brief description of types/functions here
        self.document_members(all_members)

        # detailed description
        self.add_line(u'', sourcename)
        self.add_line(u'.. rubric:: Detailed Description', sourcename)
        self.add_line(u'', sourcename)
        self.brief = False
        self.add_content(more_content)

        # member doc
        self.document_members(all_members)

class DoxygenMethodDocumenter(DoxygenDocumenter):
    objtype = 'doxymethod'
    directivetype = 'function'
    domain = 'f'
    priority = 100

    @classmethod
    def can_document_member(cls, member, membername, isattr, parent):
        if ET.iselement(member) and member.tag == 'memberdef' and member.get('kind') == 'function':
            return True
        return False

    def add_directive_header(self, sig):
        """Add the directive header and options to the generated content."""
        domain = self.domain
        # use the <type> field, without other information (e.g. public)
        name = self.format_name()
        sourcename = self.get_sourcename()

        typefield = self.object.find('type').text
        if typefield is None:
            print('ERROR: null typefield in %s %s' % (name, sourcename))
            return

        directive = 'subroutine' if 'subroutine' in typefield else 'function'

        self.add_line(u'.. %s:%s:: %s%s' % (domain, directive, name, sig),
                      sourcename)

    def parse_id(self, id):
        # try to search our parent node instead of the entire tree
        parent = self.parent
        if parent is None:
            parent = get_doxygen_root()

        xp = './/*[@id="%s"]' % id
        match = parent.xpath(xp)
        if len(match) > 0:
            match = match[0]
            self.fullname = match.find('./definition').text.split()[-1]
            self.modname = self.fullname
            self.objname = match.find('./name').text
            self.object = match
        return False

    def import_object(self):
        if ET.iselement(self.object):
            # self.object already set from DoxygenDocumenter.parse_name(),
            # caused by passing in the `id` of the node instead of just a
            # classname or method name
            return True

        return False

    def get_doc(self, encoding):
        doc = [format_xml_paragraph(self.object.find('briefdescription'))]
        # add parameter documentation (in detaileddescription) for main function documentation
        if not self.brief:
            doc += [format_xml_paragraph(self.object.find('detaileddescription'))]

        return doc

    def format_name(self):
        def text(el):
            if el.text is not None:
                return el.text
            return ''

        def tail(el):
            if el.tail is not None:
                return el.tail
            return ''

        rtype_el = self.object.find('type')
        rtype_el_ref = rtype_el.find('ref')
        if rtype_el_ref is not None:
            rtype = text(rtype_el) + text(rtype_el_ref) + tail(rtype_el_ref)
        else:
            rtype = rtype_el.text

        signame = (rtype and (rtype + ' ') or '') + self.objname
        return self.format_template_name() + signame

    def format_template_name(self):
        types = [e.text for e in self.object.findall('templateparamlist/param/type')]
        if len(types) == 0:
            return ''
        return 'template <%s>\n' % ','.join(types)

    def format_signature(self):
        args = self.object.find('argsstring').text
        return args

    def document_members(self, all_members=False):
        pass
<commit_msg>Strip Doxygen annotations from function signatures<commit_after>from __future__ import print_function, absolute_import, division

import re

from six import itervalues
from lxml import etree as ET
from sphinx.ext.autodoc import Documenter, AutoDirective, members_option, ALL
from sphinx.errors import ExtensionError

from . import get_doxygen_root
from .xmlutils import format_xml_paragraph

class DoxygenDocumenter(Documenter):
    # Variables to store the names of the object being documented. modname and fullname are redundant,
    # and objpath is always the empty list. This is inelegant, but we need to work with the superclass.

    fullname = None  # example: "OpenMM::NonbondedForce" or "OpenMM::NonbondedForce::methodName""
    modname = None   # example: "OpenMM::NonbondedForce" or "OpenMM::NonbondedForce::methodName""
    objname = None   # example: "NonbondedForce"  or "methodName"
    objpath = []     # always the empty list
    object = None    # the xml node for the object
    titles_allowed = True

    option_spec = {
        'members': members_option,
    }

    def __init__(self, directive, name, indent=u'', id=None, brief=False, parent=None):
        super().__init__(directive, name, indent)

        self.parent = parent
        if id is not None:
            self.parse_id(id)
        self.brief = brief

    def parse_id(self, id):
        return False

    def parse_name(self):
        """Determine what module to import and what attribute to document.
        Returns True and sets *self.modname*, *self.objname*, *self.fullname*,
        if parsing and resolving was successful.
        """
        # To view the context and order in which all of these methods get called,
        # See, Documenter.generate(). That's the main "entry point" that first
        # calls parse_name(), follwed by import_object(), format_signature(),
        # add_directive_header(), and then add_content() (which calls get_doc())

        # methods in the superclass sometimes use '.' to join namespace/class
        # names with method names, and we don't want that.
        self.name = self.name.replace('.', '::')
        self.fullname = self.name
        self.modname = self.fullname
        self.objpath = []

        if '::' in self.name:
            parts = self.name.split('::')
            self.objname = parts[-1]
        else:
            self.objname = self.name

        return True

    def document_members(self, all_members=False):
        """Generate reST for member documentation.
        If *all_members* is True, do all members, else those given by
        *self.options.members*.
        """
        want_all = all_members or self.options.inherited_members or \
            self.options.members is ALL
        # find out which members are documentable
        members_check_module, members = self.get_object_members(want_all)

        # remove members given by exclude-members
        if self.options.exclude_members:
            members = [(membername, member) for (membername, member) in members
                       if membername not in self.options.exclude_members]

        # document non-skipped members
        memberdocumenters = []
        for (mname, member, isattr) in self.filter_members(members, want_all):
            classes = [cls for cls in itervalues(AutoDirective._registry)
                       if cls.can_document_member(member, mname, isattr, self)]
            if not classes:
                # don't know how to document this member
                continue

            # prefer the documenter with the highest priority
            classes.sort(key=lambda cls: cls.priority)

            documenter = classes[-1](self.directive, mname, indent=self.indent,
                                     id=member.get('id'), brief=self.brief,
                                     parent=self.object)
            memberdocumenters.append((documenter, isattr))

        for documenter, isattr in memberdocumenters:
            documenter.generate(
                all_members=True, real_modname=self.real_modname,
                check_module=members_check_module and not isattr)

        # reset current objects
        self.env.temp_data['autodoc:module'] = None
        self.env.temp_data['autodoc:class'] = None

class DoxygenClassDocumenter(DoxygenDocumenter):
    objtype = 'doxyclass'
    directivetype = 'class'
    domain = 'cpp'
    priority = 100

    option_spec = {
        'members': members_option,
    }

    @classmethod
    def can_document_member(cls, member, membername, isattr, parent):
        # this method is only called from Documenter.document_members
        # when a higher level documenter (module or namespace) is trying
        # to choose the appropriate documenter for each of its lower-level
        # members. Currently not implemented since we don't have a higher-level
        # doumenter like a DoxygenNamespaceDocumenter.
        return False

    def import_object(self):
        """Import the object and set it as *self.object*.  In the call sequence, this
        is executed right after parse_name(), so it can use *self.fullname*, *self.objname*,
        and *self.modname*.

        Returns True if successful, False if an error occurred.
        """
        xpath_query = './compounddef/compoundname[text()="%s"]/..' % self.fullname
        match = get_doxygen_root().xpath(xpath_query)
        if len(match) != 1:
            raise ExtensionError('[autodoc_doxygen] could not find class (fullname="%s"). I tried'
                                 'the following xpath: "%s"' % (self.fullname, xpath_query))

        self.object = match[0]
        return True

    def format_signaure(self):
        return ''

    def format_name(self):
        return self.fullname

    def get_doc(self, encoding):
        if self.brief:
            description = self.object.find('briefdescription')
        else:
            description = self.object.find('detaileddescription')
        doc = [format_xml_paragraph(description)]
        return doc

    def get_object_members(self, want_all):
        all_members = self.object.xpath('.//sectiondef[@kind="func" '
            'or @kind="public-static-func"]/memberdef[@kind="function"]')

        if want_all:
            return False, ((m.find('name').text, m) for m in all_members)
        else:
            if not self.options.members:
                return False, []
            else:
                return False, ((m.find('name').text, m) for m in all_members
                               if m.find('name').text in self.options.members)

    def filter_members(self, members, want_all):
        ret = []
        for (membername, member) in members:
            ret.append((membername, member, False))
        return ret

    def document_members(self, all_members=False):
        super().document_members(all_members=all_members)
        # Uncomment to view the generated rst for the class.
        # print('\n'.join(self.directive.result))

    def generate(self, more_content=None, real_modname=None,
                 check_module=False, all_members=False):
        if not self.parse_name():
            self.directive.warn("don't know which module to import for autodocumenting %r" % self.name)
            return

        if not self.import_object():
            return

        self.real_modname = real_modname or self.get_real_modname()

        # we can't import anything, since we're not Python
        self.analyzer = None

        if check_module and not self.check_module():
            return

        sourcename = self.get_sourcename()

        # start with blank line
        self.add_line(u'', sourcename)
        title = '%s module reference' % self.format_name()

        # add title
        self.add_line(u'='*len(title), sourcename)
        self.add_line(title, sourcename)
        self.add_line(u'='*len(title), sourcename)
        self.add_line(u'', sourcename)

        # brief description
        self.brief = True
        self.add_content(None)

        # we want a brief description of types/functions here
        self.document_members(all_members)

        # detailed description
        self.add_line(u'', sourcename)
        self.add_line(u'.. rubric:: Detailed Description', sourcename)
        self.add_line(u'', sourcename)
        self.brief = False
        self.add_content(more_content)

        # member doc
        self.document_members(all_members)

class DoxygenMethodDocumenter(DoxygenDocumenter):
    objtype = 'doxymethod'
    directivetype = 'function'
    domain = 'f'
    priority = 100

    @classmethod
    def can_document_member(cls, member, membername, isattr, parent):
        if ET.iselement(member) and member.tag == 'memberdef' and member.get('kind') == 'function':
            return True
        return False

    def add_directive_header(self, sig):
        """Add the directive header and options to the generated content."""
        domain = self.domain
        # use the <type> field, without other information (e.g. public)
        name = self.format_name()
        sourcename = self.get_sourcename()

        typefield = self.object.find('type').text
        if typefield is None:
            print('ERROR: null typefield in %s %s' % (name, sourcename))
            return

        directive = 'subroutine' if 'subroutine' in typefield else 'function'

        self.add_line(u'.. %s:%s:: %s%s' % (domain, directive, name, sig),
                      sourcename)

    def parse_id(self, id):
        # try to search our parent node instead of the entire tree
        parent = self.parent
        if parent is None:
            parent = get_doxygen_root()

        xp = './/*[@id="%s"]' % id
        match = parent.xpath(xp)
        if len(match) > 0:
            match = match[0]
            self.fullname = match.find('./definition').text.split()[-1]
            self.modname = self.fullname
            self.objname = match.find('./name').text
            self.object = match
        return False

    def import_object(self):
        if ET.iselement(self.object):
            # self.object already set from DoxygenDocumenter.parse_name(),
            # caused by passing in the `id` of the node instead of just a
            # classname or method name
            return True

        return False

    def get_doc(self, encoding):
        doc = [format_xml_paragraph(self.object.find('briefdescription'))]
        # add parameter documentation (in detaileddescription) for main function documentation
        if not self.brief:
            doc += [format_xml_paragraph(self.object.find('detaileddescription'))]

        return doc

    def format_name(self):
        # we just want to get the bare part of the "type" field
        # i.e. subroutine or <type> function
        typefield = self.object.find('type').text
        if typefield is None:
            rtype = None
        elif 'function' in typefield:
            # get the return type
            rtype = re.search(r'(\S+)\s+function', typefield).group(0)
        else:
            rtype = 'subroutine' if 'subroutine' in typefield else 'unknown'

        signame = ((rtype + ' ') if rtype is not None else '') + self.objname
        return self.format_template_name() + signame

    def format_template_name(self):
        types = [e.text for e in self.object.findall('templateparamlist/param/type')]
        if len(types) == 0:
            return ''
        return 'template <%s>\n' % ','.join(types)

    def format_signature(self):
        args = self.object.find('argsstring').text
        return args

    def document_members(self, all_members=False):
        pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
'''Unit test suite.
'''
import unittest, sys, os

class TestCase(unittest.TestCase):
  def assertContains(self, collection1, collection2):
    '''Test that all items in `collection1` is contained within `collection2`.

    The order of the contained items does not matter.

    :param collection1: Collection 1
    :type  collection1: collection
    :param collection2: Collection 2
    :type  collection2: collection
    :rtype: None
    '''
    for item in collection1:
      if item not in collection2:
        raise AssertionError(u'%r !contains %r' % (collection1, collection2))

def load_suites(module_names):
  suites = []
  if isinstance(module_names, str):
    module_names = module_names.strip().split('\n')
  for modname in module_names:
    modname = modname.strip()
    if modname:
      __import__(modname)
      mod = sys.modules[modname]
      try:
        suites.append(getattr(mod, 'suite')())
      except AttributeError, e:
        if "has no attribute 'suite'" in e.args[0]:
          e.args = ('module %s has no suite' % mod,)
        raise e
  return suites

def suite():
  # for some reeeeally weird reason, the live test must be run before mvc.*
  suites = load_suites('''
    smisk.test.live.negotiate_charset

    smisk.test.config
    smisk.test.core.url
    smisk.test.core.xml
    smisk.test.inflection
    smisk.test.mvc.control
    smisk.test.mvc.routing
    smisk.test.serialization
    smisk.test.util.introspect
    smisk.test.util.string_
  ''')
  return unittest.TestSuite(suites)

def test():
  runner = unittest.TextTestRunner()
  return runner.run(suite())

if __name__ == "__main__":
  test()
<commit_msg>removed the live test from default tests<commit_after>#!/usr/bin/env python
# encoding: utf-8
'''Unit test suite.
'''
import unittest, sys, os

class TestCase(unittest.TestCase):
  def assertContains(self, collection1, collection2):
    '''Test that all items in `collection1` is contained within `collection2`.

    The order of the contained items does not matter.

    :param collection1: Collection 1
    :type  collection1: collection
    :param collection2: Collection 2
    :type  collection2: collection
    :rtype: None
    '''
    for item in collection1:
      if item not in collection2:
        raise AssertionError(u'%r !contains %r' % (collection1, collection2))

def load_suites(module_names):
  suites = []
  if isinstance(module_names, str):
    module_names = module_names.strip().split('\n')
  for modname in module_names:
    modname = modname.strip()
    if modname:
      __import__(modname)
      mod = sys.modules[modname]
      try:
        suites.append(getattr(mod, 'suite')())
      except AttributeError, e:
        if "has no attribute 'suite'" in e.args[0]:
          e.args = ('module %s has no suite' % mod,)
        raise e
  return suites

def suite():
  suites = load_suites('''
    smisk.test.config
    smisk.test.core.url
    smisk.test.core.xml
    smisk.test.inflection
    smisk.test.mvc.control
    smisk.test.mvc.routing
    smisk.test.serialization
    smisk.test.util.introspect
    smisk.test.util.string_
  ''')
  return unittest.TestSuite(suites)

def test():
  runner = unittest.TextTestRunner()
  return runner.run(suite())

if __name__ == "__main__":
  test()
<|endoftext|>


<commit_before>#!/usr/bin/python

# script for populating security group from command line in easy way
#
# Examples:
#
# open ports 80 443 and 22 for CIDRs 212.68.9.98/32 193.166.24.0/23 in security
# group "default":
# $ managesecgroup -n default -p 80 443 22 -i 212.68.9.98/32 193.166.24.0/23
#
# create security group called "testenv" and populate with rules opening
# 80,443 and 22 to 212.68.9.98/32 193.166.24.0/23
# $ managesecgroup -n testenv -c -p 80 443 22 \
#                  -i 212.68.9.98/32 193.166.24.0/23

import argparse
import sys
import socket

import util

i = util.logger.info

CIDR_ALIASES = {'digile': '83.150.108.249/32', 'forge': '193.166.24.0/23' }

desc = ('Create of modify security group')

def check_CIDR(cidr):
    i("Checking if '%s' is a proper CIDR address" % cidr)
    ip, maskbitcount = cidr.split('/')
    maskbitcount = int(maskbitcount)
    if not (0 <= maskbitcount <= 32):
        raise Exception("the number after slash must be 0-32")
    socket.inet_aton(ip)

def get_args(args_list):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description=desc)

    help_create = 'create the security group first'

    parser.add_argument('-c', '--create', help=help_create, default=False,
                        action='store_true')

    parser.add_argument('-n', '--name', help='name of the sec group',
                        default='default', required=True)

    parser.add_argument('-p', '--ports', help='space separated list of ports',
                        nargs='+', default=['443', '80', '22'])

    parser.add_argument('-i', '--cidrs', help='space separated list of CIDRs',
                        nargs='+', required=True)

    parser.add_argument('-t', '--test', help='test run, dont change anything',
                        action='store_true')

    return parser.parse_args(args_list)

def main(args_list):
    args = get_args(args_list)
    _nova = util.NovaProxy

    sec_group = None

    if not args.test:
        if args.create:
            sec_group = _nova().security_groups.create(args.name, args.name)
        else:
            sec_group = _nova().security_groups.find(name=args.name)

    for port in args.ports:
        for cidr in args.cidrs:
            true_cidr = CIDR_ALIASES.get(cidr, cidr)
            check_CIDR(true_cidr)
            i("adding rule: port %s, cidr %s " % (port, cidr))
            if not args.test:
                _nova().security_group_rules.create(sec_group.id,
                    ip_protocol='tcp', from_port=port, to_port=port,
                    # if cidr is alias, try to get it from the alias dict,
                    # if it's not an alias, use it as CIDR:
                    cidr=true_cidr)
            else:
                i('This is just a test, the rule is not added')

if __name__ == '__main__':
    main(sys.argv[1:])

<commit_msg>added ip of admin machine<commit_after>#!/usr/bin/python

# script for populating security group from command line in easy way
#
# Examples:
#
# open ports 80 443 and 22 for CIDRs 212.68.9.98/32 193.166.24.0/23 in security
# group "default":
# $ managesecgroup -n default -p 80 443 22 -i 212.68.9.98/32 193.166.24.0/23
#
# create security group called "testenv" and populate with rules opening
# 80,443 and 22 to 212.68.9.98/32 193.166.24.0/23
# $ managesecgroup -n testenv -c -p 80 443 22 \
#                  -i 212.68.9.98/32 193.166.24.0/23

import argparse
import sys
import socket

import util

i = util.logger.info

CIDR_ALIASES = {'digile': '83.150.108.249/32',
                'forge': '193.166.24.0/23'
                'admin': '86.50.27.230/32'}

desc = ('Create of modify security group')

def check_CIDR(cidr):
    i("Checking if '%s' is a proper CIDR address" % cidr)
    ip, maskbitcount = cidr.split('/')
    maskbitcount = int(maskbitcount)
    if not (0 <= maskbitcount <= 32):
        raise Exception("the number after slash must be 0-32")
    socket.inet_aton(ip)

def get_args(args_list):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
        description=desc)

    help_create = 'create the security group first'

    parser.add_argument('-c', '--create', help=help_create, default=False,
                        action='store_true')

    parser.add_argument('-n', '--name', help='name of the sec group',
                        default='default', required=True)

    parser.add_argument('-p', '--ports', help='space separated list of ports',
                        nargs='+', default=['443', '80', '22'])

    parser.add_argument('-i', '--cidrs', help='space separated list of CIDRs',
                        nargs='+', required=True)

    parser.add_argument('-t', '--test', help='test run, dont change anything',
                        action='store_true')

    return parser.parse_args(args_list)

def main(args_list):
    args = get_args(args_list)
    _nova = util.NovaProxy

    sec_group = None

    if not args.test:
        if args.create:
            sec_group = _nova().security_groups.create(args.name, args.name)
        else:
            sec_group = _nova().security_groups.find(name=args.name)

    for port in args.ports:
        for cidr in args.cidrs:
            true_cidr = CIDR_ALIASES.get(cidr, cidr)
            check_CIDR(true_cidr)
            i("adding rule: port %s, cidr %s " % (port, cidr))
            if not args.test:
                _nova().security_group_rules.create(sec_group.id,
                    ip_protocol='tcp', from_port=port, to_port=port,
                    # if cidr is alias, try to get it from the alias dict,
                    # if it's not an alias, use it as CIDR:
                    cidr=true_cidr)
            else:
                i('This is just a test, the rule is not added')

if __name__ == '__main__':
    main(sys.argv[1:])

<|endoftext|>


<commit_before>from mongoengine import *
import json, datetime

class Alarm(Document):
    name = StringField(required=True)
    time = DateTimeField(required=True)
    activated = BooleanField(default=True)

def returnJsonFromRaw(rawJson):
    raw_data = json.loads(rawJson)
    if not raw_data:
        print "No data was sent"
        return
    if not (raw_data['name'] and raw_data['time']):
        print "The fields either could not be parsed, or one of them is missing"
        return
    return  raw_data

def writeJsonToDatabase(obj):
    alarm = Alarm(name=obj['name'], time=obj['time'])
    alarm.save()

def getAlarms():
    return Alarm.objects

def getTodayAlarms():
    yesterday = Date.today() - timedelta(1)
    return Alarm.objects(time__gt=yesterday)

def activateAlarm(name):
    alarm = Alarm.objects(name__exact=name)
    alarm.activated = True
    alarm.save()
    print "Alarm " + name + " saved."
    return

<commit_msg>Added to_json to Alarm<commit_after>from mongoengine import *
import json, datetime, bson

class Alarm(Document):
    name = StringField(required=True)
    time = DateTimeField(required=True)
    activated = BooleanField(default=True)

    def to_json(self):
        return bson.json_util.dumps(obj)

def returnJsonFromRaw(rawJson):
    raw_data = json.loads(rawJson)
    if not raw_data:
        print "No data was sent"
        return
    if not (raw_data['name'] and raw_data['time']):
        print "The fields either could not be parsed, or one of them is missing"
        return
    return  raw_data

def writeJsonToDatabase(obj):
    alarm = Alarm(name=obj['name'], time=obj['time'])
    alarm.save()

def getAlarms():
    return Alarm.objects

def getTodayAlarms():
    yesterday = Date.today() - timedelta(1)
    return Alarm.objects(time__gt=yesterday)

def activateAlarm(name):
    alarm = Alarm.objects(name__exact=name)
    alarm.activated = True
    alarm.save()
    print "Alarm " + name + " saved."
    return

<|endoftext|>


<commit_before>"""
Various utility functions used by Manhattan. It is not expected that these will
be used externally.
"""

import os
import random
import bisect

"""
A 1 pixel transparent GIF as a bytestring. For use as a tracking "beacon" in an
HTML document.
"""
transparent_pixel = (
    'GIF89a'
    '\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00!\xf9\x04'
    '\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02'
    '\x02D\x01\x00;')

def pixel_tag(path):
    """
    Build the HTML surrounding a given image path that will be injected as a
    tracking pixel.

    :param path:
        URL path to the tracking pixel.
    :type request:
        string
    :returns:
        HTML markup for image tag.
    :rtype:
        string
    """
    return ('' % path)

def nonce():
    """
    Build a cryptographically random nonce.

    :returns:
        Hex string, with 20 bytes (40 hex chars).
    :rtype:
        string
    """
    return os.urandom(20).encode('hex')

def nonrandom_choice(seed, seq):
    """
    Pick an element from the specified sequence ``seq`` based on the value of
    the specified string ``seed``. Guaranteed to be deterministic, tries to be
    uniform.

    :param seed:
        Any string specifier to control the element.
    :type seed:
        string
    :param seq:
        Python sequence to pick from.
    :type seq:
        sequence object
    :returns:
        An element from ``seq``.
    """
    return random.Random(seed).choice(seq)

def nonrandom(seed, n):
    """
    Return a deterministic but pseudo-random number between 0 and ``n``, based
    on the value of ``seed``. Guaranteed to be deterministic, tries to be
    uniform.

    :param seed:
        Any string specifier to control the output.
    :type seed:
        string
    :param n:
        Range of output value.
    :type n:
        float
    :returns:
        Float between 0 and ``n``.
    :rtype:
        float
    """
    return random.Random(seed).random() * n

def choose_population(seed, populations=None):
    """
    Randomly pick an element from populations according to type.

    :param seed:
        Any string specifier to control the output.
    :type seed:
        string
    :param populations:
        If not specified, perform a straight AB test between True and False. If
        specified as a sequence, pick uniformly from the sequence. If specified
        as a dict, use the keys as population names and the values as the
        weight that each population should receive, and distribute between them
        according to weight.
    :type populations:
        None, sequence, or dict
    :returns:
        Selected population
    """
    # Uniform distribution between True, False
    if populations is None:
        return nonrandom_choice(seed, (True, False))

    # Uniform distribution over populations
    if isinstance(populations, list):
        return nonrandom_choice(seed, populations)

    # Weighted distribution over populations
    if isinstance(populations, dict):
        pop_name = []
        pop_mass = []
        running_mass = 0
        for name, mass in populations.iteritems():
            if mass > 0:
                pop_name.append(name)
                pop_mass.append(running_mass)
                running_mass += mass

        if running_mass == 0:
            raise ValueError("Need at least one option with probability > 0")

        r = nonrandom(seed, running_mass)
        i = bisect.bisect(pop_mass, r) - 1
        return pop_name[i]

    raise ValueError("Invalid population description")

def decode_http_header(raw):
    """
    Decode a raw HTTP header into a unicode string. RFC 2616 specifies that
    they should be latin1-encoded (a.k.a. iso-8859-1). If the passed-in value
    is None, return an empty unicode string.

    :param raw:
        Raw HTTP header string.
    :type raw:
        string (non-unicode)
    :returns:
        Decoded HTTP header.
    :rtype:
        unicode string
    """
    if raw:
        return raw.decode('iso-8859-1', 'replace')
    else:
        return u''
<commit_msg>use binascii.hexlify(s) instead of s.encode('hex'), it's more future-proof<commit_after>"""
Various utility functions used by Manhattan. It is not expected that these will
be used externally.
"""

import os
import random
import bisect
import binascii

"""
A 1 pixel transparent GIF as a bytestring. For use as a tracking "beacon" in an
HTML document.
"""
transparent_pixel = (
    'GIF89a'
    '\x01\x00\x01\x00\x80\x00\x00\xff\xff\xff\x00\x00\x00!\xf9\x04'
    '\x00\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02'
    '\x02D\x01\x00;')

def pixel_tag(path):
    """
    Build the HTML surrounding a given image path that will be injected as a
    tracking pixel.

    :param path:
        URL path to the tracking pixel.
    :type request:
        string
    :returns:
        HTML markup for image tag.
    :rtype:
        string
    """
    return ('' % path)

def nonce():
    """
    Build a cryptographically random nonce.

    :returns:
        Hex string, with 20 bytes (40 hex chars).
    :rtype:
        string
    """
    return binascii.hexlify(os.urandom(20))

def nonrandom_choice(seed, seq):
    """
    Pick an element from the specified sequence ``seq`` based on the value of
    the specified string ``seed``. Guaranteed to be deterministic, tries to be
    uniform.

    :param seed:
        Any string specifier to control the element.
    :type seed:
        string
    :param seq:
        Python sequence to pick from.
    :type seq:
        sequence object
    :returns:
        An element from ``seq``.
    """
    return random.Random(seed).choice(seq)

def nonrandom(seed, n):
    """
    Return a deterministic but pseudo-random number between 0 and ``n``, based
    on the value of ``seed``. Guaranteed to be deterministic, tries to be
    uniform.

    :param seed:
        Any string specifier to control the output.
    :type seed:
        string
    :param n:
        Range of output value.
    :type n:
        float
    :returns:
        Float between 0 and ``n``.
    :rtype:
        float
    """
    return random.Random(seed).random() * n

def choose_population(seed, populations=None):
    """
    Randomly pick an element from populations according to type.

    :param seed:
        Any string specifier to control the output.
    :type seed:
        string
    :param populations:
        If not specified, perform a straight AB test between True and False. If
        specified as a sequence, pick uniformly from the sequence. If specified
        as a dict, use the keys as population names and the values as the
        weight that each population should receive, and distribute between them
        according to weight.
    :type populations:
        None, sequence, or dict
    :returns:
        Selected population
    """
    # Uniform distribution between True, False
    if populations is None:
        return nonrandom_choice(seed, (True, False))

    # Uniform distribution over populations
    if isinstance(populations, list):
        return nonrandom_choice(seed, populations)

    # Weighted distribution over populations
    if isinstance(populations, dict):
        pop_name = []
        pop_mass = []
        running_mass = 0
        for name, mass in populations.iteritems():
            if mass > 0:
                pop_name.append(name)
                pop_mass.append(running_mass)
                running_mass += mass

        if running_mass == 0:
            raise ValueError("Need at least one option with probability > 0")

        r = nonrandom(seed, running_mass)
        i = bisect.bisect(pop_mass, r) - 1
        return pop_name[i]

    raise ValueError("Invalid population description")

def decode_http_header(raw):
    """
    Decode a raw HTTP header into a unicode string. RFC 2616 specifies that
    they should be latin1-encoded (a.k.a. iso-8859-1). If the passed-in value
    is None, return an empty unicode string.

    :param raw:
        Raw HTTP header string.
    :type raw:
        string (non-unicode)
    :returns:
        Decoded HTTP header.
    :rtype:
        unicode string
    """
    if raw:
        return raw.decode('iso-8859-1', 'replace')
    else:
        return u''
<|endoftext|>