<commit_before># -*- coding: utf-8 -*-
#
# Session route handlers
#
import logging
from webapp2 import RequestHandler
from helpers import google_api
from helpers.sessions import SessionHandler
from helpers.views import render
from controllers.root import error_500

# Build oAuth2 request and redirect to Google authentication endpoint
class LoginRoute(RequestHandler):
    def get(self):
        if 'approval_prompt' in self.request.GET.keys():
            # Use /auth/login?approval_prompt to force refresh of authentication tokens
            flow = google_api.oauth2_flow(approval_prompt='force')
        else:
            flow = google_api.oauth2_flow()
        self.redirect(flow.step1_get_authorize_url())

# Clear user session and return to home page
class LogoutRoute(SessionHandler):
    def get(self):
        if 'credentials' in self.session:
            del self.session['credentials']
        self.redirect('/')

# Handle oAuth2 callback
# Create session and (if required) user account
class OAuth2CallbackRoute(SessionHandler):
    def get(self):

        code = self.request.GET.get('code')
        if None == code:
            return self.response.write('No authentication code returned')
            # TODO Display cancelled login page
            self.redirect('/')

        try:
            flow = google_api.oauth2_flow()
            auth = flow.step2_exchange(code)

            # TODO Attempt to fetch user record from DB with matching google_id

            # TODO if not user exists with google id
                # TODO Create user record in DB

            # If we have no refresh token, redirect to /auth/login?approval_prompt to get one
            # TODO also check DB to prevent needless double-auth
            if None == auth.refresh_token:
                self.redirect('/auth/login?approval_prompt')

            # Get user info
            user_info = google_api.user_info(auth.to_json())
            # TODO Store updated user info in DB
            # TODO Store updated credentials in DB

            # Create session
            self.credentials(auth.to_json())

            # TODO if any post-login redirects have been stored in the session
                # TODO delete redirect from session
                # TODO issue redirect

            # Show results
            # TODO redirect to dashboard instead
            data = {'message': 'Authenticated!', 'user_info': user_info, 'session': self.session}
            self.response.write(render('index.html', data))

        except Exception as e:
            error_500(self.request, self.response, e)<commit_msg>Comment edits<commit_after># -*- coding: utf-8 -*-
#
# Session route handlers
#
import logging
from webapp2 import RequestHandler
from helpers import google_api
from helpers.sessions import SessionHandler
from helpers.views import render
from controllers.root import error_500

# Build oAuth2 request and redirect to Google authentication endpoint
class LoginRoute(RequestHandler):
    def get(self):
        if 'approval_prompt' in self.request.GET.keys():
            # Use /auth/login?approval_prompt to force refresh of authentication tokens
            flow = google_api.oauth2_flow(approval_prompt='force')
        else:
            flow = google_api.oauth2_flow()
        self.redirect(flow.step1_get_authorize_url())

# Clear user session and return to home page
class LogoutRoute(SessionHandler):
    def get(self):
        if 'credentials' in self.session:
            # TODO delete user_id from session
            del self.session['credentials']
        self.redirect('/')

# Handle oAuth2 callback
# Create session and (if required) user account
class OAuth2CallbackRoute(SessionHandler):
    def get(self):

        code = self.request.GET.get('code')
        if None == code:
            return self.response.write('No authentication code returned')
            # TODO Display cancelled login page
            self.redirect('/')

        try:
            flow = google_api.oauth2_flow()
            auth = flow.step2_exchange(code)

            # TODO Attempt to fetch user record from DB with matching google_id

            # TODO if not user exists with google id
                # TODO Create user record in DB

            # If we have no refresh token, redirect to /auth/login?approval_prompt to get one
            # TODO also check DB to prevent needless double-auth
            if None == auth.refresh_token:
                self.redirect('/auth/login?approval_prompt')

            # Get user info
            user_info = google_api.user_info(auth.to_json())
            # TODO Store updated user info in DB
            # TODO Store updated credentials in DB

            # Create session
            # TODO Store DB user_id in session
            self.credentials(auth.to_json())

            # TODO if any post-login redirects have been stored in the session
                # TODO delete redirect from session
                # TODO issue redirect

            # Show results
            # TODO redirect to dashboard instead
            data = {'message': 'Authenticated!', 'user_info': user_info, 'session': self.session}
            self.response.write(render('index.html', data))

        except Exception as e:
            error_500(self.request, self.response, e)<|endoftext|>


<commit_before>import cPickle
import gzip
import os

import numpy as np

from convnet.layers import *
from convnet.layers.convolutional_layer import _ConvolutionalLayer
from convnet.net import ConvNet
from convnet.utils import y_1d_to_3d

np.set_printoptions(precision=4, linewidth=120)

def transform_X(X_train):
    X = []
    for i in xrange(X_train.shape[0]):
        x = X_train[i]
        x = x.reshape((28, 28))[:, :, np.newaxis]
        X.append(x)
    return X

def transform_Y(Y_train):
    Y = []
    uniq = np.unique(Y_train)
    labels_count = uniq.size
    for i in xrange(Y_train.shape[0]):
        y = Y_train[i]
        y_arr = np.zeros(labels_count)
        index = np.where(uniq == y)[0][0]
        y_arr[index] = 1
        Y.append(y_1d_to_3d(y_arr))
    return Y

def get_examples(X_train, Y_train, labels=None, count=None):
    if isinstance(count, int):
        count = [count] * len(labels)
    assert len(labels) == len(count)

    X = None
    Y = None
    for i, label in enumerate(labels):
        indices = np.where(Y_train == label)[0]
        indices = indices[:count[i]]
        if X is None:
            X = X_train[indices]
        else:
            X = np.concatenate([X, X_train[indices]])

        if Y is None:
            Y = Y_train[indices]
        else:
            Y = np.concatenate([Y, Y_train[indices]])

    return X, Y

def shuffle_in_unison_inplace(a, b):
    assert len(a) == len(b)
    p = np.random.permutation(len(a))
    return a[p], b[p]

# noinspection PyPep8Naming
def mnist():
    script_path = os.path.dirname(os.path.abspath(__file__))
    f = gzip.open(os.path.join(script_path, './mnist.pkl.gz'), 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    f.close()

    X_train, Y_train = train_set

    X_train, Y_train = get_examples(X_train, Y_train, labels=np.arange(0, 10), count=5)
    X_train, Y_train = shuffle_in_unison_inplace(X_train, Y_train)
    X_train = transform_X(X_train)
    Y_train = transform_Y(Y_train)

    X_test, Y_test = test_set
    X_test, Y_test = get_examples(X_test, Y_test, labels=np.arange(0, 10), count=2)
    X_test, Y_test = shuffle_in_unison_inplace(X_test, Y_test)
    X_test = transform_X(X_test)
    Y_test = transform_Y(Y_test)

    net = ConvNet(iterations_count=30, batch_size=10, learning_rate=0.001, momentum=0.8, weight_decay=0.001)
    net.setup_layers([
        InputLayer(InputLayerSettings(in_shape=X_train[0].shape)),

        ConvolutionalLayer(ConvolutionalLayerSettings(filters_count=16, filter_size=5, stride=1, zero_padding=0)),
        ReluLayer(ReluLayerSettings(activation='max')),
        PoolingLayer(PoolingLayerSettings(filter_size=2, stride=2)),

        ConvolutionalLayer(ConvolutionalLayerSettings(filters_count=8, filter_size=5, stride=1, zero_padding=0)),
        ReluLayer(ReluLayerSettings(activation='max')),
        PoolingLayer(PoolingLayerSettings(filter_size=3, stride=3)),

        FullConnectedLayer(FullConnectedLayerSettings(neurons_count=Y_train[0].shape[-1], activation='sigmoid')),
    ])

    examples_count = 100000

    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    # net = ConvNet.load_net(os.path.join(BASE_DIR, './convnet1.pkl'))
    net.fit(X_train[:examples_count], Y_train[:examples_count])

    matched = 0
    for x, y in zip(X_test[:examples_count], Y_test[:examples_count]):
        h = net.predict(x)
        h_res = h.argmax()
        y_res = y.argmax()
        print("predicted = {}; max = {}".format(h, h.argmax()))
        print("real =      {}; max = {}".format(y, y.argmax()))
        print("\n")
        matched += int(h_res == y_res)

    print("Accuracy {}/{}".format(matched, len(X_test[:examples_count])))

    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    path = os.path.join(BASE_DIR, "./convnet1.pkl")
    net.dump_net(path)
    print("Dumped to {}".format(path))

if __name__ == "__main__":
    mnist()
    # net = ConvNet.load_net('/home/igor/Desktop/convnet.pkl')
    pass
<commit_msg>[convnet_web] some changes again<commit_after>import cPickle
import gzip
import os

import numpy as np

from convnet.layers import *
from convnet.layers.convolutional_layer import _ConvolutionalLayer
from convnet.net import ConvNet
from convnet.utils import y_1d_to_3d

np.set_printoptions(precision=4, linewidth=120)

def transform_X(X_train):
    X = []
    for i in xrange(X_train.shape[0]):
        x = X_train[i]
        x = x.reshape((28, 28))[:, :, np.newaxis]
        X.append(x)
    return X

def transform_Y(Y_train):
    Y = []
    uniq = np.unique(Y_train)
    labels_count = uniq.size
    for i in xrange(Y_train.shape[0]):
        y = Y_train[i]
        y_arr = np.zeros(labels_count)
        index = np.where(uniq == y)[0][0]
        y_arr[index] = 1
        Y.append(y_1d_to_3d(y_arr))
    return Y

def get_examples(X_train, Y_train, labels=None, count=None):
    if isinstance(count, int):
        count = [count] * len(labels)
    assert len(labels) == len(count)

    X = None
    Y = None
    for i, label in enumerate(labels):
        indices = np.where(Y_train == label)[0]
        indices = indices[:count[i]]
        if X is None:
            X = X_train[indices]
        else:
            X = np.concatenate([X, X_train[indices]])

        if Y is None:
            Y = Y_train[indices]
        else:
            Y = np.concatenate([Y, Y_train[indices]])

    return X, Y

def shuffle_in_unison_inplace(a, b):
    assert len(a) == len(b)
    p = np.random.permutation(len(a))
    return a[p], b[p]

# noinspection PyPep8Naming
def mnist():
    script_path = os.path.dirname(os.path.abspath(__file__))
    f = gzip.open(os.path.join(script_path, './mnist.pkl.gz'), 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    f.close()

    X_train, Y_train = train_set

    X_train, Y_train = get_examples(X_train, Y_train, labels=np.arange(0, 10), count=5)
    X_train, Y_train = shuffle_in_unison_inplace(X_train, Y_train)
    X_train = transform_X(X_train)
    Y_train = transform_Y(Y_train)

    X_test, Y_test = test_set
    X_test, Y_test = get_examples(X_test, Y_test, labels=np.arange(0, 10), count=2)
    X_test, Y_test = shuffle_in_unison_inplace(X_test, Y_test)
    X_test = transform_X(X_test)
    Y_test = transform_Y(Y_test)

    net = ConvNet(iterations_count=30, batch_size=10, learning_rate=0.001, momentum=0.8, weight_decay=0.001)
    net.setup_layers([
        InputLayer(InputLayerSettings(in_shape=X_train[0].shape)),

        ConvolutionalLayer(ConvolutionalLayerSettings(filters_count=16, filter_size=5, stride=1, zero_padding=0)),
        ReluLayer(ReluLayerSettings(activation='max')),
        PoolingLayer(PoolingLayerSettings(filter_size=2, stride=2)),

        ConvolutionalLayer(ConvolutionalLayerSettings(filters_count=8, filter_size=5, stride=1, zero_padding=0)),
        ReluLayer(ReluLayerSettings(activation='max')),
        PoolingLayer(PoolingLayerSettings(filter_size=3, stride=3)),

        FullConnectedLayer(FullConnectedLayerSettings(neurons_count=Y_train[0].shape[-1], activation='sigmoid')),
    ])

    examples_count = 100000

    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    # net = ConvNet.load_net(os.path.join(BASE_DIR, './convnet1.pkl'))
    net.fit(X_train[:examples_count], Y_train[:examples_count])

    matched = 0
    for x, y in zip(X_train[:examples_count], Y_train[:examples_count]):
        h = net.predict(x)
        h_res = h.argmax()
        y_res = y.argmax()
        print("predicted = {}; max = {}".format(h, h.argmax()))
        print("real =      {}; max = {}".format(y, y.argmax()))
        print("\n")
        matched += int(h_res == y_res)

    print("Accuracy train {}/{}".format(matched, len(X_train[:examples_count])))

    matched = 0
    for x, y in zip(X_test[:examples_count], Y_test[:examples_count]):
        h = net.predict(x)
        h_res = h.argmax()
        y_res = y.argmax()
        print("predicted = {}; max = {}".format(h, h.argmax()))
        print("real =      {}; max = {}".format(y, y.argmax()))
        print("\n")
        matched += int(h_res == y_res)

    print("Accuracy test {}/{}".format(matched, len(X_test[:examples_count])))

    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    path = os.path.join(BASE_DIR, "./convnet1.pkl")
    net.dump_net(path)
    print("Dumped to {}".format(path))

if __name__ == "__main__":
    mnist()
    # net = ConvNet.load_net('/home/igor/Desktop/convnet.pkl')
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
cookiecutter.prompt
---------------------

Functions for prompting the user for project info.
"""

from __future__ import unicode_literals
import sys

import click

from .compat import iteritems, is_string
from jinja2.environment import Environment

def read_user_variable(var_name, default_value):
    """Prompt the user for the given variable and return the entered value
    or the given default.

    :param str var_name: Variable of the context to query the user
    :param default_value: Value that will be returned if no input happens
    """
    return click.prompt(var_name, default=default_value)

def read_user_yes_no(question, default_value):
    """Prompt the user to reply with 'yes' or 'no' (or equivalent values).

    Note:
      Possible choices are 'true', '1', 'yes', 'y' or 'false', '0', 'no', 'n'

    :param str question: Question to the user
    :param default_value: Value that will be returned if no input happens
    """
    return click.prompt(
        question,
        default=default_value,
        type=click.BOOL
    )

def read_response(prompt=''):
    """Prompt the user and return the entered value or an empty string.

    :param str prompt: Text to display to the user
    """
    # Please see http://click.pocoo.org/4/api/#click.prompt
    # default: use an empty string if no input happens
    # prompt_suffix: do not add a suffix to the prompt
    # show_default: the prompt is expected to mention the default itself
    return click.prompt(
        prompt,
        default='',
        prompt_suffix='',
        show_default=False,
    )

def prompt_for_config(context, no_input=False):
    """
    Prompts the user to enter new config, using context as a source for the
    field names and sample values.

    :param no_input: Prompt the user at command line for manual configuration?
    """
    cookiecutter_dict = {}
    env = Environment()

    for key, raw in iteritems(context['cookiecutter']):
        if key.startswith('_'):
            cookiecutter_dict[key] = raw
            continue

        raw = raw if is_string(raw) else str(raw)
        val = env.from_string(raw).render(cookiecutter=cookiecutter_dict)

        if not no_input:
            val = read_user_variable(key, val)

        cookiecutter_dict[key] = val
    return cookiecutter_dict
<commit_msg>Get rid of read_response in prompt.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
cookiecutter.prompt
---------------------

Functions for prompting the user for project info.
"""

from __future__ import unicode_literals
import sys

import click

from .compat import iteritems, is_string
from jinja2.environment import Environment

def read_user_variable(var_name, default_value):
    """Prompt the user for the given variable and return the entered value
    or the given default.

    :param str var_name: Variable of the context to query the user
    :param default_value: Value that will be returned if no input happens
    """
    # Please see http://click.pocoo.org/4/api/#click.prompt
    return click.prompt(var_name, default=default_value)

def read_user_yes_no(question, default_value):
    """Prompt the user to reply with 'yes' or 'no' (or equivalent values).

    Note:
      Possible choices are 'true', '1', 'yes', 'y' or 'false', '0', 'no', 'n'

    :param str question: Question to the user
    :param default_value: Value that will be returned if no input happens
    """
    # Please see http://click.pocoo.org/4/api/#click.prompt
    return click.prompt(
        question,
        default=default_value,
        type=click.BOOL
    )

def prompt_for_config(context, no_input=False):
    """
    Prompts the user to enter new config, using context as a source for the
    field names and sample values.

    :param no_input: Prompt the user at command line for manual configuration?
    """
    cookiecutter_dict = {}
    env = Environment()

    for key, raw in iteritems(context['cookiecutter']):
        if key.startswith('_'):
            cookiecutter_dict[key] = raw
            continue

        raw = raw if is_string(raw) else str(raw)
        val = env.from_string(raw).render(cookiecutter=cookiecutter_dict)

        if not no_input:
            val = read_user_variable(key, val)

        cookiecutter_dict[key] = val
    return cookiecutter_dict
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from collections import defaultdict
import matplotlib.pyplot as plt
import re
import os
import pandas as pd
import sys
import traceback

import bayeslite
import bayeslite.core
from bayeslite.loggers import BqlLogger, logged_query
from bayeslite.exception import BayesLiteException as BLE
import bdbcontrib
from bdbcontrib.population import Population
from bdbcontrib.population_method import population_method
from py_utils import helpsub

BqlRecipes = Population

@population_method(population=0)
@helpsub('bdbcontrib_pairplot', bdbcontrib.plot_utils.pairplot.__doc__)
def pairplot_vars(self, vars, plotfile=None, colorby=None, **kwargs):
  """Wrap bdbcontrib.plot_utils.pairplot to show the given columns.

  Specifies bdb, query with the given columns, and generator_name:
  bdbcontrib_pairplot
  """
  if len(vars) < 1:
    raise BLE(ValueError('Pairplot at least one variable.'))
  qvars = vars if colorby is None else set(vars + [colorby])
  query_columns = '''"%s"''' % '''", "'''.join(qvars)
  result = bdbcontrib.pairplot(
    self.bdb,
    self.q('''SELECT %s FROM %s''' % (query_columns, self.name)),
    generator_name=self.generator_name,
    colorby=colorby,
    **kwargs)
  if plotfile:
    self.logger.plot(plotfile, result)
  return result

@population_method(population=0)
def quick_explore_vars(self, vars, plotfile=None, nsimilar=20):
  """Show dependence probabilities and neighborhoods based on those.

  vars: list of strings
      At least two column names to look at dependence probabilities of,
      and to explore neighborhoods of.
  nsimilar: positive integer
      The size of the neighborhood to explore.
  """
  if len(vars) < 2:
    raise BLE(ValueError('Need to explore at least two variables.'))
  self.pairplot_vars(vars)
  query_columns = '''"%s"''' % '''", "'''.join(vars)
  deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
                       FROM PAIRWISE COLUMNS OF %s
                       FOR %s;''' % (self.generator_name, query_columns))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  self.logger.plot(plotfile, self.heatmap(deps))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  triangle = deps[deps['name0'] < deps['name1']]
  triangle = triangle.sort_values(ascending=False, by=['value'])
  self.logger.result("Pairwise dependence probability for: %s\n%s\n\n",
                     query_columns, triangle)

  for col in vars:
    neighborhood = self.query(
    '''ESTIMATE *, DEPENDENCE PROBABILITY WITH "%s"
       AS "Probability of Dependence with %s"
       FROM COLUMNS OF %s
       ORDER BY "Probability of Dependence with %s"
       DESC LIMIT %d;'''
       % (col, col, self.generator_name, col, nsimilar))
    neighbor_columns = ('''"%s"''' %
                        '''", "'''.join(neighborhood["name"].tolist()))
    deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
        FROM PAIRWISE COLUMNS OF %s
        FOR %s;''' % (self.generator_name, neighbor_columns))
    deps.columns = ['genid', 'name0', 'name1', 'value']
    self.logger.plot(plotfile, self.heatmap(deps))
    self.logger.result("Pairwise dependence probability of %s with its " +
                       "strongest dependents:\n%s\n\n", col, neighborhood)

@population_method(population=0)
def quick_similar_rows(self, identify_row_by, nsimilar=10):
  """Explore rows similar to the identified one.

  identify_row_by : dict
      Dictionary of column names to their values. These will be turned into
      a WHERE clause in BQL, and must identify one unique row.
  nsimilar : positive integer
      The number of similar rows to retrieve.
  """
  import hashlib
  table_name = 'tmptbl_' + hashlib.md5('\x00'.join(
      [repr(identify_row_by), str(self.status)])).hexdigest()
  column_name = 'similarity_to_' + "__".join(
      re.sub(r'\W', '_', str(val)) for val in identify_row_by.values())
  query_params = []
  query_columns = []
  for k, v in identify_row_by.iteritems():
    query_columns.append('''%s = ? ''' % bayeslite.bql_quote_name(k))
    query_params.append(v)
  query_attrs = ' and '.join(query_columns)

  with self.bdb.savepoint():
    row_exists = self.query('SELECT COUNT(*) FROM %s WHERE %s;' %
                            (self.name, query_attrs))
    if row_exists.ix[0][0] != 1:
      raise BLE(NotImplementedError(
          'identify_row_by found %d rows instead of exactly 1 in %s.' %
          (row_exists.ix[0][0], self.csv_path)))
    creation_query = ('''CREATE TEMP TABLE IF NOT EXISTS %s AS ESTIMATE *,
                         SIMILARITY TO (%s) AS %s FROM %%g LIMIT %d;''' %
                      (table_name, query_attrs, column_name, nsimilar))
    self.query(creation_query, query_params)
    result = self.query('''SELECT * FROM %s ORDER BY %s DESC;''' %
                        (table_name, column_name))
  return result

quickstart = bdbcontrib.quickstart
<commit_msg>Oops, missing import.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2016, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

from collections import defaultdict
import matplotlib.pyplot as plt
import re
import os
import pandas as pd
import sys
import traceback

import bayeslite
import bayeslite.core
from bayeslite.loggers import BqlLogger, logged_query
from bayeslite.exception import BayesLiteException as BLE
import bdbcontrib
import bdbcontrib.plot_utils # To helpsub from it.
from bdbcontrib.population import Population
from bdbcontrib.population_method import population_method
from py_utils import helpsub

BqlRecipes = Population

@population_method(population=0)
@helpsub('bdbcontrib_pairplot', bdbcontrib.plot_utils.pairplot.__doc__)
def pairplot_vars(self, vars, plotfile=None, colorby=None, **kwargs):
  """Wrap bdbcontrib.plot_utils.pairplot to show the given columns.

  Specifies bdb, query with the given columns, and generator_name:
  bdbcontrib_pairplot
  """
  if len(vars) < 1:
    raise BLE(ValueError('Pairplot at least one variable.'))
  qvars = vars if colorby is None else set(vars + [colorby])
  query_columns = '''"%s"''' % '''", "'''.join(qvars)
  result = bdbcontrib.pairplot(
    self.bdb,
    self.q('''SELECT %s FROM %s''' % (query_columns, self.name)),
    generator_name=self.generator_name,
    colorby=colorby,
    **kwargs)
  if plotfile:
    self.logger.plot(plotfile, result)
  return result

@population_method(population=0)
def quick_explore_vars(self, vars, plotfile=None, nsimilar=20):
  """Show dependence probabilities and neighborhoods based on those.

  vars: list of strings
      At least two column names to look at dependence probabilities of,
      and to explore neighborhoods of.
  nsimilar: positive integer
      The size of the neighborhood to explore.
  """
  if len(vars) < 2:
    raise BLE(ValueError('Need to explore at least two variables.'))
  self.pairplot_vars(vars)
  query_columns = '''"%s"''' % '''", "'''.join(vars)
  deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
                       FROM PAIRWISE COLUMNS OF %s
                       FOR %s;''' % (self.generator_name, query_columns))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  self.logger.plot(plotfile, self.heatmap(deps))
  deps.columns = ['genid', 'name0', 'name1', 'value']
  triangle = deps[deps['name0'] < deps['name1']]
  triangle = triangle.sort_values(ascending=False, by=['value'])
  self.logger.result("Pairwise dependence probability for: %s\n%s\n\n",
                     query_columns, triangle)

  for col in vars:
    neighborhood = self.query(
    '''ESTIMATE *, DEPENDENCE PROBABILITY WITH "%s"
       AS "Probability of Dependence with %s"
       FROM COLUMNS OF %s
       ORDER BY "Probability of Dependence with %s"
       DESC LIMIT %d;'''
       % (col, col, self.generator_name, col, nsimilar))
    neighbor_columns = ('''"%s"''' %
                        '''", "'''.join(neighborhood["name"].tolist()))
    deps = self.query('''ESTIMATE DEPENDENCE PROBABILITY
        FROM PAIRWISE COLUMNS OF %s
        FOR %s;''' % (self.generator_name, neighbor_columns))
    deps.columns = ['genid', 'name0', 'name1', 'value']
    self.logger.plot(plotfile, self.heatmap(deps))
    self.logger.result("Pairwise dependence probability of %s with its " +
                       "strongest dependents:\n%s\n\n", col, neighborhood)

@population_method(population=0)
def quick_similar_rows(self, identify_row_by, nsimilar=10):
  """Explore rows similar to the identified one.

  identify_row_by : dict
      Dictionary of column names to their values. These will be turned into
      a WHERE clause in BQL, and must identify one unique row.
  nsimilar : positive integer
      The number of similar rows to retrieve.
  """
  import hashlib
  table_name = 'tmptbl_' + hashlib.md5('\x00'.join(
      [repr(identify_row_by), str(self.status)])).hexdigest()
  column_name = 'similarity_to_' + "__".join(
      re.sub(r'\W', '_', str(val)) for val in identify_row_by.values())
  query_params = []
  query_columns = []
  for k, v in identify_row_by.iteritems():
    query_columns.append('''%s = ? ''' % bayeslite.bql_quote_name(k))
    query_params.append(v)
  query_attrs = ' and '.join(query_columns)

  with self.bdb.savepoint():
    row_exists = self.query('SELECT COUNT(*) FROM %s WHERE %s;' %
                            (self.name, query_attrs))
    if row_exists.ix[0][0] != 1:
      raise BLE(NotImplementedError(
          'identify_row_by found %d rows instead of exactly 1 in %s.' %
          (row_exists.ix[0][0], self.csv_path)))
    creation_query = ('''CREATE TEMP TABLE IF NOT EXISTS %s AS ESTIMATE *,
                         SIMILARITY TO (%s) AS %s FROM %%g LIMIT %d;''' %
                      (table_name, query_attrs, column_name, nsimilar))
    self.query(creation_query, query_params)
    result = self.query('''SELECT * FROM %s ORDER BY %s DESC;''' %
                        (table_name, column_name))
  return result

quickstart = bdbcontrib.quickstart
<|endoftext|>


<commit_before>import time

import django.core.exceptions
import commands
import random
from core.common.models import RequestStat, AllRequests
from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 600
    notcachedRemoteAddress = ['188.184.185.129']

    def __init__(self):
        pass

    def process_request(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = request.META.get('HTTP_REFERER'),
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0
        )

        # we limit number of requests per hour
        if ('json' in request.GET):
            if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
                startdate = timezone.now() - timedelta(hours=2)
                enddate = timezone.now()
                query = {'remote':x_forwarded_for,
                         'qtime__range': [startdate, enddate],
                         'id__gte':36207430}
                countRequest = []
                countRequest.extend(RequestStat.objects.filter(**query).values('remote').annotate(Count('remote')))
                if len(countRequest) > 0:
                    if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour:
                        reqs.is_rejected = 1
                        reqs.save()
                        return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), content_type='text/html')

        reqs.save()
        return None

<commit_msg>added hammercloud monitoring exception<commit_after>import time

import django.core.exceptions
import commands
import random
from core.common.models import RequestStat, AllRequests
from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 600
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72']

    def __init__(self):
        pass

    def process_request(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = request.META.get('HTTP_REFERER'),
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0
        )

        # we limit number of requests per hour
        if ('json' in request.GET):
            if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
                startdate = timezone.now() - timedelta(hours=2)
                enddate = timezone.now()
                query = {'remote':x_forwarded_for,
                         'qtime__range': [startdate, enddate],
                         'id__gte':36207430}
                countRequest = []
                countRequest.extend(RequestStat.objects.filter(**query).values('remote').annotate(Count('remote')))
                if len(countRequest) > 0:
                    if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour:
                        reqs.is_rejected = 1
                        reqs.save()
                        return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), content_type='text/html')

        reqs.save()
        return None

<|endoftext|>


<commit_before>import logging
import ipaddress
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta, datetime
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

_logger = logging.getLogger('bigpandamon')
# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 600
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86', '144.206.131.154',
                              '188.184.90.172' # J..h M......n request
                              ]
    blacklist = ['130.132.21.90','192.170.227.149']
    maxAllowedJSONRequstesParallel = 1
    maxAllowedSimultaneousRequestsToFileBrowser = 1
    listOfServerBackendNodesIPs = ['188.184.93.101', '188.184.116.46', '188.184.104.150',
                                   '188.184.84.149', '188.184.108.134', '188.184.108.131']

    restrictedIPs = ['137.138.77.2', # Incident on 13-01-2020 14:30:00
                     '188.185.76.164', #EI Machine
                     '147.156.116.63', #EI Machine
                     '147.156.116.43', #EI Machine
                     '147.156.116.44', #EI Machine
                     '147.156.116.81', #EI Machine
                     '147.156.116.83', #EI Machine
                     ]

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        try:
            _logger.info('Request: {}'.format(request.get_full_path()))
        except:
            _logger.exception('Can not get full path of request')

        # check if remote is a valid IP address
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for is not None:
            try:
                ip = ipaddress.ip_address(x_forwarded_for)
            except:
                _logger.warning('Provided HTTP_X_FORWARDED_FOR={} is not a correct IP address.'.format(x_forwarded_for))
                return HttpResponse(
                    json.dumps({'message': 'provided remote address is wrong'}),
                    status=400,
                    content_type='application/json')

        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''

        cursor = connection.cursor()
        dbtotalsess, dbactivesess = 0, 0
        # try:
        #     cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        #     rows = cursor.fetchall()
        #     for row in rows:
        #         dbactivesess = row[0]
        #         dbtotalsess = row[1]
        #         break
        # except:
        #     _logger.warning('Failed to get connections number from ATLAS_DBA')

        sqlRequest = "SELECT ATLAS_PANDABIGMON.ALL_REQUESTS_SEQ.NEXTVAL as my_req_token FROM dual;"
        cursor.execute(sqlRequest)
        requestToken = cursor.fetchall()
        requestToken = requestToken[0][0]
        cursor.close()

        reqs = AllRequests(
            id = requestToken,
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )
        reqs.save()

        # we limit number of requests per hour for a set of IPs
        try:
            useragent = request.META.get('HTTP_USER_AGENT')
        except:
            useragent = None

        if (not x_forwarded_for is None) and x_forwarded_for in self.restrictedIPs:
            _logger.info('[DDOS protection] got request from agent: {}'.format(useragent))
            countRestictedrequests = []
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            eiquery = {
                'qtime__range': [startdate, enddate],
                'remote':x_forwarded_for,
                'is_rejected': 0,
                'rtime': None,
            }
            countRestictedrequests.extend(
                AllRequests.objects.filter(**eiquery).values('remote').exclude(urlview='/grafana/').annotate(
                    Count('remote')))
            if len(countRestictedrequests) > 0 and 'remote__count' in countRestictedrequests[0]:
                _logger.info('[DDOS protection] found number of non rejected request for last minute: {}'.format(countRestictedrequests[0]['remote__count']))
                if countRestictedrequests[0]['remote__count'] > self.maxAllowedJSONRequstesParallel:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429,
                        content_type='application/json')

            response = self.get_response(request)
            reqs.rtime = datetime.utcnow()
            reqs.save()
            return response

        #We restrinct number of requets per hour
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
                # x_forwarded_for = '141.108.38.22'
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            query = {
                'remote':x_forwarded_for,
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))

            #Check against general number of request
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), status=429, content_type='application/json')

        #Check against number of unprocessed requests to filebrowser from ART subsystem
        #if 1==1:
        if request.path == '/filebrowser/' and x_forwarded_for in self.listOfServerBackendNodesIPs:
            startdate = datetime.utcnow() - timedelta(minutes=20)
            enddate = datetime.utcnow()
            query = {
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                'urlview': '/filebrowser/',
                'rtime': None,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').annotate(Count('urlview')))
            if len(countRequest) > 0:
                if countRequest[0]['urlview__count'] > self.maxAllowedSimultaneousRequestsToFileBrowser or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429, content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = datetime.utcnow()
        reqs.save()
        return response

<commit_msg>ddosprotection | relax protection against wrong remote address<commit_after>import logging
import ipaddress, re
from core.common.models import AllRequests

from django.utils import timezone
from datetime import timedelta, datetime
from django.db.models import Count, Sum
from django.http import HttpResponse
import json
import psutil
from django.db import connection

_logger = logging.getLogger('bigpandamon')
# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 600
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72', '188.184.116.46', '188.184.28.86', '144.206.131.154',
                              '188.184.90.172' # J..h M......n request
                              ]
    blacklist = ['130.132.21.90','192.170.227.149']
    maxAllowedJSONRequstesParallel = 1
    maxAllowedSimultaneousRequestsToFileBrowser = 1
    listOfServerBackendNodesIPs = ['188.184.93.101', '188.184.116.46', '188.184.104.150',
                                   '188.184.84.149', '188.184.108.134', '188.184.108.131']

    restrictedIPs = ['137.138.77.2', # Incident on 13-01-2020 14:30:00
                     '188.185.76.164', #EI Machine
                     '147.156.116.63', #EI Machine
                     '147.156.116.43', #EI Machine
                     '147.156.116.44', #EI Machine
                     '147.156.116.81', #EI Machine
                     '147.156.116.83', #EI Machine
                     ]

    def __init__(self, get_response):
        self.get_response = get_response

    def __call__(self, request):
        try:
            _logger.info('Request: {}'.format(request.get_full_path()))
        except:
            _logger.exception('Can not get full path of request')

        # check if remote is a valid IP address
        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
        if x_forwarded_for is not None:
            try:
                x_forwarded_for = re.findall(r'[0-9]+(?:\.[0-9]+){3}', x_forwarded_for)[0]
                ip = ipaddress.ip_address(x_forwarded_for)
            except:
                _logger.warning('Provided HTTP_X_FORWARDED_FOR={} is not a correct IP address.'.format(x_forwarded_for))
                return HttpResponse(
                    json.dumps({'message': 'provided remote address is wrong'}),
                    status=400,
                    content_type='application/json')

        try:
            x_referer = request.META.get('HTTP_REFERER')
        except:
            x_referer = ''

        cursor = connection.cursor()
        dbtotalsess, dbactivesess = 0, 0
        # try:
        #     cursor.execute("SELECT SUM(NUM_ACTIVE_SESS), SUM(NUM_SESS) FROM ATLAS_DBA.COUNT_PANDAMON_SESSIONS")
        #     rows = cursor.fetchall()
        #     for row in rows:
        #         dbactivesess = row[0]
        #         dbtotalsess = row[1]
        #         break
        # except:
        #     _logger.warning('Failed to get connections number from ATLAS_DBA')

        sqlRequest = "SELECT ATLAS_PANDABIGMON.ALL_REQUESTS_SEQ.NEXTVAL as my_req_token FROM dual;"
        cursor.execute(sqlRequest)
        requestToken = cursor.fetchall()
        requestToken = requestToken[0][0]
        cursor.close()

        reqs = AllRequests(
            id = requestToken,
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = x_referer[:3900] if x_referer and len(x_referer) > 3900 else x_referer,
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0,
            load=psutil.cpu_percent(interval=1),
            mem = psutil.virtual_memory().percent,
            dbtotalsess = dbtotalsess,
            dbactivesess = dbactivesess
        )
        reqs.save()

        # we limit number of requests per hour for a set of IPs
        try:
            useragent = request.META.get('HTTP_USER_AGENT')
        except:
            useragent = None

        if (not x_forwarded_for is None) and x_forwarded_for in self.restrictedIPs:
            _logger.info('[DDOS protection] got request from agent: {}'.format(useragent))
            countRestictedrequests = []
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            eiquery = {
                'qtime__range': [startdate, enddate],
                'remote':x_forwarded_for,
                'is_rejected': 0,
                'rtime': None,
            }
            countRestictedrequests.extend(
                AllRequests.objects.filter(**eiquery).values('remote').exclude(urlview='/grafana/').annotate(
                    Count('remote')))
            if len(countRestictedrequests) > 0 and 'remote__count' in countRestictedrequests[0]:
                _logger.info('[DDOS protection] found number of non rejected request for last minute: {}'.format(countRestictedrequests[0]['remote__count']))
                if countRestictedrequests[0]['remote__count'] > self.maxAllowedJSONRequstesParallel:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429,
                        content_type='application/json')

            response = self.get_response(request)
            reqs.rtime = datetime.utcnow()
            reqs.save()
            return response

        #We restrinct number of requets per hour
        if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
                # x_forwarded_for = '141.108.38.22'
            startdate = datetime.utcnow() - timedelta(hours=1)
            enddate = datetime.utcnow()
            query = {
                'remote':x_forwarded_for,
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').exclude(urlview='/grafana/').annotate(Count('remote')))

            #Check against general number of request
            if len(countRequest) > 0:
                if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), status=429, content_type='application/json')

        #Check against number of unprocessed requests to filebrowser from ART subsystem
        #if 1==1:
        if request.path == '/filebrowser/' and x_forwarded_for in self.listOfServerBackendNodesIPs:
            startdate = datetime.utcnow() - timedelta(minutes=20)
            enddate = datetime.utcnow()
            query = {
                'qtime__range': [startdate, enddate],
                'is_rejected': 0,
                'urlview': '/filebrowser/',
                'rtime': None,
                 }
            countRequest = []
            countRequest.extend(AllRequests.objects.filter(**query).values('remote').annotate(Count('urlview')))
            if len(countRequest) > 0:
                if countRequest[0]['urlview__count'] > self.maxAllowedSimultaneousRequestsToFileBrowser or x_forwarded_for in self.blacklist:
                    reqs.is_rejected = 1
                    reqs.save()
                    return HttpResponse(
                        json.dumps({'message': 'your IP produces too many requests per hour, please try later'}),
                        status=429, content_type='application/json')

        response = self.get_response(request)
        reqs.rtime = datetime.utcnow()
        reqs.save()
        return response

<|endoftext|>


<commit_before>import time

import django.core.exceptions
import commands
import random
from core.common.models import RequestStat, AllRequests
from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 300
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72']
    clients = ['curl']
    isForbidden = False

    def __init__(self):
        pass

    def process_request(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = request.META.get('HTTP_REFERER'),
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0
        )

        for client in self.clients:
            if client in request.META.get('HTTP_USER_AGENT') and 'json' not in request.GET:
                self.isForbidden = True
                self.maxAllowedJSONRequstesPerHour = 200

        # we limit number of requests per hour
        if ('json' in request.GET) or self.isForbidden:
            if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
                startdate = timezone.now() - timedelta(hours=2)
                enddate = timezone.now()
                query = {'remote':x_forwarded_for,
                         'qtime__range': [startdate, enddate],
                         'id__gte':36207430}
                countRequest = []
                countRequest.extend(RequestStat.objects.filter(**query).values('remote').annotate(Count('remote')))
                if len(countRequest) > 0:
                    if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour:
                        reqs.is_rejected = 1
                        reqs.save()
                        return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), content_type='text/html')

        reqs.save()
        return None

<commit_msg>Hot fix<commit_after>import time

import django.core.exceptions
import commands
import random
from core.common.models import RequestStat, AllRequests
from django.utils import timezone
from datetime import timedelta
from django.db.models import Count, Sum
from django.http import HttpResponse
import json

# We postpone JSON requests is server is overloaded
# Done for protection from bunch of requests from JSON

class DDOSMiddleware(object):

    sleepInterval = 5 #sec
    maxAllowedJSONRequstesPerHour = 300
    notcachedRemoteAddress = ['188.184.185.129', '188.185.80.72']
    clients = ['curl']
    isForbidden = False

    def __init__(self):
        pass

    def process_request(self, request):

        x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')

        reqs = AllRequests(
            server = request.META.get('HTTP_HOST'),
            remote = x_forwarded_for,
            qtime = timezone.now(),
            url= request.META.get('QUERY_STRING'),
            urlview = request.path,
            referrer = request.META.get('HTTP_REFERER'),
            useragent = request.META.get('HTTP_USER_AGENT'),
            is_rejected = 0
        )
        try:
            for client in self.clients:
                if client in request.META.get('HTTP_USER_AGENT') and ('json' not in request.GET):
                    self.isForbidden = True
                    self.maxAllowedJSONRequstesPerHour = 200
        except:
            pass

        # we limit number of requests per hour
        if ('json' in request.GET) or self.isForbidden:
            if (not x_forwarded_for is None) and x_forwarded_for not in self.notcachedRemoteAddress:
#                x_forwarded_for = '141.108.38.22'
                startdate = timezone.now() - timedelta(hours=2)
                enddate = timezone.now()
                query = {'remote':x_forwarded_for,
                         'qtime__range': [startdate, enddate],
                         'id__gte':36207430}
                countRequest = []
                countRequest.extend(RequestStat.objects.filter(**query).values('remote').annotate(Count('remote')))
                if len(countRequest) > 0:
                    if countRequest[0]['remote__count'] > self.maxAllowedJSONRequstesPerHour:
                        reqs.is_rejected = 1
                        reqs.save()
                        return HttpResponse(json.dumps({'message':'your IP produces too many requests per hour, please try later'}), content_type='text/html')

        reqs.save()
        return None

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import unittest
import sys
sys.path.append("./src/")
import pdb
#  pdb.set_trace();

import scipy.io

import logging
logger = logging.getLogger(__name__)

import argparse
import numpy as np

def vesselSegmentation(data, segmentation, threshold=1185, dataFiltering=False, nObj=1):
    """ Volumetric vessel segmentation from liver.
    data: CT (or MRI) 3D data
    segmentation: labeled image with same size as data where label:
    1 mean liver pixels,
    -1 interesting tissuse (bones)
    0 othrewise
    """
#   Funkce pracuje z počátku na principu jednoduchého prahování. Nalezne se
#   největší souvislý objekt nad stanoveným prahem, Průběžně bude segmentace
#   zpřesňována. Bude nutné hledat cévy, které se spojují mimo játra, ale
#   ignorovat žebra.
#   Proměnné threshold, dataFiltering a nObj se postupně pokusíme eliminovat a
#   navrhnout je automaticky.
#   threshold: ručně určený práh
#   dataFiltering: označuje, jestli budou data filtrována uvnitř funkce, nebo
#   již vstupují filtovaná. False znamená, že vstupují filtrovaná.
#   nObj: označuje kolik největších objektů budeme hledat
    if data.shape != segmentation.shape:
        raise Exception('Input size error','Shape if input data and segmentation must be same')

    return segmentation

# --------------------------tests-----------------------------
class Tests(unittest.TestCase):
    def test_t(self):
        pass
    def setUp(self):
        """ Nastavení společných proměnných pro testy  """
        datashape = [220,115,30]
        self.datashape = datashape
        self.rnddata = np.random.rand(datashape[0], datashape[1], datashape[2])
        self.segmcube = np.zeros(datashape)
        self.segmcube[130:190, 40:90,5:15] = 1

    def test_same_size_input_and_output(self):
        """Funkce testuje stejnost vstupních a výstupních dat"""
        outputdata = vesselSegmentation(self.rnddata,self.segmcube)
        self.assertEqual(outputdata.shape, self.rnddata.shape)

#
#    def test_different_data_and_segmentation_size(self):
#        """ Funkce ověřuje vyhození výjimky při různém velikosti vstpních
#        dat a segmentace """
#        pdb.set_trace();
#        self.assertRaises(Exception, vesselSegmentation, (self.rnddata, self.segmcube[2:,:,:]) )
#

# --------------------------main------------------------------
if __name__ == "__main__":
    logger = logging.getLogger()
    logger.setLevel(logging.WARNING)
# při vývoji si necháme vypisovat všechny hlášky
    logger.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
#   output configureation
    #logging.basicConfig(format='%(asctime)s %(message)s')
    logging.basicConfig(format='%(message)s')

    formatter = logging.Formatter("%(levelname)-5s [%(module)s:%(funcName)s:%(lineno)d] %(message)s")
    # add formatter to ch
    ch.setFormatter(formatter)

    logger.addHandler(ch)

    # input parser
    parser = argparse.ArgumentParser(description='Segment vessels from liver')
    parser.add_argument('filename', type=str,
            help='*.mat file with variables "data", "segmentation" and "threshod"')
    parser.add_argument('-d', '--debug', action='store_true',
            help='run in debug mode')
    parser.add_argument('-t', '--tests', action='store_true',
            help='run unittest')
    parser.add_argument('-o', '--outputfile', type=str,
        default='output.mat',help='output file name')
    args = parser.parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)

    if args.tests:
        # hack for use argparse and unittest in one module
        sys.argv[1:]=[]
        unittest.main()

#   load all
    mat = scipy.io.loadmat(args.filename)
    logger.debug( mat.keys())

    # load specific variable
    matthreshold = scipy.io.loadmat(args.filename, variable_names=['threshold'])

    logger.debug(matthreshold['threshold'][0][0])

    # zastavení chodu programu pro potřeby debugu,
    # ovládá se klávesou's','c',...
    # zakomentovat
    pdb.set_trace();

    output = vesselSegmentation(mat['data'],mat['segmentation'], mat['threshold'] )
    scipy.io.savemat(args.outputfile,{'vesselSegm':output})

<commit_msg>Voxelsize<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import unittest
import sys
sys.path.append("./src/")
import pdb
#  pdb.set_trace();

import scipy.io

import logging
logger = logging.getLogger(__name__)

import argparse
import numpy as np

def vesselSegmentation(data, segmentation, threshold=1185, dataFiltering=False, nObj=1, voxelsizemm=[[1],[1],[1]]):
    """ Volumetric vessel segmentation from liver.
    data: CT (or MRI) 3D data
    segmentation: labeled image with same size as data where label:
    1 mean liver pixels,
    -1 interesting tissuse (bones)
    0 othrewise
    """
#   Funkce pracuje z počátku na principu jednoduchého prahování. Nalezne se
#   největší souvislý objekt nad stanoveným prahem, Průběžně bude segmentace
#   zpřesňována. Bude nutné hledat cévy, které se spojují mimo játra, ale
#   ignorovat žebra.
#   Proměnné threshold, dataFiltering a nObj se postupně pokusíme eliminovat a
#   navrhnout je automaticky.
#   threshold: ručně určený práh
#   dataFiltering: označuje, jestli budou data filtrována uvnitř funkce, nebo
#   již vstupují filtovaná. False znamená, že vstupují filtrovaná.
#   nObj: označuje kolik největších objektů budeme hledat
    if data.shape != segmentation.shape:
        raise Exception('Input size error','Shape if input data and segmentation must be same')

    return segmentation

# --------------------------tests-----------------------------
class Tests(unittest.TestCase):
    def test_t(self):
        pass
    def setUp(self):
        """ Nastavení společných proměnných pro testy  """
        datashape = [220,115,30]
        self.datashape = datashape
        self.rnddata = np.random.rand(datashape[0], datashape[1], datashape[2])
        self.segmcube = np.zeros(datashape)
        self.segmcube[130:190, 40:90,5:15] = 1

    def test_same_size_input_and_output(self):
        """Funkce testuje stejnost vstupních a výstupních dat"""
        outputdata = vesselSegmentation(self.rnddata,self.segmcube)
        self.assertEqual(outputdata.shape, self.rnddata.shape)

#
#    def test_different_data_and_segmentation_size(self):
#        """ Funkce ověřuje vyhození výjimky při různém velikosti vstpních
#        dat a segmentace """
#        pdb.set_trace();
#        self.assertRaises(Exception, vesselSegmentation, (self.rnddata, self.segmcube[2:,:,:]) )
#

# --------------------------main------------------------------
if __name__ == "__main__":
    logger = logging.getLogger()
    logger.setLevel(logging.WARNING)
# při vývoji si necháme vypisovat všechny hlášky
    logger.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
#   output configureation
    #logging.basicConfig(format='%(asctime)s %(message)s')
    logging.basicConfig(format='%(message)s')

    formatter = logging.Formatter("%(levelname)-5s [%(module)s:%(funcName)s:%(lineno)d] %(message)s")
    # add formatter to ch
    ch.setFormatter(formatter)

    logger.addHandler(ch)

    # input parser
    parser = argparse.ArgumentParser(description='Segment vessels from liver')
    parser.add_argument('filename', type=str,
            help='*.mat file with variables "data", "segmentation" and "threshod"')
    parser.add_argument('-d', '--debug', action='store_true',
            help='run in debug mode')
    parser.add_argument('-t', '--tests', action='store_true',
            help='run unittest')
    parser.add_argument('-o', '--outputfile', type=str,
        default='output.mat',help='output file name')
    args = parser.parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)

    if args.tests:
        # hack for use argparse and unittest in one module
        sys.argv[1:]=[]
        unittest.main()

#   load all
    mat = scipy.io.loadmat(args.filename)
    logger.debug( mat.keys())

    # load specific variable
    matthreshold = scipy.io.loadmat(args.filename, variable_names=['threshold'])

    logger.debug(matthreshold['threshold'][0][0])

    # zastavení chodu programu pro potřeby debugu,
    # ovládá se klávesou's','c',...
    # zakomentovat
    pdb.set_trace();

    # zde by byl prostor pro ruční (interaktivní) zvolení prahu z klávesnice
    #tě ebo jinak

    output = vesselSegmentation(mat['data'],mat['segmentation'], mat['threshold'], mat['voxelsizemm'] )
    scipy.io.savemat(args.outputfile,{'vesselSegm':output})

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
#  $Id$
#  Project:  RX Cadre Data Visualization
#  Purpose:  Import csv data into tables
#  Author:   Kyle Shannon <kyle@pobox.com>
#            Kegan Rabil <krabil@yourdatasmarter.com>
#
###############################################################################
#
#  This is free and unencumbered software released into the public domain.
#
#  Anyone is free to copy, modify, publish, use, compile, sell, or
#  distribute this software, either in source code form or as a compiled
#  binary, for any purpose, commercial or non-commercial, and by any
#  means.
#
#  In jurisdictions that recognize copyright laws, the author or authors
#  of this software dedicate any and all copyright interest in the
#  software to the public domain. We make this dedication for the benefit
#  of the public at large and to the detriment of our heirs and
#  successors. We intend this dedication to be an overt act of
#  relinquishment in perpetuity of all present and future rights to this
#  software under copyright law.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
#  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
#  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#  OTHER DEALINGS IN THE SOFTWARE.
#
#  For more information, please refer to <http://unlicense.org/>
#
###############################################################################
"""
Module for importing spatial time-series data into a searchable format for
subsetting in space and time.  Typical import format would be csv files.  The
values are imported into a sqlite3 database.  'Meta' tables store information
about what to display.
"""

import csv
import os
import sqlite3
import json

class RxCadreIOError(Exception):pass
class RxCadreInvalidDbError(Exception):pass

class RxCadre:
    """
    Main interface for RX Cadre data.
    """

    def init_new_db(self, filename):
        """
        Create a new, empty database with appropriate metatables.  If the file
        previously exists, we fail before connecting using sqlite.  It must be
        a new file.
        """
        if os.path.exists(filename):
            raise RxCadreIOError("Database file already exists")
        db = sqlite3.connect(filename)
        cursor = db.cursor()
        sql = """CREATE TABLE plot_location(plot_id TEXT NOT NULL PRIMARY KEY,
                                            geometry TEXT, plot_type TEXT)"""
        cursor.execute(sql)
        sql = """CREATE TABLE event(project_name TEXT,
                                    event_name TEXT NOT NULL,
                                    event_start TEXT NOT NULL,
                                    event_end TEXT NOT NULL,
                                    PRIMARY KEY(project_name, event_name))"""
        cursor.execute(sql)
        sql = """CREATE TABLE obs_table(obs_table_name TEXT NOT NULL,
                                        geometry_column TEXT NOT NULL,
                                        obs_cols TEXT NOT NULL,
                                        obs_col_names TEXT)"""
        cursor.execute(sql)
        db.commit()
        valid = self.check_valid_db(db)
        db.close()
        if not valid:
            e = "Failed to create a valid database."
            raise RxCadreInvalidDbError(e)

    def check_valid_db(self, db):
        """
        Check the schema of an existing db.  This involves checking the
        metatables, and checking to make sure tables registered in obs_tables
        exist.
        """

        cursor = db.cursor()
        required_tables = set([u'plot_location', u'event', u'obs_table'])
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
        table_names = [t[0] for t in cursor.fetchall()]
        if not set(table_names) <= required_tables:
            e = "Database does not contain the required tables, missing:"
            e += ",".join(required_tables.difference(table_names))
            return False
        cursor.execute("select obs_table_name from obs_table")
        obs_names = cursor.fetchall()
        for name in obs_names:
            if n not in table_names:
                e = "Database is invalid, missing table: %s" % n
                return False
        return True

    def import_data(self, db, file_path, import_fx=None):
        """
        Import a new data set into the database.  An obs table should be
        created and registered with obs_tables.  It also should populate the
        plot_location data if possible.  The import_fx is an optional function
        that does the importing.
        """
        con = sqlite3.connect(db)

        data_file = open(file_path,"r")
        header = data_file.readline().split(",")

        cursor = con.cursor()
        hold_name = header[0]
        cursor.execute("CREAT TABLE " + hold_name + "(" + header[0] + ")")
        for i in range(1,len(header)):
            cursor.execute("ALTER TABLE " + hold_name + " ADD COLUMN " + json.dumps(header[i]))

        #The line below is what I'm trying to fix
        cursor.execute("insert into hold_name values (?)", data_file.readline().split(","))
        cursor.execute("insert into obs_table values(header[0],header[1],header[2],header[3])")

<commit_msg>Updated import_db function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
#
#  $Id$
#  Project:  RX Cadre Data Visualization
#  Purpose:  Import csv data into tables
#  Author:   Kyle Shannon <kyle@pobox.com>
#            Kegan Rabil <krabil@yourdatasmarter.com>
#
###############################################################################
#
#  This is free and unencumbered software released into the public domain.
#
#  Anyone is free to copy, modify, publish, use, compile, sell, or
#  distribute this software, either in source code form or as a compiled
#  binary, for any purpose, commercial or non-commercial, and by any
#  means.
#
#  In jurisdictions that recognize copyright laws, the author or authors
#  of this software dedicate any and all copyright interest in the
#  software to the public domain. We make this dedication for the benefit
#  of the public at large and to the detriment of our heirs and
#  successors. We intend this dedication to be an overt act of
#  relinquishment in perpetuity of all present and future rights to this
#  software under copyright law.
#
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
#  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
#  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
#  OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
#  ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#  OTHER DEALINGS IN THE SOFTWARE.
#
#  For more information, please refer to <http://unlicense.org/>
#
###############################################################################
"""
Module for importing spatial time-series data into a searchable format for
subsetting in space and time.  Typical import format would be csv files.  The
values are imported into a sqlite3 database.  'Meta' tables store information
about what to display.
"""

import csv
import os
import sqlite3
import json

class RxCadreIOError(Exception):pass
class RxCadreInvalidDbError(Exception):pass

class RxCadre:
    """
    Main interface for RX Cadre data.
    """

    def init_new_db(self, filename):
        """
        Create a new, empty database with appropriate metatables.  If the file
        previously exists, we fail before connecting using sqlite.  It must be
        a new file.
        """
        if os.path.exists(filename):
            raise RxCadreIOError("Database file already exists")
        db = sqlite3.connect(filename)
        cursor = db.cursor()
        sql = """CREATE TABLE plot_location(plot_id TEXT NOT NULL PRIMARY KEY,
                                            geometry TEXT, plot_type TEXT)"""
        cursor.execute(sql)
        sql = """CREATE TABLE event(project_name TEXT,
                                    event_name TEXT NOT NULL,
                                    event_start TEXT NOT NULL,
                                    event_end TEXT NOT NULL,
                                    PRIMARY KEY(project_name, event_name))"""
        cursor.execute(sql)
        sql = """CREATE TABLE obs_table(obs_table_name TEXT NOT NULL,
                                        geometry_column TEXT NOT NULL,
                                        obs_cols TEXT NOT NULL,
                                        obs_col_names TEXT)"""
        cursor.execute(sql)
        db.commit()
        valid = self.check_valid_db(db)
        db.close()
        if not valid:
            e = "Failed to create a valid database."
            raise RxCadreInvalidDbError(e)

    def check_valid_db(self, db):
        """
        Check the schema of an existing db.  This involves checking the
        metatables, and checking to make sure tables registered in obs_tables
        exist.
        """

        cursor = db.cursor()
        required_tables = set([u'plot_location', u'event', u'obs_table'])
        cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
        table_names = [t[0] for t in cursor.fetchall()]
        if not set(table_names) <= required_tables:
            e = "Database does not contain the required tables, missing:"
            e += ",".join(required_tables.difference(table_names))
            return False
        cursor.execute("select obs_table_name from obs_table")
        obs_names = cursor.fetchall()
        for name in obs_names:
            if n not in table_names:
                e = "Database is invalid, missing table: %s" % n
                return False
        return True

    def import_data(self, db, file_path, import_fx=None):
        """
        Import a new data set into the database.  An obs table should be
        created and registered with obs_tables.  It also should populate the
        plot_location data if possible.  The import_fx is an optional function
        that does the importing.
        """
        con = sqlite3.connect(db)

        data_file = open(file_path,"r")
        header = data_file.readline().split(",")

        sql = """CREATE TABLE my_table("""
        for i, h in enumerate(header.split(",")):
            sql += h + ' text'
            if(i < len(header)):
                sql += ','
        sql += ');'

        cursor = con.cursor()
        hold_name = header[0]

        cursor.execute(sql)

        qs = "("+ (len(header)-1)* "?," +"?)"
        insrt = "insert into " + hold_name+ " values "
        insrt += qs
        con.text_factory = str
        n = 0
        while (data_file.readline() != None):
            line = data_file.readline()
            n = n+1
            if len(line.split(",")) < len(header):
                print 'Error at line ' + str(n)
                break
            else:
                cursor.execute(insrt, line.split(","))

<|endoftext|>


<commit_before>import string
import re

from atom import Symbol
from atom import String
from number import Number, Integral, LongInt, Float
from lisp import Lisp

DELIM = string.whitespace + Lisp.SPECIAL

class Scanner:
    def __init__(self, str=None):
       print("Initializing the scanner...")
       self.raw_source = str
       self.index = 0
       self.length = 0
       self.sexpr = []

       if str:
           self.sexpr = self.get_sexpr()

    def get_sexpr(self, source=None):
        if source:
            self.raw_source = source
            self.length = len(self.raw_source)
            self.index = 0

        token = self.get_token()

        if token == ')':
            raise ValueError("Unexpected right paren")
        elif token == '(':
            token = self.get_token()

            while token != ')':
                if token == '(':
                    # Start parsing again.
                    self.index = self.index - 1
                elif token == None:
                    raise ValueError("Invalid end of expression")

                token = self.get_token()

        return token

    def get_token(self):
        if self.index >= self.length:
            return None

        # Kill whitespace
        while self.index < self.length and self.raw_source[self.index] in string.whitespace:
            self.index = self.index + 1

        # Check if we had a string of whitespace
        if self.index == self.length:
            return None

        if self.raw_source[self.index] in Lisp.SPECIAL:
            self.index = self.index + 1

            return self.raw_source[self.index - 1]
        else:
            token_str = ""

            # Build the token string
            while self.index < self.length:
                token_str = token_str + self.raw_source[self.index]
                self.index = self.index + 1

            if Integral.REGEX.match(token_str):
                return Integral(int(token_str))
            elif Float.REGEX.match(token_str):
                return Float(float(token_str))
            elif LongInt.REGEX.match(token_str):
                return LongInt(int(token_str))
            else:
                return Symbol(token_str)

        return None
<commit_msg>Recognize a list and return it as an instance<commit_after>import string
import re

from atom import Symbol
from atom import String
from number import Number, Integral, LongInt, Float
from lisp import Lisp

DELIM = string.whitespace + Lisp.SPECIAL

class Scanner:
    def __init__(self, str=None):
       print("Initializing the scanner...")
       self.raw_source = str
       self.index = 0
       self.length = 0
       self.sexpr = []

       if str:
           self.sexpr = self.get_sexpr()

    def get_sexpr(self, source=None):
        if source:
            self.raw_source = source
            self.length = len(self.raw_source)
            self.index = 0

        token = self.get_token()
        expr = None

        if token == ')':
            raise ValueError("Unexpected right paren")
        elif token == ')':
            expr = []
            token = self.get_token()

            while token != ')':
                if token == '(':
                    # Start parsing again.
                    self.index = self.index - 1
                elif token == None:
                    raise ValueError("Invalid end of expression")
                else:
                    expr.append(token)

                token = self.get_token()
            return List(expr)
        else:
            return token

    def get_token(self):
        if self.index >= self.length:
            return None

        # Kill whitespace
        while self.index < self.length and self.raw_source[self.index] in string.whitespace:
            self.index = self.index + 1

        # Check if we had a string of whitespace
        if self.index == self.length:
            return None

        if self.raw_source[self.index] in Lisp.SPECIAL:
            self.index = self.index + 1

            return self.raw_source[self.index - 1]
        else:
            token_str = ""

            # Build the token string
            while self.index < self.length:
                token_str = token_str + self.raw_source[self.index]
                self.index = self.index + 1

            if Integral.REGEX.match(token_str):
                return Integral(int(token_str))
            elif Float.REGEX.match(token_str):
                return Float(float(token_str))
            elif LongInt.REGEX.match(token_str):
                return LongInt(int(token_str))
            else:
                return Symbol(token_str)

        return None
<|endoftext|>


<commit_before>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import base64
import logging

from conary.lib import cfg as cny_cfg
from conary.lib import cfgtypes
from rmake3.lib import chutney
from rmake3.lib.jabberlink import message
from rmake3.lib.jabberlink.handlers import link

from rmake3.core import types
from rmake3.core import plug_dispatcher

from rmake3.worker import plug_worker

from twisted.web import client
from twisted.web import server
from twisted.web import resource
from twisted.internet import ssl
from twisted.internet import reactor

NS = 'http://rpath.com/permanent/xmpp/repeater-1.0'
log = logging.getLogger(__name__)

class RestForwardingConfig(cny_cfg.ConfigFile):
    # Launcher
    key                     = (cfgtypes.CfgString, None)
    cert                    = (cfgtypes.CfgString, None)
    httpPort                = (cfgtypes.CfgInt, None)
    httpsPort               = (cfgtypes.CfgInt, None)

    # Dispatcher
    repeaterTarget          = (cfgtypes.CfgString, None)

class RestForwardingPlugin(plug_dispatcher.DispatcherPlugin,
    plug_worker.LauncherPlugin):

    def launcher_post_setup(self, launcher):
        """ The Sputnik end of the rMake topology """
        endpoint = EndPoint(launcher.bus)

        cfg = self.populateConfigFromOptions(RestForwardingConfig())
        if cfg.httpPort:
            reactor.listenTCP(cfg.httpPort,
                    server.Site(resource.IResource(endpoint)))
        if cfg.httpsPort:
            reactor.listenSSL(cfg.httpsPort,
                    server.Site(resource.IResource(endpoint)),
                    ssl.DefaultOpenSSLContextFactory(cfg.key, cfg.cert))

    def dispatcher_post_setup(self, dispatcher):
        """ The rBuilder end of the rMake topology """

        cfg = self.populateConfigFromOptions(RestForwardingConfig())
        if cfg.repeaterTarget:
            dispatcher.bus.link.addMessageHandler(
                    RepeaterMessageHandler(cfg.repeaterTarget,
                        dispatcher.workers))

class RepeaterMessageHandler(message.MessageHandler):
    namespace = NS
    XHeader = 'X-rPath-Management-Zone'
    XRepeaterHeader = 'X-rPath-Repeater'

    def __init__(self, host, workers):
        self.host = host
        self.workers = workers

    def getManagementZone(self, neighbor):
        jid = link.toJID(neighbor.jid.full())
        worker = self.workers.get(jid)
        if worker is None:
            return None
        zones = self.getWorkerZones(worker)
        if not zones:
            return None
        # We only care about one zone for now
        return base64.b64encode(zones[0])

    @classmethod
    def getWorkerZones(cls, worker):
        # XXX it would be nice if this was a property of the worker
        zoneNames = [ x.zoneName for x in worker.caps
            if isinstance(x, types.ZoneCapability) ]
        return zoneNames

    def onMessage(self, neighbor, msg):
        reqDict = chutney.loads(msg.payload)
        method = reqDict['method']
        url = reqDict['url']
        body = reqDict['body']
        rawHeaders = reqDict['headers']
        headers = client.Headers(rawHeaders)
        headers.removeHeader(self.XHeader)
        managementZone = self.getManagementZone(neighbor)
        if managementZone is not None:
            headers.addRawHeader(self.XHeader, managementZone)
        # This header flags a request as _not_ being originated from localhost
        # Some of the management interfaces require localhost access, but
        # everything forwarded through the repeater looks like it's
        # originating from localhost, unless this header is present
        headers.addRawHeader(self.XRepeaterHeader, 'remote')
        # XXX this is where multi-valued headers go down the drain
        headers = dict((k.lower(), v[-1])
            for (k, v) in headers.getAllRawHeaders())

        fact = HTTPClientFactory(url, method=method.upper(), postdata=body,
            headers=headers)
        @fact.deferred.addCallback
        def processResult(args):
            (status, statusMessage, headers, body) = args
            reply = dict(
                status = int(status),
                message = statusMessage,
                headers = headers,
                body = body,
            )
            neighbor.send(message.Message(self.namespace, chutney.dumps(reply),
                                           in_reply_to=msg))
            return args

        @fact.deferred.addErrback
        def processError(error):
            # There was an error talking upstream. Return a 502 Bad Gateway
            # XXX Ideally we want to explain what the original error was
            reply = dict(
                    status=502,
                    message='Bad Gateway',
                    headers={},
                    body='',
                    )
            neighbor.send(message.Message(self.namespace, chutney.dumps(reply),
                                           in_reply_to=msg))

        reactor.connectTCP(self.host, 80, fact)

class EndPoint(resource.Resource):
    isLeaf=True

    def __init__(self, bus):
        self.bus = bus

    def addMessageHandler(self, messageHandler):
        self.bus.addHandler(messageHandler)

    def render_GET(self, request):
        self.sendMsg(request, 'GET')
        return server.NOT_DONE_YET

    def render_POST(self, request):
        self.sendMsg(request, 'POST')
        return server.NOT_DONE_YET

    def render_PUT(self, request):
        self.sendMsg(request, 'PUT')
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        self.sendMsg(request, 'DELETE')
        return server.NOT_DONE_YET

    def getChild(self, path, request):
        return self

    def sendMsg(self, request, method):
        request.content.seek(0, 0)
        body = request.content.read()

        content = {
            'url': request.uri,
            'method': method.upper(),
            'body': body,
            'headers': dict(request.requestHeaders.getAllRawHeaders()),
        }

        content = chutney.dumps(content)
        msg = message.Message(NS, content)

        d = self.bus.link.sendWithDeferred(self.bus.targetJID, msg)

        @d.addCallback
        def on_reply(replies):
            for reply in replies:
                dict = chutney.loads(reply.payload)
                request.setResponseCode(dict['status'])
                for key, value in dict.get('headers', {}).items():
                    if key.lower() in ('connection', 'transfer-encoding'):
                        continue
                    request.setHeader(key, value)

                responseBody = dict['body']
                if responseBody:
                    request.write(responseBody)

                if not request._disconnected:
                    request.finish()

        return d

class HTTPClientFactory(client.HTTPClientFactory):
    def __init__(self, url, *args, **kwargs):
        client.HTTPClientFactory.__init__(self, url, *args, **kwargs)
        self.status = None
        self.deferred.addCallback(
            lambda data: (self.status, self.message, self.response_headers,
                data))
<commit_msg>Forward http response errors back to the client instead of masking with a 502<commit_after>#
# Copyright (c) SAS Institute Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import base64
import logging

from conary.lib import cfg as cny_cfg
from conary.lib import cfgtypes
from rmake3.lib import chutney
from rmake3.lib import logger
from rmake3.lib.jabberlink import message
from rmake3.lib.jabberlink.handlers import link

from rmake3.core import types
from rmake3.core import plug_dispatcher

from rmake3.worker import plug_worker

from twisted.web import client
from twisted.web import server
from twisted.web import resource
from twisted.internet import ssl
from twisted.internet import reactor

NS = 'http://rpath.com/permanent/xmpp/repeater-1.0'
log = logging.getLogger(__name__)

class RestForwardingConfig(cny_cfg.ConfigFile):
    # Launcher
    key                     = (cfgtypes.CfgString, None)
    cert                    = (cfgtypes.CfgString, None)
    httpPort                = (cfgtypes.CfgInt, None)
    httpsPort               = (cfgtypes.CfgInt, None)

    # Dispatcher
    repeaterTarget          = (cfgtypes.CfgString, None)

class RestForwardingPlugin(plug_dispatcher.DispatcherPlugin,
    plug_worker.LauncherPlugin):

    def launcher_post_setup(self, launcher):
        """ The Sputnik end of the rMake topology """
        endpoint = EndPoint(launcher.bus)

        cfg = self.populateConfigFromOptions(RestForwardingConfig())
        if cfg.httpPort:
            reactor.listenTCP(cfg.httpPort,
                    server.Site(resource.IResource(endpoint)))
        if cfg.httpsPort:
            reactor.listenSSL(cfg.httpsPort,
                    server.Site(resource.IResource(endpoint)),
                    ssl.DefaultOpenSSLContextFactory(cfg.key, cfg.cert))

    def dispatcher_post_setup(self, dispatcher):
        """ The rBuilder end of the rMake topology """

        cfg = self.populateConfigFromOptions(RestForwardingConfig())
        if cfg.repeaterTarget:
            dispatcher.bus.link.addMessageHandler(
                    RepeaterMessageHandler(cfg.repeaterTarget,
                        dispatcher.workers))

class RepeaterMessageHandler(message.MessageHandler):
    namespace = NS
    XHeader = 'X-rPath-Management-Zone'
    XRepeaterHeader = 'X-rPath-Repeater'

    def __init__(self, host, workers):
        self.host = host
        self.workers = workers

    def getManagementZone(self, neighbor):
        jid = link.toJID(neighbor.jid.full())
        worker = self.workers.get(jid)
        if worker is None:
            return None
        zones = self.getWorkerZones(worker)
        if not zones:
            return None
        # We only care about one zone for now
        return base64.b64encode(zones[0])

    @classmethod
    def getWorkerZones(cls, worker):
        # XXX it would be nice if this was a property of the worker
        zoneNames = [ x.zoneName for x in worker.caps
            if isinstance(x, types.ZoneCapability) ]
        return zoneNames

    def onMessage(self, neighbor, msg):
        reqDict = chutney.loads(msg.payload)
        method = reqDict['method']
        url = reqDict['url']
        body = reqDict['body']
        rawHeaders = reqDict['headers']
        headers = client.Headers(rawHeaders)
        headers.removeHeader(self.XHeader)
        managementZone = self.getManagementZone(neighbor)
        if managementZone is not None:
            headers.addRawHeader(self.XHeader, managementZone)
        # This header flags a request as _not_ being originated from localhost
        # Some of the management interfaces require localhost access, but
        # everything forwarded through the repeater looks like it's
        # originating from localhost, unless this header is present
        headers.addRawHeader(self.XRepeaterHeader, 'remote')
        # XXX this is where multi-valued headers go down the drain
        headers = dict((k.lower(), v[-1])
            for (k, v) in headers.getAllRawHeaders())

        fact = HTTPClientFactory(url, method=method.upper(), postdata=body,
            headers=headers)
        @fact.deferred.addCallback
        def processResult(args):
            (status, statusMessage, headers, body) = args
            reply = dict(
                status = int(status),
                message = statusMessage,
                headers = headers,
                body = body,
            )
            neighbor.send(message.Message(self.namespace, chutney.dumps(reply),
                                           in_reply_to=msg))
            return args

        @fact.deferred.addErrback
        def processError(error):
            logger.logFailure(error, "Error in proxied REST request:")
            reply = dict(
                    status=500,
                    message='Internal Server Error',
                    headers={},
                    body='',
                    )
            neighbor.send(message.Message(self.namespace, chutney.dumps(reply),
                                           in_reply_to=msg))

        reactor.connectTCP(self.host, 80, fact)

class EndPoint(resource.Resource):
    isLeaf=True

    def __init__(self, bus):
        self.bus = bus

    def addMessageHandler(self, messageHandler):
        self.bus.addHandler(messageHandler)

    def render_GET(self, request):
        self.sendMsg(request, 'GET')
        return server.NOT_DONE_YET

    def render_POST(self, request):
        self.sendMsg(request, 'POST')
        return server.NOT_DONE_YET

    def render_PUT(self, request):
        self.sendMsg(request, 'PUT')
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        self.sendMsg(request, 'DELETE')
        return server.NOT_DONE_YET

    def getChild(self, path, request):
        return self

    def sendMsg(self, request, method):
        request.content.seek(0, 0)
        request.requestHeaders.setRawHeaders('x-forwarded-for',
                [request.getClientIP()])
        body = request.content.read()

        content = {
            'url': request.uri,
            'method': method.upper(),
            'body': body,
            'headers': dict(request.requestHeaders.getAllRawHeaders()),
        }

        content = chutney.dumps(content)
        msg = message.Message(NS, content)

        d = self.bus.link.sendWithDeferred(self.bus.targetJID, msg)

        @d.addCallback
        def on_reply(replies):
            for reply in replies:
                dict = chutney.loads(reply.payload)
                request.setResponseCode(dict['status'])
                for key, values in dict.get('headers', {}).items():
                    if key.lower() in ('connection', 'transfer-encoding'):
                        continue
                    request.responseHeaders.setRawHeaders(key, values)

                responseBody = dict['body']
                if responseBody:
                    request.write(responseBody)

                if not request._disconnected:
                    request.finish()

        return d

class HTTPPageGetter(client.HTTPPageGetter):

    def handleStatusDefault(self):
        # Pass through all HTTP responses regardless of status
        pass

class HTTPClientFactory(client.HTTPClientFactory):
    protocol = HTTPPageGetter

    def __init__(self, url, *args, **kwargs):
        client.HTTPClientFactory.__init__(self, url, *args, **kwargs)
        self.status = None
        self.deferred.addCallback(
            lambda data: (self.status, self.message, self.response_headers,
                data))
<|endoftext|>