<commit_before>import readline

from jedi import utils
from .helpers import TestCase

class TestSetupReadline(TestCase):
    class NameSpace():
        pass

    def __init__(self, *args, **kwargs):
        super(type(self), self).__init__(*args, **kwargs)

        self.namespace = self.NameSpace()
        utils.setup_readline(self.namespace)

    def completions(self, text):
        completer = readline.get_completer()
        i = 0
        completions = []
        while True:
            completion = completer(text, i)
            if completion is None:
                break
            completions.append(completion)
            i += 1
        return completions

    def test_simple(self):
        assert self.completions('list') == ['list']
        assert self.completions('importerror') == ['ImportError']
        s = "print BaseE"
        assert self.completions(s) == [s + 'xception']

    def test_nested(self):
        assert self.completions('list.Insert') == ['list.insert']
        assert self.completions('list().Insert') == ['list().insert']

    def test_magic_methods(self):
        assert self.completions('list.__getitem__') == ['list.__getitem__']
        assert self.completions('list().__getitem__') == ['list().__getitem__']

    def test_modules(self):
        import sys
        import os
        self.namespace.sys = sys
        self.namespace.os = os

        assert self.completions('os.path.join') == ['os.path.join']
        assert self.completions('os.path.join().upper') == ['os.path.join().upper']

        c = set(['os.' + d for d in dir(os) if d.startswith('ch')])
        assert set(self.completions('os.ch')) == set(c)

        del self.namespace.sys
        del self.namespace.os

    def test_calls(self):
        s = 'str(bytes'
        assert self.completions(s) == [s, 'str(BytesWarning']

    def test_import(self):
        s = 'from os.path import a'
        assert set(self.completions(s)) == set([s + 'ltsep', s + 'bspath'])
        assert self.completions('import keyword') == ['import keyword']

        import os
        s = 'from os import '
        goal = set([s + el for el in dir(os)])
        # There are minor differences, e.g. the dir doesn't include deleted
        # items as well as items that are not only available on linux.
        assert len(set(self.completions(s)).symmetric_difference(goal)) < 20

    def test_local_import(self):
        s = 'import test_utils'
        assert self.completions(s) == [s]

    def test_preexisting_values(self):
        self.namespace.a = range(10)
        assert set(self.completions('a.')) == set(['a.' + n for n in dir(range(1))])
        del self.namespace.a

    def test_colorama(self):
        """
        Only test it if colorama library is available.

        This module is being tested because it uses ``setattr`` at some point,
        which Jedi doesn't understand, but it should still work in the REPL.
        """
        try:
            # if colorama is installed
            import colorama
        except ImportError:
            pass
        else:
            self.namespace.colorama = colorama
            assert self.completions('colorama')
            assert self.completions('colorama.Fore.BLACK') == ['colorama.Fore.BLACK']
            del self.namespace.colorama
<commit_msg>use cwd_at to avoid test failures if called from different places<commit_after>import readline

from jedi import utils
from .helpers import TestCase, cwd_at

class TestSetupReadline(TestCase):
    class NameSpace():
        pass

    def __init__(self, *args, **kwargs):
        super(type(self), self).__init__(*args, **kwargs)

        self.namespace = self.NameSpace()
        utils.setup_readline(self.namespace)

    def completions(self, text):
        completer = readline.get_completer()
        i = 0
        completions = []
        while True:
            completion = completer(text, i)
            if completion is None:
                break
            completions.append(completion)
            i += 1
        return completions

    def test_simple(self):
        assert self.completions('list') == ['list']
        assert self.completions('importerror') == ['ImportError']
        s = "print BaseE"
        assert self.completions(s) == [s + 'xception']

    def test_nested(self):
        assert self.completions('list.Insert') == ['list.insert']
        assert self.completions('list().Insert') == ['list().insert']

    def test_magic_methods(self):
        assert self.completions('list.__getitem__') == ['list.__getitem__']
        assert self.completions('list().__getitem__') == ['list().__getitem__']

    def test_modules(self):
        import sys
        import os
        self.namespace.sys = sys
        self.namespace.os = os

        assert self.completions('os.path.join') == ['os.path.join']
        assert self.completions('os.path.join().upper') == ['os.path.join().upper']

        c = set(['os.' + d for d in dir(os) if d.startswith('ch')])
        assert set(self.completions('os.ch')) == set(c)

        del self.namespace.sys
        del self.namespace.os

    def test_calls(self):
        s = 'str(bytes'
        assert self.completions(s) == [s, 'str(BytesWarning']

    def test_import(self):
        s = 'from os.path import a'
        assert set(self.completions(s)) == set([s + 'ltsep', s + 'bspath'])
        assert self.completions('import keyword') == ['import keyword']

        import os
        s = 'from os import '
        goal = set([s + el for el in dir(os)])
        # There are minor differences, e.g. the dir doesn't include deleted
        # items as well as items that are not only available on linux.
        assert len(set(self.completions(s)).symmetric_difference(goal)) < 20

    @cwd_at('test')
    def test_local_import(self):
        s = 'import test_utils'
        assert self.completions(s) == [s]

    def test_preexisting_values(self):
        self.namespace.a = range(10)
        assert set(self.completions('a.')) == set(['a.' + n for n in dir(range(1))])
        del self.namespace.a

    def test_colorama(self):
        """
        Only test it if colorama library is available.

        This module is being tested because it uses ``setattr`` at some point,
        which Jedi doesn't understand, but it should still work in the REPL.
        """
        try:
            # if colorama is installed
            import colorama
        except ImportError:
            pass
        else:
            self.namespace.colorama = colorama
            assert self.completions('colorama')
            assert self.completions('colorama.Fore.BLACK') == ['colorama.Fore.BLACK']
            del self.namespace.colorama
<|endoftext|>


<commit_before>#!/usr/bin/env python3
from NER_utils import transform_dataset
from NER_utils import dump_POS_tags
from NER_utils import load_dataset
import pycrfsuite
import sys
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import f1_score, classification_report
from collections import Counter

"""
Usage: python CRF_NER.py named_ent_dtest.txt named_ent_etest.txt model.txt
"""
def global_eval(ypred, ytrue):
    """
    measures macro f1 score for the classification
    """
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    binarizer = LabelBinarizer() # makes one vs all matrices for labels
    y_true_bin = binarizer.fit_transform(merged_ytrue)
    y_pred_bin = binarizer.transform(merged_ypred)
    tags = [tag for tag in binarizer.classes_]
    cl_indices = {cls: idx for idx, cls in enumerate(binarizer.classes_)}
    return f1_score(y_true_bin, y_pred_bin, labels=[cl_indices[tag] for tag in tags],average='macro')

def global_eval2(ypred, ytrue):
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    tags = set(merged_ytrue)
    correct_per_tag_pred = Counter()
    incorrect_per_tag_pred = Counter()
    correct_per_tag_true = Counter()
    incorrect_per_tag_true = Counter()
    incorrect = 0
    correct = 0
    for yp, yt in zip(merged_ypred, merged_ytrue):
        if yp == yt:
            correct += 1
            correct_per_tag_pred[yp] += 1
            correct_per_tag_true[yt] += 1
        else:
            incorrect += 1
            incorrect_per_tag_pred[yp] += 1
            incorrect_per_tag_true[yt] += 1
    precision = correct/incorrect
    per_tag_prec_pred = [(tag, cor_per_tag_pred[tag]/incor_per_tag_pred) for tag in            tags]

def per_token_eval(ypred, ytrue):
    """
    measures per-token precision, ignoring the not_named_entity tag
    """
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    binarizer = LabelBinarizer()
    y_true_bin = binarizer.fit_transform(merged_ytrue)
    y_pred_bin = binarizer.transform(merged_ypred)
    tags = [tag for tag in binarizer.classes_ if tag != 'O']
    cl_indices = {cls: idx for idx, cls in enumerate(binarizer.classes_)}
    return classification_report(y_true_bin,
                                 y_pred_bin,
                                 labels=[cl_indices[tag] for tag in tags],
                                 target_names=tags)

def output_evalutation(classification_report, f1_score, model_name):
    with open(model_name + '.log', 'w') as f:
        f.write('F1 macro score: {}'.format(f1_score))
        f.write('\n')
        f.write(classification_report)
        f.close()

def parse_commands(filename):
    """
    Reads the configuration file with the model name and features, separated by spaces
    model_name feat1 feat2 feat3 ....
    """
    models = []
    with open(filename) as f:
        for line in f.read().split('\n'):
            tokens = line.split(' ')
            if len(tokens[1:]) == 0:
                continue
            models.append((tokens[0], tokens[1:]))
    return models

def main():
    train_filename = sys.argv[1]
    test_filename = sys.argv[2]
    models = parse_commands(sys.argv[3])
    tr_raw = load_dataset(train_filename)
    te_raw = load_dataset(test_filename)
    for model, params in models:
        trainer = pycrfsuite.Trainer(verbose=False)
        tr_label, tr_feature = transform_dataset(tr_raw, params)
        te_label, te_feature = transform_dataset(te_raw, params)
        dump_POS_tags(tr_raw, "train_POS.json")
        dump_POS_tags(te_raw, "test_POS.json")
        for lab, feat in zip(tr_label, tr_feature):
            trainer.append(feat, lab)
        trainer.train(model+'.crfmodel')
        tagger = pycrfsuite.Tagger()
        tagger.open(model+'.crfmodel')
        predictions = [tagger.tag(sentence) for sentence in te_feature]
        per_class_prec = per_token_eval(predictions, te_label)
        f1_score = global_eval(predictions, te_label)
        output_evalutation(per_class_prec, f1_score, model)

if __name__ == '__main__':
    main()
<commit_msg>removed json dumping<commit_after>#!/usr/bin/env python3
from NER_utils import transform_dataset
from NER_utils import dump_POS_tags
from NER_utils import load_dataset
import pycrfsuite
import sys
from sklearn.preprocessing import LabelBinarizer
from sklearn.metrics import f1_score, classification_report
from collections import Counter

"""
Usage: python CRF_NER.py named_ent_dtest.txt named_ent_etest.txt model.txt
"""
def global_eval(ypred, ytrue):
    """
    measures macro f1 score for the classification
    """
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    binarizer = LabelBinarizer() # makes one vs all matrices for labels
    y_true_bin = binarizer.fit_transform(merged_ytrue)
    y_pred_bin = binarizer.transform(merged_ypred)
    tags = [tag for tag in binarizer.classes_]
    cl_indices = {cls: idx for idx, cls in enumerate(binarizer.classes_)}
    return f1_score(y_true_bin, y_pred_bin, labels=[cl_indices[tag] for tag in tags],average='macro')

def global_eval2(ypred, ytrue):
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    tags = set(merged_ytrue)
    correct_per_tag_pred = Counter()
    incorrect_per_tag_pred = Counter()
    correct_per_tag_true = Counter()
    incorrect_per_tag_true = Counter()
    incorrect = 0
    correct = 0
    for yp, yt in zip(merged_ypred, merged_ytrue):
        if yp == yt:
            correct += 1
            correct_per_tag_pred[yp] += 1
            correct_per_tag_true[yt] += 1
        else:
            incorrect += 1
            incorrect_per_tag_pred[yp] += 1
            incorrect_per_tag_true[yt] += 1
    precision = correct/incorrect
    per_tag_prec_pred = [(tag, cor_per_tag_pred[tag]/incor_per_tag_pred) for tag in            tags]

def per_token_eval(ypred, ytrue):
    """
    measures per-token precision, ignoring the not_named_entity tag
    """
    merged_ypred = [item for sublist in ypred for item in sublist]
    merged_ytrue = [item for sublist in ytrue for item in sublist]
    binarizer = LabelBinarizer()
    y_true_bin = binarizer.fit_transform(merged_ytrue)
    y_pred_bin = binarizer.transform(merged_ypred)
    tags = [tag for tag in binarizer.classes_ if tag != 'O']
    cl_indices = {cls: idx for idx, cls in enumerate(binarizer.classes_)}
    return classification_report(y_true_bin,
                                 y_pred_bin,
                                 labels=[cl_indices[tag] for tag in tags],
                                 target_names=tags)

def output_evalutation(classification_report, f1_score, model_name):
    with open(model_name + '.log', 'w') as f:
        f.write('F1 macro score: {}'.format(f1_score))
        f.write('\n')
        f.write(classification_report)
        f.close()

def parse_commands(filename):
    """
    Reads the configuration file with the model name and features, separated by spaces
    model_name feat1 feat2 feat3 ....
    """
    models = []
    with open(filename) as f:
        for line in f.read().split('\n'):
            tokens = line.split(' ')
            if len(tokens[1:]) == 0:
                continue
            models.append((tokens[0], tokens[1:]))
    return models

def main():
    train_filename = sys.argv[1]
    test_filename = sys.argv[2]
    models = parse_commands(sys.argv[3])
    tr_raw = load_dataset(train_filename)
    te_raw = load_dataset(test_filename)
    for model, params in models:
        trainer = pycrfsuite.Trainer(verbose=False)
        tr_label, tr_feature = transform_dataset(tr_raw, params)
        te_label, te_feature = transform_dataset(te_raw, params)
        for lab, feat in zip(tr_label, tr_feature):
            trainer.append(feat, lab)
        trainer.train(model+'.crfmodel')
        tagger = pycrfsuite.Tagger()
        tagger.open(model+'.crfmodel')
        predictions = [tagger.tag(sentence) for sentence in te_feature]
        per_class_prec = per_token_eval(predictions, te_label)
        f1_score = global_eval(predictions, te_label)
        output_evalutation(per_class_prec, f1_score, model)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
Django settings for scotland_yard project.

Generated by 'django-admin startproject' using Django 1.9.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import dj_database_url

DEBUG = True

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = os.environ.get('SECRET_KEY')

CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True

ALLOWED_HOSTS = []

INSTALLED_APPS = [
    'syard_main.apps.SyardMainConfig',
    'syard_api.apps.SyardApiConfig',
    'corsheaders',
    'rest_framework',
    'rest_framework.authtoken',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = [
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'scotland_yard.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'scotland_yard.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get('DJANGO_DATABASE_URL'))
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, "scotland_yard", "static")
<commit_msg>expose headers<commit_after>"""
Django settings for scotland_yard project.

Generated by 'django-admin startproject' using Django 1.9.5.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os
import dj_database_url

DEBUG = True

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = os.environ.get('SECRET_KEY')

CORS_ORIGIN_ALLOW_ALL = True
CORS_ALLOW_CREDENTIALS = True
CORS_EXPOSE_HEADERS = ("authToken",)

ALLOWED_HOSTS = []

INSTALLED_APPS = [
    'syard_main.apps.SyardMainConfig',
    'syard_api.apps.SyardApiConfig',
    'corsheaders',
    'rest_framework',
    'rest_framework.authtoken',
    'django.contrib.auth',
    'django.contrib.admin',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = [
    'corsheaders.middleware.CorsMiddleware',
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'scotland_yard.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'scotland_yard.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': dj_database_url.config(default=os.environ.get('DJANGO_DATABASE_URL'))
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, "scotland_yard", "static")
<|endoftext|>


<commit_before># $Id: VTKtoITK.py 2110 2006-06-22 15:11:21Z cpbotha $

import itk
import module_kits.itk_kit as itk_kit
from moduleBase import moduleBase
from moduleMixins import scriptedConfigModuleMixin
import vtk

class VTKtoITK(scriptedConfigModuleMixin, moduleBase):

    def __init__(self, moduleManager):
        moduleBase.__init__(self, moduleManager)

        self._input = None

        self._image_cast = vtk.vtkImageCast()
        self._vtk2itk = None
        # this stores the short_string of the current converter, e.g.
        # F3 or US3, etc.
        self._vtk2itk_short_string = None

        self._config.autotype = True
        # this will store the current type as full text, e.g. "unsigned char"
        self._config.type = 'float'

        config_list = [
            ('AutoType:', 'autotype', 'base:bool', 'checkbox',
             'If activated, output data type is set to '
             'input type.'),
            ('Data type:', 'type', 'base:str', 'choice',
             'Data will be cast to this type if AutoType is not used.',
             ['float', 'unsigned char', 'unsigned long', 'unsigned short'])]

        scriptedConfigModuleMixin.__init__(self, config_list)

        self._createWindow(
            {'Module (self)' : self})

        self.configToLogic()
        self.logicToConfig()
        self.configToView()

    def close(self):
        # we play it safe... (the graph_editor/module_manager should have
        # disconnected us by now)
        for inputIdx in range(len(self.getInputDescriptions())):
            self.setInput(inputIdx, None)

        # this will take care of all display thingies
        scriptedConfigModuleMixin.close(self)
        # and the baseclass close
        moduleBase.close(self)

        del self._image_cast
        del self._vtk2itk

    def executeModule(self):
        if self._input:

            # calculate input type
            try:
                self._input.Update()
                if self._input.IsA('vtkImageData'):
                    input_type = self._input.GetScalarTypeAsString()
            except AttributeError:
                raise TypeError, 'VTKtoITK requires VTK image data as input.'

            # input_type is a text repr, e.g. 'float' or 'unsigned char'

            # now calculate output type
            if self._config.autotype:
                output_type = input_type

            else:
                # this is also just a text description
                output_type = self._config.type

            dims = self._input.GetDataDimension()

            # see if we need a new converter
            short_string = '%s%s' % \
                           (''.join([i[0].upper()
                                     for i in output_type.split()]),
                            dims)

            if short_string != self._vtk2itk_short_string:
                self._vtk2itk = itk.VTKImageToImageFilter[
                    getattr(itk.Image, short_string)].New()
                self._vtk2itk_short_string = short_string

            if output_type == input_type:
                # we don't need the cast
                self._vtk2itk.SetInput(self._input)

            else:
                # we do need the cast
                self._image_cast.SetInput(self._input)
                # turn unsgned char into UnsignedChar
                vtk_type_string = '%s%s' % (output_type[0].upper(),
                                            output_type[1:])
                # cast function
                cast_function = getattr(
                    self._image_cast,
                    'SetOutputScalarTypeTo%s' % (vtk_type_string,))

                cast_function()

                # and connect it up
                self._vtk2itk.SetInput(self._image_cast.GetOutput())

        self._vtk2itk.Update()

    def getInputDescriptions(self):
        return ('VTK Image Data',)

    def setInput(self, idx, input_stream):
        self._input = input_stream

    def getOutputDescriptions(self):
        return ('ITK Image (3D',)

    def getOutput(self, idx):
        if self._vtk2itk:
            return self._vtk2itk.GetOutput()
        else:
            return None

    def logicToConfig(self):
        return False

    def configToLogic(self):
        # if the user has pressed on Apply, we change our state
        return True

<commit_msg>VTKtoITK now recreates the converter filter every time.  (it can get easily confused)<commit_after># $Id: VTKtoITK.py 2110 2006-06-22 15:11:21Z cpbotha $

import itk
import module_kits.itk_kit as itk_kit
from moduleBase import moduleBase
from moduleMixins import scriptedConfigModuleMixin
import vtk

class VTKtoITK(scriptedConfigModuleMixin, moduleBase):

    def __init__(self, moduleManager):
        moduleBase.__init__(self, moduleManager)

        self._input = None

        self._image_cast = vtk.vtkImageCast()
        self._vtk2itk = None
        # this stores the short_string of the current converter, e.g.
        # F3 or US3, etc.
        self._vtk2itk_short_string = None

        self._config.autotype = True
        # this will store the current type as full text, e.g. "unsigned char"
        self._config.type = 'float'

        config_list = [
            ('AutoType:', 'autotype', 'base:bool', 'checkbox',
             'If activated, output data type is set to '
             'input type.'),
            ('Data type:', 'type', 'base:str', 'choice',
             'Data will be cast to this type if AutoType is not used.',
             ['float', 'unsigned char', 'unsigned long', 'unsigned short'])]

        scriptedConfigModuleMixin.__init__(self, config_list)

        self._createWindow(
            {'Module (self)' : self})

        self.configToLogic()
        self.logicToConfig()
        self.configToView()

    def close(self):
        # we play it safe... (the graph_editor/module_manager should have
        # disconnected us by now)
        for inputIdx in range(len(self.getInputDescriptions())):
            self.setInput(inputIdx, None)

        # this will take care of all display thingies
        scriptedConfigModuleMixin.close(self)
        # and the baseclass close
        moduleBase.close(self)

        del self._image_cast
        del self._vtk2itk

    def executeModule(self):
        if self._input:

            # calculate input type
            try:
                self._input.Update()
                if self._input.IsA('vtkImageData'):
                    input_type = self._input.GetScalarTypeAsString()
            except AttributeError:
                raise TypeError, 'VTKtoITK requires VTK image data as input.'

            # input_type is a text repr, e.g. 'float' or 'unsigned char'

            # now calculate output type
            if self._config.autotype:
                output_type = input_type

            else:
                # this is also just a text description
                output_type = self._config.type

            dims = self._input.GetDataDimension()

            # see if we need a new converter
            short_string = '%s%s' % \
                           (''.join([i[0].upper()
                                     for i in output_type.split()]),
                            dims)

            # we could cache this as shown below, but experience shows
            # that the connection module often gets confused when its
            # input data remains the same w.r.t. type, but changes in
            # extent for instance.
            #if short_string != self._vtk2itk_short_string:
            self._vtk2itk = itk.VTKImageToImageFilter[
                getattr(itk.Image, short_string)].New()
            self._vtk2itk_short_string = short_string

            if output_type == input_type:
                # we don't need the cast
                self._vtk2itk.SetInput(self._input)

            else:
                # we do need the cast
                self._image_cast.SetInput(self._input)
                # turn unsgned char into UnsignedChar
                vtk_type_string = '%s%s' % (output_type[0].upper(),
                                            output_type[1:])
                # cast function
                cast_function = getattr(
                    self._image_cast,
                    'SetOutputScalarTypeTo%s' % (vtk_type_string,))

                cast_function()

                # and connect it up
                self._vtk2itk.SetInput(self._image_cast.GetOutput())

        self._vtk2itk.Update()

    def getInputDescriptions(self):
        return ('VTK Image Data',)

    def setInput(self, idx, input_stream):
        self._input = input_stream

    def getOutputDescriptions(self):
        return ('ITK Image (3D',)

    def getOutput(self, idx):
        if self._vtk2itk:
            return self._vtk2itk.GetOutput()
        else:
            return None

    def logicToConfig(self):
        return False

    def configToLogic(self):
        # if the user has pressed on Apply, we change our state
        return True

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from flask import (abort, Blueprint, render_template, request)

from .partial import send_file_partial

pileup_bp = Blueprint('pileup', __name__, template_folder='templates',
                      static_folder='static', static_url_path='/pileup/static')

@pileup_bp.route('/remote/static', methods=['OPTIONS', 'GET'])
def remote_static():
    """Stream *large* static files with special requirements."""
    file_path = request.args.get('file')

    range_header = request.headers.get('Range', None)
    if not range_header and file_path.endswith('.bam'):
        return abort(500)

    new_resp = send_file_partial(file_path)
    return new_resp

@pileup_bp.route('/pileup')
def viewer():
    """Visualize BAM alignments."""
    vcf_file = request.args.get('vcf')
    bam_files = request.args.getlist('bam')
    bai_files = request.args.getlist('bai')
    samples = request.args.getlist('sample')
    alignments = [{'bam': bam, 'bai': bai, 'sample': sample}
                  for bam, bai, sample in zip(bam_files, bai_files, samples)]

    position = {
        'contig': request.args['contig'],
        'start': request.args['start'],
        'stop': request.args['stop']
    }

    return render_template('pileup/pileup.html', alignments=alignments,
                           position=position, vcf_file=vcf_file, genome=pileup_genome_path,
                           exons=pileup_exons_path)
<commit_msg>Fixes problem with pileup view<commit_after># -*- coding: utf-8 -*-
from flask import (abort, Blueprint, render_template, request)

from .partial import send_file_partial

pileup_bp = Blueprint('pileup', __name__, template_folder='templates',
                      static_folder='static', static_url_path='/pileup/static')

@pileup_bp.route('/remote/static', methods=['OPTIONS', 'GET'])
def remote_static():
    """Stream *large* static files with special requirements."""
    file_path = request.args.get('file')

    range_header = request.headers.get('Range', None)
    if not range_header and file_path.endswith('.bam'):
        return abort(500)

    new_resp = send_file_partial(file_path)
    return new_resp

@pileup_bp.route('/pileup')
def viewer():
    """Visualize BAM alignments."""
    vcf_file = request.args.get('vcf')
    bam_files = request.args.getlist('bam')
    bai_files = request.args.getlist('bai')
    samples = request.args.getlist('sample')
    alignments = [{'bam': bam, 'bai': bai, 'sample': sample}
                  for bam, bai, sample in zip(bam_files, bai_files, samples)]

    position = {
        'contig': request.args['contig'],
        'start': request.args['start'],
        'stop': request.args['stop']
    }

    return render_template('pileup/pileup.html', alignments=alignments,
                           position=position, vcf_file=vcf_file)
<|endoftext|>


<commit_before>"""
Figshare harvester of public projects for the SHARE Notification Service
Note: At the moment, this harvester only harvests basic data on each article, and does
not make a seperate request for additional metadata for each record.

Example API query: http://api.figshare.com/v1/articles/search?search_for=*&from_date=2015-2-1&end_date=2015-2-1
"""

from __future__ import unicode_literals

import time
import json
import logging
from dateutil.parser import parse
from datetime import date, timedelta

from nameparser import HumanName

from scrapi import requests
from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

logger = logging.getLogger(__name__)

NAME = 'figshare'
URL = 'http://api.figshare.com/v1/articles/search?search_for=*&from_date='

def harvest(days_back=0):
    start_date = date.today() - timedelta(days_back) - timedelta(1)
    end_date = date.today() - timedelta(1)
    search_url = '{0}{1}-{2}-{3}&end_date={4}-{5}-{6}'.format(URL,
                                                              start_date.year,
                                                              start_date.month,
                                                              start_date.day,
                                                              end_date.year,
                                                              end_date.month,
                                                              end_date.day)

    records = get_records(search_url)

    record_list = []
    for record in records:
        doc_id = record['article_id']

        record_list.append(
            RawDocument(
                {
                    'doc': json.dumps(record),
                    'source': NAME,
                    'docID': unicode(doc_id),
                    'filetype': 'json'
                }
            )
        )

    return record_list

def get_records(search_url):
    records = requests.get(search_url)
    total_records = records.json()['items_found']
    page = 1

    all_records = []
    while len(all_records) < total_records:
        logger.info('Requesting records for url: {}&page={}'.format(search_url, str(page)))
        record_list = records.json()['items']

        for record in record_list:
            if len(all_records) < total_records:
                all_records.append(record)

        page += 1
        records = requests.get(search_url + '&page={}'.format(str(page)))
        time.sleep(3)

    return all_records

def get_contributors(record):

    authors = record['authors']

    contributor_list = []
    for person in authors:
        name = HumanName(person['author_name'])
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': '',
        }
        contributor_list.append(contributor)

    return contributor_list

def get_ids(record):
    # Right now, only take the last DOI - others in properties
    doi = record['DOI']
    try:
        doi = doi.replace('http://dx.doi.org/', '')
    except AttributeError:
        for item in doi:
            item.replace('http://dx.doi.org/', '')
            doi = item

    return {
        'serviceID': unicode(record['article_id']),
        'url': record['url'],
        'doi': doi
    }

def get_properties(record):
    return {
        'article_id': record['article_id'],
        'defined_type': record['defined_type'],
        'type': record['type'],
        'links': record['links'],
        'doi': record['DOI'],
        'publishedDate': record['published_date']
    }

def normalize(raw_doc):
    doc = raw_doc.get('doc')
    record = json.loads(doc)

    normalized_dict = {
        'title': record['title'],
        'contributors': get_contributors(record),
        'properties': get_properties(record),
        'description': record['description'],
        'tags': [],
        'id': get_ids(record),
        'source': NAME,
        'dateUpdated': unicode(parse(record['modified_date']).isoformat())
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(harvest, normalize))
<commit_msg>Remove redundant logging from figshare<commit_after>"""
Figshare harvester of public projects for the SHARE Notification Service
Note: At the moment, this harvester only harvests basic data on each article, and does
not make a seperate request for additional metadata for each record.

Example API query: http://api.figshare.com/v1/articles/search?search_for=*&from_date=2015-2-1&end_date=2015-2-1
"""

from __future__ import unicode_literals

import time
import json
import logging
from dateutil.parser import parse
from datetime import date, timedelta

from nameparser import HumanName

from scrapi import requests
from scrapi.linter import lint
from scrapi.linter.document import RawDocument, NormalizedDocument

logger = logging.getLogger(__name__)

NAME = 'figshare'
URL = 'http://api.figshare.com/v1/articles/search?search_for=*&from_date='

def harvest(days_back=0):
    start_date = date.today() - timedelta(days_back) - timedelta(1)
    end_date = date.today() - timedelta(1)
    search_url = '{0}{1}-{2}-{3}&end_date={4}-{5}-{6}'.format(URL,
                                                              start_date.year,
                                                              start_date.month,
                                                              start_date.day,
                                                              end_date.year,
                                                              end_date.month,
                                                              end_date.day)

    records = get_records(search_url)

    record_list = []
    for record in records:
        doc_id = record['article_id']

        record_list.append(
            RawDocument(
                {
                    'doc': json.dumps(record),
                    'source': NAME,
                    'docID': unicode(doc_id),
                    'filetype': 'json'
                }
            )
        )

    return record_list

def get_records(search_url):
    records = requests.get(search_url)
    total_records = records.json()['items_found']
    page = 1

    all_records = []
    while len(all_records) < total_records:
        record_list = records.json()['items']

        for record in record_list:
            if len(all_records) < total_records:
                all_records.append(record)

        page += 1
        records = requests.get(search_url + '&page={}'.format(str(page)))
        time.sleep(3)

    return all_records

def get_contributors(record):

    authors = record['authors']

    contributor_list = []
    for person in authors:
        name = HumanName(person['author_name'])
        contributor = {
            'prefix': name.title,
            'given': name.first,
            'middle': name.middle,
            'family': name.last,
            'suffix': name.suffix,
            'email': '',
            'ORCID': '',
        }
        contributor_list.append(contributor)

    return contributor_list

def get_ids(record):
    # Right now, only take the last DOI - others in properties
    doi = record['DOI']
    try:
        doi = doi.replace('http://dx.doi.org/', '')
    except AttributeError:
        for item in doi:
            item.replace('http://dx.doi.org/', '')
            doi = item

    return {
        'serviceID': unicode(record['article_id']),
        'url': record['url'],
        'doi': doi
    }

def get_properties(record):
    return {
        'article_id': record['article_id'],
        'defined_type': record['defined_type'],
        'type': record['type'],
        'links': record['links'],
        'doi': record['DOI'],
        'publishedDate': record['published_date']
    }

def normalize(raw_doc):
    doc = raw_doc.get('doc')
    record = json.loads(doc)

    normalized_dict = {
        'title': record['title'],
        'contributors': get_contributors(record),
        'properties': get_properties(record),
        'description': record['description'],
        'tags': [],
        'id': get_ids(record),
        'source': NAME,
        'dateUpdated': unicode(parse(record['modified_date']).isoformat())
    }

    return NormalizedDocument(normalized_dict)

if __name__ == '__main__':
    print(lint(harvest, normalize))
<|endoftext|>


<commit_before># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013 Zachary Spector,  zacharyspector@gmail.com
from math import cos, sin, hypot, atan
from LiSE.util import (
    wedge_offsets_rise_run,
    truncated_line,
    fortyfive)
from kivy.graphics import Line, Color
from kivy.uix.widget import Widget
from kivy.properties import (
    ObjectProperty,
    ListProperty)
from kivy.clock import Clock

def get_points(ox, orx, oy, ory, dx, drx, dy, dry, taillen):
    ox += orx
    oy += ory
    dx += drx
    dy += dry
    if drx > dry:
        dr = drx
    else:
        dr = dry
    if dy < oy:
        yco = -1
    else:
        yco = 1
    if dx < ox:
        xco = -1
    else:
        xco = 1
    (leftx, boty, rightx, topy) = truncated_line(
        float(ox * xco), float(oy * yco),
        float(dx * xco), float(dy * yco),
        dr + 1)
    rise = topy - boty
    run = rightx - leftx
    if rise == 0:
        xoff1 = cos(fortyfive) * taillen
        yoff1 = xoff1
        xoff2 = xoff1
        yoff2 = -1 * yoff1
    elif run == 0:
        xoff1 = sin(fortyfive) * taillen
        yoff1 = xoff1
        xoff2 = -1 * xoff1
        yoff2 = yoff1
    else:
        (xoff1, yoff1, xoff2, yoff2) = wedge_offsets_rise_run(
            rise, run, taillen)
    x1 = (rightx - xoff1) * xco
    x2 = (rightx - xoff2) * xco
    y1 = (topy - yoff1) * yco
    y2 = (topy - yoff2) * yco
    endx = rightx * xco
    endy = topy * yco
    r = [ox, oy,
         endx, endy, x1, y1,
         endx, endy, x2, y2,
         endx, endy]
    return r

class Arrow(Widget):
    """A widget that points from one :class:`~LiSE.gui.board.Spot` to
    another.

    :class:`Arrow`s are the graphical representations of
    :class:`~LiSE.model.Portal`s. They point from the :class:`Spot`
    representing the :class:`Portal`'s origin, to the one representing
    its destination.

    """
    margin = 10
    """When deciding whether a touch collides with me, how far away can
    the touch get before I should consider it a miss?"""
    w = 1
    """The width of the inner, brighter portion of the :class:`Arrow`. The
    whole :class:`Arrow` will end up thicker."""
    board = ObjectProperty()
    """The board on which I am displayed."""
    portal = ObjectProperty()
    """The portal that I represent."""
    """Pawns that are part-way through me. Each needs to present a
    'progress' property to let me know how far through me they ought to be
    repositioned."""
    pawns_here = ListProperty([])

    def __init__(self, **kwargs):
        """Bind some properties, and put the relevant instructions into the
        canvas--but don't put any point data into the instructions
        just yet. For that, wait until ``on_parent``, when we are
        guaranteed to know the positions of our endpoints.

        """
        Widget.__init__(self, **kwargs)
        self.trigger_repoint = Clock.create_trigger(
            self.repoint, timeout=-1)
        self.board.arrowdict[unicode(self.portal)] = self
        orign = unicode(self.portal.origin)
        destn = unicode(self.portal.destination)
        self.board.spotdict[orign].bind(
            pos=self.trigger_repoint,
            size=self.trigger_repoint)
        self.board.spotdict[destn].bind(
            pos=self.trigger_repoint,
            size=self.trigger_repoint)
        self.board.host.closet.register_time_listener(self.repawn)
        self.bind(pos=self.trigger_repoint)
        self.bind(size=self.trigger_repoint)
        self.bg_color = Color(0.25, 0.25, 0.25)
        self.fg_color = Color(1.0, 1.0, 1.0)
        self.bg_line = Line(width=self.w * 1.4)
        self.fg_line = Line(width=self.w)
        self.canvas.add(self.bg_color)
        self.canvas.add(self.bg_line)
        self.canvas.add(self.fg_color)
        self.canvas.add(self.fg_line)

    def __unicode__(self):
        """Return Unicode name of my :class:`Portal`"""
        return unicode(self.portal)

    def __str__(self):
        """Return string name of my :class:`Portal`"""
        return str(self.portal)

    @property
    def reciprocal(self):
        """If it exists, return the edge of the :class:`Portal` that connects
        the same two places that I do, but in the opposite
        direction. Otherwise, return ``None``.

        """
        # Return the edge of the portal that connects the same two
        # places in the opposite direction, supposing it exists
        try:
            return self.portal.reciprocal.arrow
        except KeyError:
            return None

    def handle_time(self, b, t):
        pass

    def get_points(self):
        """Return the coordinates of the points that describe my shape."""
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        try:
            (ow, oh) = orig.size
        except AttributeError:
            (ow, oh) = (0, 0)
        taillen = float(self.board.arrowhead_size)
        orx = ow / 2
        ory = ow / 2
        (dx, dy) = dest.pos
        try:
            (dw, dh) = dest.size
        except AttributeError:
            (dw, dh) = (0, 0)
        drx = dw / 2
        dry = dh / 2
        return get_points(ox, orx, oy, ory, dx, drx, dy, dry, taillen)

    def get_slope(self):
        """Return a float of the increase in y divided by the increase in x,
        both from left to right."""
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        ox = orig.x
        oy = orig.y
        dx = dest.x
        dy = dest.y
        if oy == dy:
            return 0
        elif ox == dx:
            return None
        else:
            rise = dy - oy
            run = dx - ox
            return rise / run

    def get_b(self):
        """Return my Y-intercept.

        I probably don't really hit the left edge of the window, but
        this is where I would, if I were long enough.

        """
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        (dx, dy) = dest.pos
        denominator = dx - ox
        x_numerator = (dy - oy) * ox
        y_numerator = denominator * oy
        return ((y_numerator - x_numerator), denominator)

    def repoint(self, *args):
        """Recalculate all my points and redraw. Reposition any pawns on
        me."""
        points = self.get_points()
        self.bg_line.points = points
        self.fg_line.points = points
        origspot = self.board.spotdict[self.portal.origin.name]
        destspot = self.board.spotdict[self.portal.destination.name]
        (ox, oy) = origspot.pos
        (dx, dy) = destspot.pos
        (branch, tick) = self.board.host.sanetime(None, None)

    def collide_point(self, x, y):
        """Return True iff the point falls sufficiently close to my core line
        segment to count as a hit.

        """
        if not super(Arrow, self).collide_point(x, y):
            return False
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        (dx, dy) = dest.pos
        if ox == dx:
            return abs(y - dy) <= self.w
        elif oy == dy:
            return abs(x - dx) <= self.w
        else:
            correct_angle_a = atan(dy / dx)
            observed_angle_a = atan(y / x)
            error_angle_a = abs(observed_angle_a - correct_angle_a)
            error_seg_len = hypot(x, y)
            return sin(error_angle_a) * error_seg_len <= self.margin

    def repawn(self, branch, tick):
        for pawn in self.pawns_here:
            locations = pawn.thing.get_locations(
                self.board.facade.observer, branch)
            bone = locations.value_during(tick)
            t1 = bone.tick
            try:
                t2 = locations.key_after(tick)
            except ValueError:
                continue
            if t2 is None:
                continue
            else:
                duration = float(t2 - t1)
                passed = float(tick - t1)
                progress = passed / duration
            os = self.board.spotdict[unicode(self.portal.origin)]
            ds = self.board.spotdict[unicode(self.portal.destination)]
            (ox, oy) = os.pos
            (dx, dy) = ds.pos
            w = dx - ox
            h = dy - oy
            x = ox + w * progress
            y = oy + h * progress
            pawn.pos = (x, y)

    def on_pawns_here(self, i, v):
        (branch, tick) = self.board.host.sanetime(None, None)
        self.repawn(branch, tick)
<commit_msg>how 'bout now<commit_after># This file is part of LiSE, a framework for life simulation games.
# Copyright (c) 2013 Zachary Spector,  zacharyspector@gmail.com
from math import cos, sin, hypot, atan
from LiSE.util import (
    wedge_offsets_rise_run,
    truncated_line,
    fortyfive)
from kivy.graphics import Line, Color
from kivy.uix.widget import Widget
from kivy.properties import (
    ObjectProperty,
    ListProperty)
from kivy.clock import Clock

def get_points(ox, orx, oy, ory, dx, drx, dy, dry, taillen):
    ox += orx
    oy += ory
    dx += drx
    dy += dry
    if drx > dry:
        dr = drx
    else:
        dr = dry
    if dy < oy:
        yco = -1
    else:
        yco = 1
    if dx < ox:
        xco = -1
    else:
        xco = 1
    (leftx, boty, rightx, topy) = truncated_line(
        float(ox * xco), float(oy * yco),
        float(dx * xco), float(dy * yco),
        dr + 1)
    rise = topy - boty
    run = rightx - leftx
    if rise == 0:
        xoff1 = cos(fortyfive) * taillen
        yoff1 = xoff1
        xoff2 = xoff1
        yoff2 = -1 * yoff1
    elif run == 0:
        xoff1 = sin(fortyfive) * taillen
        yoff1 = xoff1
        xoff2 = -1 * xoff1
        yoff2 = yoff1
    else:
        (xoff1, yoff1, xoff2, yoff2) = wedge_offsets_rise_run(
            rise, run, taillen)
    x1 = (rightx - xoff1) * xco
    x2 = (rightx - xoff2) * xco
    y1 = (topy - yoff1) * yco
    y2 = (topy - yoff2) * yco
    endx = rightx * xco
    endy = topy * yco
    r = [ox, oy,
         endx, endy, x1, y1,
         endx, endy, x2, y2,
         endx, endy]
    return r

class Arrow(Widget):
    """A widget that points from one :class:`~LiSE.gui.board.Spot` to
    another.

    :class:`Arrow`s are the graphical representations of
    :class:`~LiSE.model.Portal`s. They point from the :class:`Spot`
    representing the :class:`Portal`'s origin, to the one representing
    its destination.

    """
    margin = 10
    """When deciding whether a touch collides with me, how far away can
    the touch get before I should consider it a miss?"""
    w = 1
    """The width of the inner, brighter portion of the :class:`Arrow`. The
    whole :class:`Arrow` will end up thicker."""
    board = ObjectProperty()
    """The board on which I am displayed."""
    portal = ObjectProperty()
    """The portal that I represent."""
    """Pawns that are part-way through me. Each needs to present a
    'progress' property to let me know how far through me they ought to be
    repositioned."""
    pawns_here = ListProperty([])

    def __init__(self, **kwargs):
        """Bind some properties, and put the relevant instructions into the
        canvas--but don't put any point data into the instructions
        just yet. For that, wait until ``on_parent``, when we are
        guaranteed to know the positions of our endpoints.

        """
        Widget.__init__(self, **kwargs)
        self.trigger_repoint = Clock.create_trigger(
            self.repoint)
        self.board.arrowdict[unicode(self.portal)] = self
        orign = unicode(self.portal.origin)
        destn = unicode(self.portal.destination)
        self.board.spotdict[orign].bind(
            pos=self.trigger_repoint,
            size=self.trigger_repoint)
        self.board.spotdict[destn].bind(
            pos=self.trigger_repoint,
            size=self.trigger_repoint)
        self.board.host.closet.register_time_listener(self.repawn)
        self.bind(pos=self.trigger_repoint)
        self.bind(size=self.trigger_repoint)
        self.bg_color = Color(0.25, 0.25, 0.25)
        self.fg_color = Color(1.0, 1.0, 1.0)
        self.bg_line = Line(width=self.w * 1.4)
        self.fg_line = Line(width=self.w)
        self.canvas.add(self.bg_color)
        self.canvas.add(self.bg_line)
        self.canvas.add(self.fg_color)
        self.canvas.add(self.fg_line)

    def __unicode__(self):
        """Return Unicode name of my :class:`Portal`"""
        return unicode(self.portal)

    def __str__(self):
        """Return string name of my :class:`Portal`"""
        return str(self.portal)

    @property
    def reciprocal(self):
        """If it exists, return the edge of the :class:`Portal` that connects
        the same two places that I do, but in the opposite
        direction. Otherwise, return ``None``.

        """
        # Return the edge of the portal that connects the same two
        # places in the opposite direction, supposing it exists
        try:
            return self.portal.reciprocal.arrow
        except KeyError:
            return None

    def handle_time(self, b, t):
        pass

    def get_points(self):
        """Return the coordinates of the points that describe my shape."""
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        try:
            (ow, oh) = orig.size
        except AttributeError:
            (ow, oh) = (0, 0)
        taillen = float(self.board.arrowhead_size)
        orx = ow / 2
        ory = ow / 2
        (dx, dy) = dest.pos
        try:
            (dw, dh) = dest.size
        except AttributeError:
            (dw, dh) = (0, 0)
        drx = dw / 2
        dry = dh / 2
        return get_points(ox, orx, oy, ory, dx, drx, dy, dry, taillen)

    def get_slope(self):
        """Return a float of the increase in y divided by the increase in x,
        both from left to right."""
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        ox = orig.x
        oy = orig.y
        dx = dest.x
        dy = dest.y
        if oy == dy:
            return 0
        elif ox == dx:
            return None
        else:
            rise = dy - oy
            run = dx - ox
            return rise / run

    def get_b(self):
        """Return my Y-intercept.

        I probably don't really hit the left edge of the window, but
        this is where I would, if I were long enough.

        """
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        (dx, dy) = dest.pos
        denominator = dx - ox
        x_numerator = (dy - oy) * ox
        y_numerator = denominator * oy
        return ((y_numerator - x_numerator), denominator)

    def repoint(self, *args):
        """Recalculate all my points and redraw. Reposition any pawns on
        me."""
        points = self.get_points()
        self.bg_line.points = points
        self.fg_line.points = points
        origspot = self.board.spotdict[self.portal.origin.name]
        destspot = self.board.spotdict[self.portal.destination.name]
        (ox, oy) = origspot.pos
        (dx, dy) = destspot.pos
        (branch, tick) = self.board.host.sanetime(None, None)

    def collide_point(self, x, y):
        """Return True iff the point falls sufficiently close to my core line
        segment to count as a hit.

        """
        if not super(Arrow, self).collide_point(x, y):
            return False
        orig = self.board.spotdict[unicode(self.portal.origin)]
        dest = self.board.spotdict[unicode(self.portal.destination)]
        (ox, oy) = orig.pos
        (dx, dy) = dest.pos
        if ox == dx:
            return abs(y - dy) <= self.w
        elif oy == dy:
            return abs(x - dx) <= self.w
        else:
            correct_angle_a = atan(dy / dx)
            observed_angle_a = atan(y / x)
            error_angle_a = abs(observed_angle_a - correct_angle_a)
            error_seg_len = hypot(x, y)
            return sin(error_angle_a) * error_seg_len <= self.margin

    def repawn(self, branch, tick):
        for pawn in self.pawns_here:
            locations = pawn.thing.get_locations(
                self.board.facade.observer, branch)
            bone = locations.value_during(tick)
            t1 = bone.tick
            try:
                t2 = locations.key_after(tick)
            except ValueError:
                continue
            if t2 is None:
                continue
            else:
                duration = float(t2 - t1)
                passed = float(tick - t1)
                progress = passed / duration
            os = self.board.spotdict[unicode(self.portal.origin)]
            ds = self.board.spotdict[unicode(self.portal.destination)]
            (ox, oy) = os.pos
            (dx, dy) = ds.pos
            w = dx - ox
            h = dy - oy
            x = ox + w * progress
            y = oy + h * progress
            pawn.pos = (x, y)

    def on_pawns_here(self, i, v):
        (branch, tick) = self.board.host.sanetime(None, None)
        self.repawn(branch, tick)
<|endoftext|>


<commit_before>
import types
import typing

Dependency = typing.Tuple("Dependency", [PackageName, VersionRequirement, FrameworkRestrictions])
DependencySet = typing.Set[Dependency]

with typing.NamedTuple("PackageDetails", [
    ("Name", PackageName),
    ("Source", PackageSource),
    ("DownloadLink", str),
    ("LicenseUrl", str),
    ("Unlisted", bool),
    ("DirectDependencies", DependencySet)
    ]) as PackageDetails:
    pass

with typing.NamedTuple("ResolvedPackage", [
    ("Name", PackageName),
    ("Version", SemVerInfo),
    ("Dependencies", DependencySet),
    ("Unlisted", bool),
    ("IsRuntimeDependency", bool),
    ("Settings", InstallSettings),
    ("Source", PackageSource)
    ]) as ResolvedPackage:
    ResolvedPackage.__repr__ = types.MethodType(lambda s: "%s %s" % (s.Name, s.Version))
    # TODO: needs transfer
    ResolvedPackage.__format__ = types.MethodType(lambda s:
            "%s\nDependencies -\n%s\nSource - %s\nInstall Settings\n%s" % (s.Name, s.Dependencies, s.Source, s.Settings)
            )
    ResolvedPackage.HasFrameworkRestrictions = \
        property(lambda s:
                 getExplicitRestriction(s.Settings.FrameworkRestrictions != FrameworkRestriction.NoRestriction
        ))

PackageResolution = typing.Dict[PackageName, ResolvedPackage]

def isIncluded(restriction: FrameworkRestriction, dependency: Dependency) -> bool:
    """
    """
    _, _, dependencyRestrictions = dependency
    dependencyRestrictions = getExplicitRestriction(dependencyRestrictions)
    if dependencyRestrictions == FrameworkRestriction.NoRestriction: return True
    else:
        return bool(
            restriction.RepresentedFrameworks | dependencyRestrictions.RepresentedFrameworks
            )

def filterByRestrictions(restrictions: FrameworkRestrictions, dependencies: DependencySet) -> DependencySet:
    """
    """
    restrictions = getExplicitRestriction(restrictions)
    if restrictions == FrameworkRestriction.NoRestriction: return dependencies
    else:
        return DependencySet(
            filter(isIncluded(restrictions), dependencies)
        )

def isPackageCompatible(dependencies: DependencySet, package: ResolvedPackage) -> bool:
    """
    """
    for name, requirement, restriction in dependencies:
        if name == package.Name and not requirement.IsInRange(package.Version):
            print("Incompatible dependency: %s %s conflicts with resolved version %s" % (name, requirement, package.Version))
            return False
    else:
        return True

class DependencySetFilter:
    isIncluded = isIncluded
    filterByRestrictions = filterByRestrictions
    isPackageCompatible = isPackageCompatible
<commit_msg>fixbug<commit_after>
import types
import typing

Dependency = typing.Tuple("Dependency", [PackageName, VersionRequirement, FrameworkRestrictions])
DependencySet = typing.Set[Dependency]

PackageDetails = typing.NamedTuple("PackageDetails", [
    ("Name", PackageName),
    ("Source", PackageSource),
    ("DownloadLink", str),
    ("LicenseUrl", str),
    ("Unlisted", bool),
    ("DirectDependencies", DependencySet)
    ])

ResolvedPackage = typing.NamedTuple("ResolvedPackage", [
    ("Name", PackageName),
    ("Version", SemVerInfo),
    ("Dependencies", DependencySet),
    ("Unlisted", bool),
    ("IsRuntimeDependency", bool),
    ("Settings", InstallSettings),
    ("Source", PackageSource)
    ])

class ResolvedPackage(ResolvedPackage):

    def __repr__(self):
        return "%s %s" % (s.Name, s.Version)

    # TODO: needs transfer
    def __format__(self):
        return "%s\nDependencies -\n%s\nSource - %s\nInstall Settings\n%s" % (
            s.Name, s.Dependencies, s.Source, s.Settings)

    HasFrameworkRestrictions = \
        property(lambda s:
                 getExplicitRestriction(s.Settings.FrameworkRestrictions != FrameworkRestriction.NoRestriction
        ))

PackageResolution = typing.Dict[PackageName, ResolvedPackage]

def isIncluded(restriction: FrameworkRestriction, dependency: Dependency) -> bool:
    """
    """
    _, _, dependencyRestrictions = dependency
    dependencyRestrictions = getExplicitRestriction(dependencyRestrictions)
    if dependencyRestrictions == FrameworkRestriction.NoRestriction: return True
    else:
        return bool(
            restriction.RepresentedFrameworks | dependencyRestrictions.RepresentedFrameworks
            )

def filterByRestrictions(restrictions: FrameworkRestrictions, dependencies: DependencySet) -> DependencySet:
    """
    """
    restrictions = getExplicitRestriction(restrictions)
    if restrictions == FrameworkRestriction.NoRestriction: return dependencies
    else:
        return DependencySet(
            filter(isIncluded(restrictions), dependencies)
        )

def isPackageCompatible(dependencies: DependencySet, package: ResolvedPackage) -> bool:
    """
    """
    for name, requirement, restriction in dependencies:
        if name == package.Name and not requirement.IsInRange(package.Version):
            print("Incompatible dependency: %s %s conflicts with resolved version %s" % (name, requirement, package.Version))
            return False
    else:
        return True

class DependencySetFilter:
    isIncluded = isIncluded
    filterByRestrictions = filterByRestrictions
    isPackageCompatible = isPackageCompatible
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import re
import socket
import time

from difflib import SequenceMatcher

from lib.core.agent import agent
from lib.core.common import beep
from lib.core.common import getFilteredPageContent
from lib.core.common import getUnicode
from lib.core.common import preparePageForLineComparison
from lib.core.common import randomInt
from lib.core.common import randomStr
from lib.core.common import readInput
from lib.core.common import showStaticWords
from lib.core.common import wasLastRequestError
from lib.core.common import DynamicContentItem
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapNoneDataException
from lib.core.exception import sqlmapUserQuitException
from lib.core.exception import sqlmapSilentQuitException
from lib.core.session import setString
from lib.core.session import setRegexp
from lib.request.connect import Connect as Request

def checkSqlInjection(place, parameter, value, parenthesis):
    """
    This function checks if the GET, POST, Cookie, User-Agent
    parameters are affected by a SQL injection vulnerability and
    identifies the type of SQL injection:

      * Unescaped numeric injection
      * Single quoted string injection
      * Double quoted string injection
    """

    logic = conf.logic
    randInt = randomInt()
    randStr = randomStr()
    prefix = ""
    postfix = ""

    if conf.prefix or conf.postfix:
        if conf.prefix:
            prefix = conf.prefix

        if conf.postfix:
            postfix = conf.postfix

    for case in kb.injections.root.case:
        positive = case.test.positive
        negative = case.test.negative

        if not prefix and not postfix and case.name == "custom":
            continue

        infoMsg  = "testing %s (%s) injection " % (case.desc, logic)
        infoMsg += "on %s parameter '%s'" % (place, parameter)
        logger.info(infoMsg)

        payload = agent.payload(place, parameter, value, positive.format % eval(positive.params))
        trueResult = Request.queryPage(payload, place)

        if trueResult:
            payload = agent.payload(place, parameter, value, negative.format % eval(negative.params))

            falseResult = Request.queryPage(payload, place)

            if not falseResult:
                infoMsg  = "%s parameter '%s' is %s (%s) injectable " % (place, parameter, case.desc, logic)
                infoMsg += "with %d parenthesis" % parenthesis
                logger.info(infoMsg)

                if conf.beep:
                    beep()

                return case.name

    return None

def heuristicCheckSqlInjection(place, parameter, value):
    prefix = ""
    postfix = ""

    if conf.prefix or conf.postfix:
        if conf.prefix:
            prefix = conf.prefix

        if conf.postfix:
            postfix = conf.postfix

    payload = "%s%s%s" % (prefix, randomStr(length=10, alphabet=['"', '\'', ')', '(']), postfix)
    payload = agent.payload(place, parameter, value, payload)
    Request.queryPage(payload, place, raise404=False)
    result = wasLastRequestError()

    infoMsg  = "(error based) heuristics shows that %s " % place
    infoMsg += "parameter '%s' is " % parameter

    if result:
        infoMsg += "injectable (possible DBMS: %s)" % kb.htmlFp[-1]
        logger.info(infoMsg)
    else:
        infoMsg += "not injectable"
        logger.warning(infoMsg)

def checkDynParam(place, parameter, value):
    """
    This function checks if the url parameter is dynamic. If it is
    dynamic, the content of the page differs, otherwise the
    dynamicity might depend on another parameter.
    """

    infoMsg = "testing if %s parameter '%s' is dynamic" % (place, parameter)
    logger.info(infoMsg)

    randInt = randomInt()
    payload = agent.payload(place, parameter, value, getUnicode(randInt))
    dynResult1 = Request.queryPage(payload, place)

    if True == dynResult1:
        return False

    infoMsg = "confirming that %s parameter '%s' is dynamic" % (place, parameter)
    logger.info(infoMsg)

    payload = agent.payload(place, parameter, value, "'%s" % randomStr())
    dynResult2 = Request.queryPage(payload, place)

    payload = agent.payload(place, parameter, value, "\"%s" % randomStr())
    dynResult3 = Request.queryPage(payload, place)

    condition  = True != dynResult2
    condition |= True != dynResult3

    return condition

def checkDynamicContent(firstPage, secondPage):
    """
    This function checks if the provided pages have dynamic content. If they
    are dynamic, proper markings will be made.
    """

    if kb.nullConnection:
        return

    infoMsg = "searching for dynamic content"
    logger.info(infoMsg)

    blocks = SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()
    kb.dynamicMarkings = []

    i = 0
    while i < len(blocks):
        block = blocks[i]
        (_, _, length) = block

        if length <= conf.minMatchBlock:
            blocks.remove(block)

        else:
            i += 1

    if len(blocks) > 0:
        blocks.insert(0, None)
        blocks.append(None)

        for i in xrange(len(blocks) - 1):
            prefix = firstPage[blocks[i][0]:blocks[i][0] + blocks[i][2]] if blocks[i] else None
            postfix = firstPage[blocks[i + 1][0]:blocks[i + 1][0] + blocks[i + 1][2]] if blocks[i + 1] else None

            if prefix is None and blocks[i + 1][0] == 0:
                continue

            if postfix is None and (blocks[i][0] + blocks[i][2] >= len(firstPage)):
                continue

            kb.dynamicMarkings.append((re.escape(prefix[-conf.dynMarkLength:]) if prefix else None, re.escape(postfix[:conf.dynMarkLength]) if postfix else None))

    if len(kb.dynamicMarkings) > 0:
        infoMsg = "dynamic content marked for removal (%d region%s)" % (len(kb.dynamicMarkings), 's' if len(kb.dynamicMarkings) > 1 else '')
        logger.info(infoMsg)

def checkStability():
    """
    This function checks if the URL content is stable requesting the
    same page two times with a small delay within each request to
    assume that it is stable.

    In case the content of the page differs when requesting
    the same page, the dynamicity might depend on other parameters,
    like for instance string matching (--string).
    """

    infoMsg = "testing if the url is stable, wait a few seconds"
    logger.info(infoMsg)

    firstPage, _ = Request.queryPage(content=True)
    time.sleep(1)
    secondPage, _ = Request.queryPage(content=True)

    kb.pageStable = (firstPage == secondPage)

    if kb.pageStable:
        if firstPage:
            logMsg  = "url is stable"
            logger.info(logMsg)
        else:
            errMsg  = "there was an error checking the stability of page "
            errMsg += "because of lack of content. please check the "
            errMsg += "page request results (and probable errors) by "
            errMsg += "using higher verbosity levels"
            raise sqlmapNoneDataException, errMsg

    else:
        warnMsg  = "url is not stable, sqlmap will base the page "
        warnMsg += "comparison on a sequence matcher. If no dynamic nor "
        warnMsg += "injectable parameters are detected, or in case of "
        warnMsg += "junk results, refer to user's manual paragraph "
        warnMsg += "'Page comparison' and provide a string or regular "
        warnMsg += "expression to match on"
        logger.warn(warnMsg)

        message = "how do you want to proceed? [C(ontinue)/s(tring)/r(egex)/q(uit)] "
        test = readInput(message, default="C")

        if test and test[0] in ("q", "Q"):
            raise sqlmapUserQuitException

        elif test and test[0] in ("s", "S"):
            showStaticWords(firstPage, secondPage)

            message = "please enter value for parameter 'string': "
            test = readInput(message)

            if test:
                conf.string = test
                kb.nullConnection = None
            else:
                raise sqlmapSilentQuitException

        elif test and test[0] in ("r", "R"):
            message = "please enter value for parameter 'regex': "
            test = readInput(message)

            if test:
                conf.regex = test
                kb.nullConnection = None
            else:
                raise sqlmapSilentQuitException
        else:
            checkDynamicContent(firstPage, secondPage)

    return kb.pageStable

def checkString():
    if not conf.string:
        return True

    condition = (
                  kb.resumedQueries.has_key(conf.url) and
                  kb.resumedQueries[conf.url].has_key("String") and
                  kb.resumedQueries[conf.url]["String"][:-1] == conf.string
                )

    if condition:
        return True

    infoMsg  = "testing if the provided string is within the "
    infoMsg += "target URL page content"
    logger.info(infoMsg)

    page, _ = Request.queryPage(content=True)

    if conf.string in page:
        setString()
        return True
    else:
        errMsg  = "you provided '%s' as the string to " % conf.string
        errMsg += "match, but such a string is not within the target "
        errMsg += "URL page content, please provide another string."
        logger.error(errMsg)

        return False

def checkRegexp():
    if not conf.regexp:
        return True

    condition = (
                  kb.resumedQueries.has_key(conf.url) and
                  kb.resumedQueries[conf.url].has_key("Regular expression") and
                  kb.resumedQueries[conf.url]["Regular expression"][:-1] == conf.regexp
                )

    if condition:
        return True

    infoMsg  = "testing if the provided regular expression matches within "
    infoMsg += "the target URL page content"
    logger.info(infoMsg)

    page, _ = Request.queryPage(content=True)

    if re.search(conf.regexp, page, re.I | re.M):
        setRegexp()
        return True
    else:
        errMsg  = "you provided '%s' as the regular expression to " % conf.regexp
        errMsg += "match, but such a regular expression does not have any "
        errMsg += "match within the target URL page content, please provide "
        errMsg += "another regular expression."
        logger.error(errMsg)

        return False

def checkNullConnection():
    """
    Reference: http://www.wisec.it/sectou.php?id=472f952d79293
    """

    infoMsg = "testing NULL connection to the target url"
    logger.info(infoMsg)

    try:
        page, headers = Request.getPage(method="HEAD")
        if not page and 'Content-Length' in headers:
            kb.nullConnection = "HEAD"

            infoMsg = "NULL connection is supported with HEAD header"
            logger.info(infoMsg)
        else:
            page, headers = Request.getPage(auxHeaders={"Range":"bytes=-1"})
            if page and len(page) == 1 and 'Content-Range' in headers:
                kb.nullConnection = "Range"

                infoMsg = "NULL connection is supported with GET header "
                infoMsg += "'%s'" % kb.nullConnection
                logger.info(infoMsg)
    except sqlmapConnectionException, errMsg:
        errMsg = getUnicode(errMsg)
        raise sqlmapConnectionException, errMsg

    return kb.nullConnection is not None

def checkConnection():
    try:
        socket.gethostbyname(conf.hostname)
    except socket.gaierror:
        errMsg = "host '%s' does not exist" % conf.hostname
        raise sqlmapConnectionException, errMsg

    infoMsg = "testing connection to the target url"
    logger.info(infoMsg)

    try:
        page, _ = Request.getPage()
        conf.seqMatcher.set_seq1(page if not conf.textOnly else getFilteredPageContent(page))

    except sqlmapConnectionException, errMsg:
        errMsg = getUnicode(errMsg)
        raise sqlmapConnectionException, errMsg

    return True
<commit_msg>added some debug messages<commit_after>#!/usr/bin/env python

"""
$Id$

Copyright (c) 2006-2010 sqlmap developers (http://sqlmap.sourceforge.net/)
See the file 'doc/COPYING' for copying permission
"""

import re
import socket
import time

from difflib import SequenceMatcher

from lib.core.agent import agent
from lib.core.common import beep
from lib.core.common import getFilteredPageContent
from lib.core.common import getUnicode
from lib.core.common import preparePageForLineComparison
from lib.core.common import randomInt
from lib.core.common import randomStr
from lib.core.common import readInput
from lib.core.common import showStaticWords
from lib.core.common import wasLastRequestError
from lib.core.common import DynamicContentItem
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.data import paths
from lib.core.exception import sqlmapConnectionException
from lib.core.exception import sqlmapNoneDataException
from lib.core.exception import sqlmapUserQuitException
from lib.core.exception import sqlmapSilentQuitException
from lib.core.session import setString
from lib.core.session import setRegexp
from lib.request.connect import Connect as Request

def checkSqlInjection(place, parameter, value, parenthesis):
    """
    This function checks if the GET, POST, Cookie, User-Agent
    parameters are affected by a SQL injection vulnerability and
    identifies the type of SQL injection:

      * Unescaped numeric injection
      * Single quoted string injection
      * Double quoted string injection
    """

    logic = conf.logic
    randInt = randomInt()
    randStr = randomStr()
    prefix = ""
    postfix = ""

    if conf.prefix or conf.postfix:
        if conf.prefix:
            prefix = conf.prefix

        if conf.postfix:
            postfix = conf.postfix

    for case in kb.injections.root.case:
        positive = case.test.positive
        negative = case.test.negative

        if not prefix and not postfix and case.name == "custom":
            continue

        infoMsg  = "testing %s (%s) injection " % (case.desc, logic)
        infoMsg += "on %s parameter '%s'" % (place, parameter)
        logger.info(infoMsg)

        payload = agent.payload(place, parameter, value, positive.format % eval(positive.params))
        trueResult = Request.queryPage(payload, place)

        if trueResult:
            payload = agent.payload(place, parameter, value, negative.format % eval(negative.params))

            falseResult = Request.queryPage(payload, place)

            if not falseResult:
                infoMsg  = "%s parameter '%s' is %s (%s) injectable " % (place, parameter, case.desc, logic)
                infoMsg += "with %d parenthesis" % parenthesis
                logger.info(infoMsg)

                if conf.beep:
                    beep()

                return case.name

    return None

def heuristicCheckSqlInjection(place, parameter, value):
    prefix = ""
    postfix = ""

    if conf.prefix or conf.postfix:
        if conf.prefix:
            prefix = conf.prefix

        if conf.postfix:
            postfix = conf.postfix

    payload = "%s%s%s" % (prefix, randomStr(length=10, alphabet=['"', '\'', ')', '(']), postfix)
    payload = agent.payload(place, parameter, value, payload)
    Request.queryPage(payload, place, raise404=False)
    result = wasLastRequestError()

    infoMsg  = "(error based) heuristics shows that %s " % place
    infoMsg += "parameter '%s' is " % parameter

    if result:
        infoMsg += "injectable (possible DBMS: %s)" % kb.htmlFp[-1]
        logger.info(infoMsg)
    else:
        infoMsg += "not injectable"
        logger.warning(infoMsg)

def checkDynParam(place, parameter, value):
    """
    This function checks if the url parameter is dynamic. If it is
    dynamic, the content of the page differs, otherwise the
    dynamicity might depend on another parameter.
    """

    infoMsg = "testing if %s parameter '%s' is dynamic" % (place, parameter)
    logger.info(infoMsg)

    randInt = randomInt()
    payload = agent.payload(place, parameter, value, getUnicode(randInt))
    dynResult1 = Request.queryPage(payload, place)

    if True == dynResult1:
        return False

    infoMsg = "confirming that %s parameter '%s' is dynamic" % (place, parameter)
    logger.info(infoMsg)

    payload = agent.payload(place, parameter, value, "'%s" % randomStr())
    dynResult2 = Request.queryPage(payload, place)

    payload = agent.payload(place, parameter, value, "\"%s" % randomStr())
    dynResult3 = Request.queryPage(payload, place)

    condition  = True != dynResult2
    condition |= True != dynResult3

    return condition

def checkDynamicContent(firstPage, secondPage):
    """
    This function checks if the provided pages have dynamic content. If they
    are dynamic, proper markings will be made.
    """

    if kb.nullConnection:
        debugMsg  = "dynamic content checking skipped "
        debugMsg += "because NULL connection used"
        logger.debug(debugMsg)
        return

    infoMsg = "searching for dynamic content"
    logger.info(infoMsg)

    blocks = SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()
    kb.dynamicMarkings = []

    i = 0
    while i < len(blocks):
        block = blocks[i]
        (_, _, length) = block

        if length <= conf.minMatchBlock:
            blocks.remove(block)

        else:
            i += 1

    if len(blocks) > 0:
        blocks.insert(0, None)
        blocks.append(None)

        for i in xrange(len(blocks) - 1):
            prefix = firstPage[blocks[i][0]:blocks[i][0] + blocks[i][2]] if blocks[i] else None
            postfix = firstPage[blocks[i + 1][0]:blocks[i + 1][0] + blocks[i + 1][2]] if blocks[i + 1] else None

            if prefix is None and blocks[i + 1][0] == 0:
                continue

            if postfix is None and (blocks[i][0] + blocks[i][2] >= len(firstPage)):
                continue

            kb.dynamicMarkings.append((re.escape(prefix[-conf.dynMarkLength:]) if prefix else None, re.escape(postfix[:conf.dynMarkLength]) if postfix else None))

    if len(kb.dynamicMarkings) > 0:
        infoMsg = "dynamic content marked for removal (%d region%s)" % (len(kb.dynamicMarkings), 's' if len(kb.dynamicMarkings) > 1 else '')
        logger.info(infoMsg)

def checkStability():
    """
    This function checks if the URL content is stable requesting the
    same page two times with a small delay within each request to
    assume that it is stable.

    In case the content of the page differs when requesting
    the same page, the dynamicity might depend on other parameters,
    like for instance string matching (--string).
    """

    infoMsg = "testing if the url is stable, wait a few seconds"
    logger.info(infoMsg)

    firstPage, _ = Request.queryPage(content=True)
    time.sleep(1)
    secondPage, _ = Request.queryPage(content=True)

    kb.pageStable = (firstPage == secondPage)

    if kb.pageStable:
        if firstPage:
            logMsg  = "url is stable"
            logger.info(logMsg)
        else:
            errMsg  = "there was an error checking the stability of page "
            errMsg += "because of lack of content. please check the "
            errMsg += "page request results (and probable errors) by "
            errMsg += "using higher verbosity levels"
            raise sqlmapNoneDataException, errMsg

    else:
        warnMsg  = "url is not stable, sqlmap will base the page "
        warnMsg += "comparison on a sequence matcher. If no dynamic nor "
        warnMsg += "injectable parameters are detected, or in case of "
        warnMsg += "junk results, refer to user's manual paragraph "
        warnMsg += "'Page comparison' and provide a string or regular "
        warnMsg += "expression to match on"
        logger.warn(warnMsg)

        message = "how do you want to proceed? [C(ontinue)/s(tring)/r(egex)/q(uit)] "
        test = readInput(message, default="C")

        if test and test[0] in ("q", "Q"):
            raise sqlmapUserQuitException

        elif test and test[0] in ("s", "S"):
            showStaticWords(firstPage, secondPage)

            message = "please enter value for parameter 'string': "
            test = readInput(message)

            if test:
                conf.string = test

                if kb.nullConnection:
                    debugMsg  = "turning off NULL connection "
                    debugMsg += "support because of string checking"
                    logger.debug(debugMsg)

                    kb.nullConnection = None
            else:
                raise sqlmapSilentQuitException

        elif test and test[0] in ("r", "R"):
            message = "please enter value for parameter 'regex': "
            test = readInput(message)

            if test:
                conf.regex = test

                if kb.nullConnection:
                    debugMsg  = "turning off NULL connection "
                    debugMsg += "support because of regex checking"
                    logger.debug(debugMsg)

                    kb.nullConnection = None
            else:
                raise sqlmapSilentQuitException
        else:
            checkDynamicContent(firstPage, secondPage)

    return kb.pageStable

def checkString():
    if not conf.string:
        return True

    condition = (
                  kb.resumedQueries.has_key(conf.url) and
                  kb.resumedQueries[conf.url].has_key("String") and
                  kb.resumedQueries[conf.url]["String"][:-1] == conf.string
                )

    if condition:
        return True

    infoMsg  = "testing if the provided string is within the "
    infoMsg += "target URL page content"
    logger.info(infoMsg)

    page, _ = Request.queryPage(content=True)

    if conf.string in page:
        setString()
        return True
    else:
        errMsg  = "you provided '%s' as the string to " % conf.string
        errMsg += "match, but such a string is not within the target "
        errMsg += "URL page content, please provide another string."
        logger.error(errMsg)

        return False

def checkRegexp():
    if not conf.regexp:
        return True

    condition = (
                  kb.resumedQueries.has_key(conf.url) and
                  kb.resumedQueries[conf.url].has_key("Regular expression") and
                  kb.resumedQueries[conf.url]["Regular expression"][:-1] == conf.regexp
                )

    if condition:
        return True

    infoMsg  = "testing if the provided regular expression matches within "
    infoMsg += "the target URL page content"
    logger.info(infoMsg)

    page, _ = Request.queryPage(content=True)

    if re.search(conf.regexp, page, re.I | re.M):
        setRegexp()
        return True
    else:
        errMsg  = "you provided '%s' as the regular expression to " % conf.regexp
        errMsg += "match, but such a regular expression does not have any "
        errMsg += "match within the target URL page content, please provide "
        errMsg += "another regular expression."
        logger.error(errMsg)

        return False

def checkNullConnection():
    """
    Reference: http://www.wisec.it/sectou.php?id=472f952d79293
    """

    infoMsg = "testing NULL connection to the target url"
    logger.info(infoMsg)

    try:
        page, headers = Request.getPage(method="HEAD")
        if not page and 'Content-Length' in headers:
            kb.nullConnection = "HEAD"

            infoMsg = "NULL connection is supported with HEAD header"
            logger.info(infoMsg)
        else:
            page, headers = Request.getPage(auxHeaders={"Range":"bytes=-1"})
            if page and len(page) == 1 and 'Content-Range' in headers:
                kb.nullConnection = "Range"

                infoMsg = "NULL connection is supported with GET header "
                infoMsg += "'%s'" % kb.nullConnection
                logger.info(infoMsg)
    except sqlmapConnectionException, errMsg:
        errMsg = getUnicode(errMsg)
        raise sqlmapConnectionException, errMsg

    return kb.nullConnection is not None

def checkConnection():
    try:
        socket.gethostbyname(conf.hostname)
    except socket.gaierror:
        errMsg = "host '%s' does not exist" % conf.hostname
        raise sqlmapConnectionException, errMsg

    infoMsg = "testing connection to the target url"
    logger.info(infoMsg)

    try:
        page, _ = Request.getPage()
        conf.seqMatcher.set_seq1(page if not conf.textOnly else getFilteredPageContent(page))

    except sqlmapConnectionException, errMsg:
        errMsg = getUnicode(errMsg)
        raise sqlmapConnectionException, errMsg

    return True
<|endoftext|>