<commit_before>import csv
import gzip
import sys
from django.core.management.base import BaseCommand
from xbrowse.core.variant_filters import get_default_variant_filter
from xbrowse.variant_search.family import get_variants_with_inheritance_mode
from xbrowse_server import mall
from xbrowse_server.base.models import Project, Family
from xbrowse_server.mall import get_mall, get_reference

def get_gene_symbol(variant):
    gene_id = variant.annotation['vep_annotation'][
        variant.annotation['worst_vep_annotation_index']]['gene']
    return get_reference().get_gene_symbol(gene_id)

def get_variants_for_inheritance_for_project(project, inheritance_mode):
    """
    Get the variants for this project / inheritance combo
    Return dict of family -> list of variants
    """

    # create search specification
    # this could theoretically differ by project, if there are different reference populations
    variant_filter = get_default_variant_filter('moderate_impact')
    variant_filter.ref_freqs.append(('g1k_all', 0.01))
    variant_filter.ref_freqs.append(('exac', 0.01))
    variant_filter.ref_freqs.append(('exac-popmax', 0.01))
    variant_filter.ref_freqs.append(('merck-wgs-3793', 0.05))
    quality_filter = {
        'filter': 'pass',
        'min_gq': 20,
        'min_ab': 15,
    }

    # run MendelianVariantSearch for each family, collect results
    family_results = {}
    families = project.get_families()
    for i, family in enumerate(families):
        print("Processing %s - family %s  (%d / %d)" % (inheritance_mode, family.family_id, i+1, len(families)))
        family_results[family] = list(get_variants_with_inheritance_mode(
            get_mall(project.project_id),
            family.xfamily(),
            inheritance_mode,
            variant_filter=variant_filter,
            quality_filter=quality_filter,
            ))

    return family_results

class Command(BaseCommand):

    def handle(self, *args, **options):
        if not args:
            sys.exit("ERROR: please specify project id on the command line")
        if len(args) > 1:
            sys.exit("ERROR: too many args: %s. Only one project id should be provided." % " ".join(args) )

        project_id = args[0]

        # create family_variants.tsv
        family_variants_f = gzip.open('family_variants_%s.tsv.gz' % project_id, 'w')
        writer = csv.writer(family_variants_f, dialect='excel', delimiter='\t')

        header_fields = [
            '#inheritance_mode',
            'project_id',
            'family_id',
            'gene',
            'chrom',
            'pos',
            'ref',
            'alt',
            'rsid',
            'annotation',
            'exac_af',
            '1kg_af',
            'exac_popmax_af',
            'merck_wgs_3793_af',
            '',
            ]

        genotype_headers = [
            'sample_id',
            'str',
            'num_alt',
            'allele_balance',
            'AD',
            'DP',
            'GQ',
            'PL',
        ]

        for i in range(0, 10):
            for h in genotype_headers:
                header_fields.append("genotype%d_%s" % (i, h))

        writer.writerow(header_fields)

        for inheritance_mode in ['homozygous_recessive', 'dominant', 'compound_het', 'de_novo', 'x_linked_recessive']:
            # collect the resources that we'll need here
            annotator = mall.get_annotator()
            custom_population_store = mall.get_custom_population_store()

            project = Project.objects.get(project_id=project_id)
            families = project.get_families()

            # get the variants for this inheritance / project combination
            family_results = get_variants_for_inheritance_for_project(project, inheritance_mode)

            for i, family in enumerate(families):
                for variant in family_results[family]:
                    custom_populations = custom_population_store.get_frequencies(variant.xpos, variant.ref, variant.alt)
                    row = [
                        inheritance_mode,
                        project_id,
                        family.family_id,
                        get_gene_symbol(variant),
                        variant.chr,
                        str(variant.pos),
                        variant.ref,
                        variant.alt,
                        variant.vcf_id,
                        variant.annotation['vep_group'],
                        str(variant.annotation['freqs']['exac']),
                        str(variant.annotation['freqs']['g1k_all']),
                        str(custom_populations.get('exac-popmax', 0.0)),
                        str(custom_populations.get('merck-wgs-3793', 0.0)),
                        '',
                    ]

                    for i, individual in enumerate(family.get_individuals()):
                        if i >= 10:
                            break

                        genotype = variant.get_genotype(individual.indiv_id)
                        genotype_str = "/".join(genotype.alleles) if genotype.alleles else "./."

                        row.extend([
                            individual.indiv_id,
                            genotype_str,
                            genotype.num_alt,
                            genotype.ab,
                            genotype.extras["ad"],
                            genotype.extras["dp"],
                            genotype.gq,
                            genotype.extras["pl"],])

                    writer.writerow(row)

        family_variants_f.close()

<commit_msg>Added asserts<commit_after>import csv
import gzip
import sys
from django.core.management.base import BaseCommand
from xbrowse.core.variant_filters import get_default_variant_filter
from xbrowse.variant_search.family import get_variants_with_inheritance_mode
from xbrowse_server import mall
from xbrowse_server.base.models import Project, Family
from xbrowse_server.mall import get_mall, get_reference

AB_threshold = 15
GQ_threshold = 20
DP_threshold = 10
g1k_freq_threshold = 0.01
exac_freq_threshold = 0.01
exac_popmax_threshold = 0.01
merck_wgs_3793_threshold = 0.05

def get_gene_symbol(variant):
    gene_id = variant.annotation['vep_annotation'][
        variant.annotation['worst_vep_annotation_index']]['gene']
    return get_reference().get_gene_symbol(gene_id)

def get_variants_for_inheritance_for_project(project, inheritance_mode):
    """
    Get the variants for this project / inheritance combo
    Return dict of family -> list of variants
    """

    # create search specification
    # this could theoretically differ by project, if there are different reference populations
    variant_filter = get_default_variant_filter('moderate_impact')
    variant_filter.ref_freqs.append(('g1k_all', g1k_freq_threshold))
    variant_filter.ref_freqs.append(('exac', exac_freq_threshold))
    variant_filter.ref_freqs.append(('exac-popmax', exac_popmax_threshold))
    variant_filter.ref_freqs.append(('merck-wgs-3793', merck_wgs_3793_threshold))
    quality_filter = {
        'filter': 'pass',
        'min_gq': GQ_threshold,
        'min_ab': AB_threshold,
    }

    # run MendelianVariantSearch for each family, collect results
    family_results = {}
    families = project.get_families()
    for i, family in enumerate(families):
        print("Processing %s - family %s  (%d / %d)" % (inheritance_mode, family.family_id, i+1, len(families)))
        family_results[family] = list(get_variants_with_inheritance_mode(
            get_mall(project.project_id),
            family.xfamily(),
            inheritance_mode,
            variant_filter=variant_filter,
            quality_filter=quality_filter,
            ))

    return family_results

class Command(BaseCommand):

    def handle(self, *args, **options):
        if not args:
            sys.exit("ERROR: please specify project id on the command line")
        if len(args) > 1:
            sys.exit("ERROR: too many args: %s. Only one project id should be provided." % " ".join(args) )

        project_id = args[0]

        # create family_variants.tsv
        family_variants_f = gzip.open('family_variants_%s.tsv.gz' % project_id, 'w')
        writer = csv.writer(family_variants_f, dialect='excel', delimiter='\t')

        header_fields = [
            '#inheritance_mode',
            'project_id',
            'family_id',
            'gene',
            'chrom',
            'pos',
            'ref',
            'alt',
            'rsid',
            'annotation',
            'exac_af',
            '1kg_af',
            'exac_popmax_af',
            'merck_wgs_3793_af',
            '',
            ]

        genotype_headers = [
            'sample_id',
            'str',
            'num_alt',
            'allele_balance',
            'AD',
            'DP',
            'GQ',
            'PL',
        ]

        for i in range(0, 10):
            for h in genotype_headers:
                header_fields.append("genotype%d_%s" % (i, h))

        writer.writerow(header_fields)

        for inheritance_mode in ['homozygous_recessive', 'dominant', 'compound_het', 'de_novo', 'x_linked_recessive']:
            # collect the resources that we'll need here
            annotator = mall.get_annotator()
            custom_population_store = mall.get_custom_population_store()

            project = Project.objects.get(project_id=project_id)
            families = project.get_families()

            # get the variants for this inheritance / project combination
            family_results = get_variants_for_inheritance_for_project(project, inheritance_mode)

            for i, family in enumerate(families):
                for variant in family_results[family]:
                    custom_populations = custom_population_store.get_frequencies(variant.xpos, variant.ref, variant.alt)
                    exac_freq = variant.annotation['freqs']['exac'],
                    g1k_freq = variant.annotation['freqs']['g1k_all']
                    exac_popmax_freq =  custom_populations.get('exac-popmax', 0.0)
                    merck_wgs_3793_freq = custom_populations.get('merck-wgs-3793', 0.0)

                    assert exac_freq <= exac_freq_threshold
                    assert g1k_freq <= g1k_freq_threshold
                    assert exac_popmax_freq <= exac_popmax_threshold
                    assert merck_wgs_3793_freq <= merck_wgs_3793_threshold

                    row = [
                        inheritance_mode,
                        project_id,
                        family.family_id,
                        get_gene_symbol(variant),
                        variant.chr,
                        str(variant.pos),
                        variant.ref,
                        variant.alt,
                        variant.vcf_id,
                        variant.annotation['vep_group'],
                        exac_freq,
                        g1k_freq,
                        exac_popmax_freq,
                        merck_wgs_3793_freq,
                        '',
                    ]

                    for i, individual in enumerate(family.get_individuals()):
                        if i >= 10:
                            break

                        genotype = variant.get_genotype(individual.indiv_id)

                        assert genotype.filter == "pass", "%s %s - filter is %s " % (variant.chr, variant.pos, genotype.filter)
                        assert genotype.gq >= GQ_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.gq)
                        assert genotype.dp >= DP_threshold, "%s %s - GQ is %s " % (variant.chr, variant.pos, genotype.extras["dp"])
                        if genotype.num_alt == 1:
                            assert genotype.ab >= AB_threshold/100., "%s %s - AB is %s " % (variant.chr, variant.pos, genotype.ab)

                        genotype_str = "/".join(genotype.alleles) if genotype.alleles else "./."

                        row.extend([
                            individual.indiv_id,
                            genotype_str,
                            genotype.num_alt,
                            genotype.ab,
                            genotype.extras["ad"],
                            genotype.extras["dp"],
                            genotype.gq,
                            genotype.extras["pl"],])

                    writer.writerow(row)
                    family_variants_f.flush()

        family_variants_f.close()

<|endoftext|>


<commit_before>from os.path import abspath, dirname, join
from configurations import Configuration

PROJECT_ROOT = dirname(dirname(abspath(__file__)))
PROJECT_NAME = '{{cookiecutter.project_name}}'

class RedisCache(object):
    CACHES = {
        'default': {
            'BACKEND': 'redis_cache.RedisCache',
            'LOCATION': '127.0.0.1:6379',
            'OPTIONS': {
                'DB': 1,
                'PARSER_CLASS': 'redis.connection.HiredisParser'
            },
        },
    }

class Common(Configuration):
    ADMINS = (
        ('{{cookiecutter.author_name}}', '{{cookiecutter.author_email}}'),
    )

    MANAGERS = ADMINS

    # SECURITY WARNING: keep the secret key used in production secret!
    SECRET_KEY = '{{cookiecutter.secret_key}}'

    # SECURITY WARNING: don't run with debug turned on in production!
    DEBUG = True

    TEMPLATE_DEBUG = True
    ALLOWED_HOSTS = []

    INSTALLED_APPS = [
        'django.contrib.admin',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.messages',
        'django.contrib.staticfiles',
        'raven.contrib.django.raven_compat',
        'debug_toolbar',
        'floppyforms',
        'django_extensions',
        'clear_cache',
    ]

    MIDDLEWARE_CLASSES = [
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
        'debug_toolbar.middleware.DebugToolbarMiddleware',
    ]

    ROOT_URLCONF = '{{cookiecutter.project_name}}.urls'

    WSGI_APPLICATION = '{{cookiecutter.project_name}}.wsgi.application'

    # Database
    # https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#databases
    # http://django-configurations.readthedocs.org/en/latest/values/#configurations.values.DatabaseURLValue

    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'dev.sqlite',
        }
    }

    # Internationalization
    # https://docs.djangoproject.com/en/{{ docs_version }}/topics/i18n/

    LANGUAGE_CODE = 'en-GB'

    TIME_ZONE = '{{cookiecutter.time_zone}}'

    USE_I18N = True

    USE_L10N = True

    USE_TZ = True

    # Static files (CSS, JavaScript, Images)
    # https://docs.djangoproject.com/en/{{ docs_version }}/howto/static-files/

    STATIC_URL = '/static/'
    STATIC_ROOT = join(PROJECT_ROOT, 'static_root')

    MEDIA_URL = '/media/'
    MEDIA_ROOT = join(PROJECT_ROOT, 'media')

    # Additional locations of static files
    STATICFILES_DIRS = [
        join(PROJECT_ROOT, 'static')
    ]

    TEMPLATE_DIRS = [
        join(PROJECT_ROOT, 'templates')
    ]

    FIXTURE_DIRS = [
        join(PROJECT_ROOT, 'fixtures')
    ]

    LOGGING = {
        'version': 1,
        'disable_existing_loggers': True,
        'root': {
            'level': 'WARNING',
            'handlers': ['sentry'],
        },
        'formatters': {
            'verbose': {
                'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
            },
        },
        'handlers': {
            'sentry': {
                'level': 'ERROR',
                'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
            },
            'console': {
                'level': 'DEBUG',
                'class': 'logging.StreamHandler',
                'formatter': 'verbose'
            }
        },
        'loggers': {
            'django.db.backends': {
                'level': 'ERROR',
                'handlers': ['console'],
                'propagate': False,
            },
            'raven': {
                'level': 'DEBUG',
                'handlers': ['console'],
                'propagate': False,
            },
            'sentry.errors': {
                'level': 'DEBUG',
                'handlers': ['console'],
                'propagate': False,
            },
        },
    }

    # App settings

    # django-debug-toolbar
    DEBUG_TOOLBAR_CONFIG = {'INTERCEPT_REDIRECTS': False}
    INTERNAL_IPS = ('127.0.0.1',)

class Dev(Common):
    DEBUG = True
    TEMPLATE_DEBUG = DEBUG
    EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
    EMAIL_FILE_PATH = '/tmp/app-emails'

class Deployed(RedisCache, Common):
    """
    Settings which are for a non local deployment, served behind nginx.
    """
    # django-debug-toolbar will throw an ImproperlyConfigured exception if DEBUG is
    # ever turned on when run with a WSGI server
    DEBUG_TOOLBAR_PATCH_SETTINGS = False

    PUBLIC_ROOT = join(PROJECT_ROOT, '../public/')
    STATIC_ROOT = join(PUBLIC_ROOT, 'static')
    MEDIA_ROOT = join(PUBLIC_ROOT, 'media')
    COMPRESS_OUTPUT_DIR = ''

    EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
    EMAIL_HOST = 'smtp.sendgrid.net'
    EMAIL_PORT = 587
    EMAIL_USE_TLS = True
    EMAIL_HOST_USER = '{{cookiecutter.email_user}}'
    EMAIL_HOST_PASSWORD = '{{cookiecutter.email_password}}'
    DEFAULT_FROM_EMAIL = ''
    SERVER_EMAIL = ''

class Stage(Deployed):
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.postgresql_psycopg2',
            'NAME': '',
            'USER': '',
            'PASSWORD': '',
            'HOST': 'localhost',
            'OPTIONS': {
                'autocommit': True,  # see https://docs.djangoproject.com/en/dev/ref/databases/#autocommit-mode
            }
        }
    }

class Prod(Deployed):
    DEBUG = False
    TEMPLATE_DEBUG = DEBUG

    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.postgresql_psycopg2',
            'NAME': '',
            'USER': '',
            'PASSWORD': '',
            'HOST': 'localhost',
            'OPTIONS': {
                'autocommit': True,  # see https://docs.djangoproject.com/en/dev/ref/databases/#autocommit-mode
            }
        }
    }

    ALLOWED_HOSTS = ['.{{cookiecutter.domain_name}}', ]  # add deployment domain here

    RAVEN_CONFIG = {
        'dsn': '{{cookiecutter.sentry_dsn}}'
    }
<commit_msg>Added bower path to STATICFILES_DIRS.<commit_after>from os.path import abspath, dirname, join
from configurations import Configuration

PROJECT_ROOT = dirname(dirname(abspath(__file__)))
PROJECT_NAME = '{{cookiecutter.project_name}}'

class RedisCache(object):
    CACHES = {
        'default': {
            'BACKEND': 'redis_cache.RedisCache',
            'LOCATION': '127.0.0.1:6379',
            'OPTIONS': {
                'DB': 1,
                'PARSER_CLASS': 'redis.connection.HiredisParser'
            },
        },
    }

class Common(Configuration):
    ADMINS = (
        ('{{cookiecutter.author_name}}', '{{cookiecutter.author_email}}'),
    )

    MANAGERS = ADMINS

    # SECURITY WARNING: keep the secret key used in production secret!
    SECRET_KEY = '{{cookiecutter.secret_key}}'

    # SECURITY WARNING: don't run with debug turned on in production!
    DEBUG = True

    TEMPLATE_DEBUG = True
    ALLOWED_HOSTS = []

    INSTALLED_APPS = [
        'django.contrib.admin',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.messages',
        'django.contrib.staticfiles',
        'raven.contrib.django.raven_compat',
        'debug_toolbar',
        'floppyforms',
        'django_extensions',
        'clear_cache',
    ]

    MIDDLEWARE_CLASSES = [
        'django.contrib.sessions.middleware.SessionMiddleware',
        'django.middleware.common.CommonMiddleware',
        'django.middleware.csrf.CsrfViewMiddleware',
        'django.contrib.auth.middleware.AuthenticationMiddleware',
        'django.contrib.messages.middleware.MessageMiddleware',
        'django.middleware.clickjacking.XFrameOptionsMiddleware',
        'debug_toolbar.middleware.DebugToolbarMiddleware',
    ]

    ROOT_URLCONF = '{{cookiecutter.project_name}}.urls'

    WSGI_APPLICATION = '{{cookiecutter.project_name}}.wsgi.application'

    # Database
    # https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#databases
    # http://django-configurations.readthedocs.org/en/latest/values/#configurations.values.DatabaseURLValue

    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'dev.sqlite',
        }
    }

    # Internationalization
    # https://docs.djangoproject.com/en/{{ docs_version }}/topics/i18n/

    LANGUAGE_CODE = 'en-GB'

    TIME_ZONE = '{{cookiecutter.time_zone}}'

    USE_I18N = True

    USE_L10N = True

    USE_TZ = True

    # Static files (CSS, JavaScript, Images)
    # https://docs.djangoproject.com/en/{{ docs_version }}/howto/static-files/

    STATIC_URL = '/static/'
    STATIC_ROOT = join(PROJECT_ROOT, 'static_root')

    MEDIA_URL = '/media/'
    MEDIA_ROOT = join(PROJECT_ROOT, 'media')

    # Additional locations of static files
    STATICFILES_DIRS = [
        join(PROJECT_ROOT, 'static'),
        join(PROJECT_ROOT, 'bower_components'),
    ]

    TEMPLATE_DIRS = [
        join(PROJECT_ROOT, 'templates')
    ]

    FIXTURE_DIRS = [
        join(PROJECT_ROOT, 'fixtures')
    ]

    LOGGING = {
        'version': 1,
        'disable_existing_loggers': True,
        'root': {
            'level': 'WARNING',
            'handlers': ['sentry'],
        },
        'formatters': {
            'verbose': {
                'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
            },
        },
        'handlers': {
            'sentry': {
                'level': 'ERROR',
                'class': 'raven.contrib.django.raven_compat.handlers.SentryHandler',
            },
            'console': {
                'level': 'DEBUG',
                'class': 'logging.StreamHandler',
                'formatter': 'verbose'
            }
        },
        'loggers': {
            'django.db.backends': {
                'level': 'ERROR',
                'handlers': ['console'],
                'propagate': False,
            },
            'raven': {
                'level': 'DEBUG',
                'handlers': ['console'],
                'propagate': False,
            },
            'sentry.errors': {
                'level': 'DEBUG',
                'handlers': ['console'],
                'propagate': False,
            },
        },
    }

    # App settings

    # django-debug-toolbar
    DEBUG_TOOLBAR_CONFIG = {'INTERCEPT_REDIRECTS': False}
    INTERNAL_IPS = ('127.0.0.1',)

class Dev(Common):
    DEBUG = True
    TEMPLATE_DEBUG = DEBUG
    EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
    EMAIL_FILE_PATH = '/tmp/app-emails'

class Deployed(RedisCache, Common):
    """
    Settings which are for a non local deployment, served behind nginx.
    """
    # django-debug-toolbar will throw an ImproperlyConfigured exception if DEBUG is
    # ever turned on when run with a WSGI server
    DEBUG_TOOLBAR_PATCH_SETTINGS = False

    PUBLIC_ROOT = join(PROJECT_ROOT, '../public/')
    STATIC_ROOT = join(PUBLIC_ROOT, 'static')
    MEDIA_ROOT = join(PUBLIC_ROOT, 'media')
    COMPRESS_OUTPUT_DIR = ''

    EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
    EMAIL_HOST = 'smtp.sendgrid.net'
    EMAIL_PORT = 587
    EMAIL_USE_TLS = True
    EMAIL_HOST_USER = '{{cookiecutter.email_user}}'
    EMAIL_HOST_PASSWORD = '{{cookiecutter.email_password}}'
    DEFAULT_FROM_EMAIL = ''
    SERVER_EMAIL = ''

class Stage(Deployed):
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.postgresql_psycopg2',
            'NAME': '',
            'USER': '',
            'PASSWORD': '',
            'HOST': 'localhost',
            'OPTIONS': {
                'autocommit': True,  # see https://docs.djangoproject.com/en/dev/ref/databases/#autocommit-mode
            }
        }
    }

class Prod(Deployed):
    DEBUG = False
    TEMPLATE_DEBUG = DEBUG

    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.postgresql_psycopg2',
            'NAME': '',
            'USER': '',
            'PASSWORD': '',
            'HOST': 'localhost',
            'OPTIONS': {
                'autocommit': True,  # see https://docs.djangoproject.com/en/dev/ref/databases/#autocommit-mode
            }
        }
    }

    ALLOWED_HOSTS = ['.{{cookiecutter.domain_name}}', ]  # add deployment domain here

    RAVEN_CONFIG = {
        'dsn': '{{cookiecutter.sentry_dsn}}'
    }
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.shortcuts import get_object_or_404, redirect
from django.views.generic import TemplateView
from endless_pagination.views import AjaxListView

from accounts.models import HierarchicUser
from .jobs import dossier_to_pdf
from libretto.models import Source, Oeuvre
from libretto.views import (
    PublishedListView, PublishedDetailView, EvenementGeoJson, EvenementExport,
    BaseEvenementListView)
from .models import CategorieDeDossiers, DossierDEvenements
from common.utils.export import launch_export

class CategorieDeDossiersList(PublishedListView):
    model = CategorieDeDossiers
    has_frontend_admin = False

class DossierDEvenementsDetail(PublishedDetailView):
    model = DossierDEvenements

class DossierDEvenementsViewMixin(object):
    view_name = 'dossierdevenements_data_detail'
    enable_default_page = False

    def get_queryset(self):
        self.object = get_object_or_404(DossierDEvenements, **self.kwargs)
        if not self.object.can_be_viewed(self.request):
            raise PermissionDenied
        return super(DossierDEvenementsViewMixin, self).get_queryset(
            base_filter=Q(pk__in=self.object.get_queryset()))

    def get_export_url(self):
        return reverse('dossierdevenements_data_export', kwargs=self.kwargs)

    def get_geojson_url(self):
        return reverse('dossierdevenements_data_geojson', kwargs=self.kwargs)

    def get_context_data(self, **kwargs):
        data = super(DossierDEvenementsViewMixin,
                     self).get_context_data(**kwargs)
        data['object'] = self.object
        return data

    def get_success_url(self):
        return reverse(self.view_name, kwargs=self.kwargs)

class DossierDEvenementsDataDetail(DossierDEvenementsViewMixin,
                                   AjaxListView, BaseEvenementListView):
    template_name = 'dossiers/dossierdevenements_data_detail.html'

class DossierDEvenementsDataExport(DossierDEvenementsViewMixin,
                                   EvenementExport):
    pass

class DossierDEvenementsDataGeoJson(DossierDEvenementsViewMixin,
                                    EvenementGeoJson):
    pass

class DossierDEvenementsDetailXeLaTeX(DossierDEvenementsDetail):
    def get_object(self, queryset=None):
        if not self.request.user.is_authenticated():
            raise PermissionDenied
        return super(DossierDEvenementsDetailXeLaTeX,
                     self).get_object(queryset)

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        launch_export(dossier_to_pdf, request, self.object.pk, 'PDF',
                      'du dossier « %s »' % self.object)
        return redirect(self.object.get_absolute_url())

class OperaComiquePresentation(TemplateView):
    template_name = 'dossiers/opera_comique_presentation.html'

    def get_context_data(self, **kwargs):
        context = super(OperaComiquePresentation,
                        self).get_context_data(**kwargs)
        context['oc_user'] = HierarchicUser.objects.get(pk=103)
        return context

class OperaComiqueListView(PublishedListView):
    model = Source
    template_name = 'dossiers/opera_comique.html'

    def get_queryset(self):
        qs = super(OperaComiqueListView, self).get_queryset()
        return qs.filter(owner_id=103)

    def get_context_data(self, **kwargs):
        context = super(OperaComiqueListView, self).get_context_data(**kwargs)
        qs = context['object_list']
        oeuvres = (
            Oeuvre.objects.filter(sources__in=qs).distinct()
            .select_related('genre', 'creation_lieu', 'creation_lieu__nature')
            .prefetch_related('auteurs__individu', 'auteurs__profession',
                              'caracteristiques__type'))
        if self.request.GET.get('order_by') == 'creation_date':
            oeuvres = oeuvres.order_by('creation_date')
        else:
            oeuvres = oeuvres.order_by(*Oeuvre._meta.ordering)
        context['oeuvres'] = oeuvres
        return context
<commit_msg>Retire un prefetch de caractéristiques.<commit_after># coding: utf-8

from __future__ import unicode_literals

from django.core.exceptions import PermissionDenied
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.shortcuts import get_object_or_404, redirect
from django.views.generic import TemplateView
from endless_pagination.views import AjaxListView

from accounts.models import HierarchicUser
from .jobs import dossier_to_pdf
from libretto.models import Source, Oeuvre
from libretto.views import (
    PublishedListView, PublishedDetailView, EvenementGeoJson, EvenementExport,
    BaseEvenementListView)
from .models import CategorieDeDossiers, DossierDEvenements
from common.utils.export import launch_export

class CategorieDeDossiersList(PublishedListView):
    model = CategorieDeDossiers
    has_frontend_admin = False

class DossierDEvenementsDetail(PublishedDetailView):
    model = DossierDEvenements

class DossierDEvenementsViewMixin(object):
    view_name = 'dossierdevenements_data_detail'
    enable_default_page = False

    def get_queryset(self):
        self.object = get_object_or_404(DossierDEvenements, **self.kwargs)
        if not self.object.can_be_viewed(self.request):
            raise PermissionDenied
        return super(DossierDEvenementsViewMixin, self).get_queryset(
            base_filter=Q(pk__in=self.object.get_queryset()))

    def get_export_url(self):
        return reverse('dossierdevenements_data_export', kwargs=self.kwargs)

    def get_geojson_url(self):
        return reverse('dossierdevenements_data_geojson', kwargs=self.kwargs)

    def get_context_data(self, **kwargs):
        data = super(DossierDEvenementsViewMixin,
                     self).get_context_data(**kwargs)
        data['object'] = self.object
        return data

    def get_success_url(self):
        return reverse(self.view_name, kwargs=self.kwargs)

class DossierDEvenementsDataDetail(DossierDEvenementsViewMixin,
                                   AjaxListView, BaseEvenementListView):
    template_name = 'dossiers/dossierdevenements_data_detail.html'

class DossierDEvenementsDataExport(DossierDEvenementsViewMixin,
                                   EvenementExport):
    pass

class DossierDEvenementsDataGeoJson(DossierDEvenementsViewMixin,
                                    EvenementGeoJson):
    pass

class DossierDEvenementsDetailXeLaTeX(DossierDEvenementsDetail):
    def get_object(self, queryset=None):
        if not self.request.user.is_authenticated():
            raise PermissionDenied
        return super(DossierDEvenementsDetailXeLaTeX,
                     self).get_object(queryset)

    def get(self, request, *args, **kwargs):
        self.object = self.get_object()
        launch_export(dossier_to_pdf, request, self.object.pk, 'PDF',
                      'du dossier « %s »' % self.object)
        return redirect(self.object.get_absolute_url())

class OperaComiquePresentation(TemplateView):
    template_name = 'dossiers/opera_comique_presentation.html'

    def get_context_data(self, **kwargs):
        context = super(OperaComiquePresentation,
                        self).get_context_data(**kwargs)
        context['oc_user'] = HierarchicUser.objects.get(pk=103)
        return context

class OperaComiqueListView(PublishedListView):
    model = Source
    template_name = 'dossiers/opera_comique.html'

    def get_queryset(self):
        qs = super(OperaComiqueListView, self).get_queryset()
        return qs.filter(owner_id=103)

    def get_context_data(self, **kwargs):
        context = super(OperaComiqueListView, self).get_context_data(**kwargs)
        qs = context['object_list']
        oeuvres = (
            Oeuvre.objects.filter(sources__in=qs).distinct()
            .select_related('genre', 'creation_lieu', 'creation_lieu__nature')
            .prefetch_related('auteurs__individu', 'auteurs__profession'))
        if self.request.GET.get('order_by') == 'creation_date':
            oeuvres = oeuvres.order_by('creation_date')
        else:
            oeuvres = oeuvres.order_by(*Oeuvre._meta.ordering)
        context['oeuvres'] = oeuvres
        return context
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    KMEE, KM Enterprising Engineering
#    Copyright (C) 2014 - Michell Stuttgart Faria (<http://www.kmee.com.br>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models

class SaleAdvancePaymentInvoice(models.Model):
    _inherit = 'sale.advance.payment.inv'

    def _get_line_qty(self, cr, uid, line, context=None):
        if (line.order_id.invoice_quantity == 'order'):
            if line.product_uos:
                return line.product_uos_qty or 0.0
        return line.product_uom_qty

    def _get_line_uom(self, cr, uid, line, context=None):
        if (line.order_id.invoice_quantity == 'order'):
            if line.product_uos:
                return line.product_uos.id
        return line.product_uom.id

    def _prepare_advance_invoice_vals(self, cr, uid, ids, context=None):

        # Lista de campos
        result = super(
            SaleAdvancePaymentInvoice, self)._prepare_advance_invoice_vals(
                cr, uid, ids, context)

        if context is None:
            context = {}

        sale_obj = self.pool['sale.order']
        wizard = self.browse(cr, uid, ids[0], context)
        # get invoice type
        payment_type = wizard.advance_payment_method

        for res in result:
            for sale in sale_obj.browse(cr, uid, [res[0]], context=context):

                # Verificamos o tipo de fatura
                if payment_type == 'percentage':
                    percent = wizard.amount / 100.0
                else:
                    percent = wizard.amount / sale.amount_gross

                # Aux list of invoice
                list_invoice = []

                # Invoice dict in res
                invoice_dict = res[1]['invoice_line'][0][2]

                for order_line in sale.order_line:

                    # Objeto sale.order.line
                    order_line_obj = self.pool.get('sale.order.line')

                    res_aux = {}
                    res_aux = (order_line_obj.
                               l10n_br_sale_prepare_order_line_invoice_line(
                                   cr, uid, order_line, res_aux, context))

                    res_aux = order_line_obj.\
                        l10n_br_sale_product_prepare_order_line_invoice_line(
                            cr, uid, order_line, res_aux, context)

                    tax_list_id = []

                    uosqty = self._get_line_qty(
                        cr, uid, order_line, context=context)
                    uos_id = self._get_line_uom(
                        cr, uid, order_line, context=context)

                    # Add the ID of tax of order line
                    for tax in order_line.tax_id:
                        tax_list_id.append(tax.id)

                    # create the invoice
                    inv_line_values = {
                        'name': order_line.name,
                        'origin': invoice_dict['origin'],
                        'account_id': invoice_dict['account_id'],
                        'price_unit': order_line.price_unit,
                        'quantity': (percent * uosqty) or 1.0,
                        'discount': invoice_dict['discount'],
                        'uos_id': uos_id,
                        'product_id': order_line.product_id.id,
                        'invoice_line_tax_id': [(6, 0, tax_list_id)],
                        'account_analytic_id': invoice_dict.get(
                            'account_analytic_id'),
                        'fiscal_category_id': res_aux.get(
                            'fiscal_category_id'),
                        'fiscal_position': res_aux.get('fiscal_position'),
                        'cfop_id': res_aux.get('cfop_id'),
                    }

                    list_invoice.append((0, 0, inv_line_values))

                res[1]['invoice_line'] = list_invoice
                res[1]['comment'] = invoice_dict['name']
        return result
<commit_msg>FIX: AccessError (Modelo de documento: sale.advance.payment.inv)<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    KMEE, KM Enterprising Engineering
#    Copyright (C) 2014 - Michell Stuttgart Faria (<http://www.kmee.com.br>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models

class SaleAdvancePaymentInvoice(models.TransientModel):
    _inherit = 'sale.advance.payment.inv'

    def _get_line_qty(self, cr, uid, line, context=None):
        if (line.order_id.invoice_quantity == 'order'):
            if line.product_uos:
                return line.product_uos_qty or 0.0
        return line.product_uom_qty

    def _get_line_uom(self, cr, uid, line, context=None):
        if (line.order_id.invoice_quantity == 'order'):
            if line.product_uos:
                return line.product_uos.id
        return line.product_uom.id

    def _prepare_advance_invoice_vals(self, cr, uid, ids, context=None):

        # Lista de campos
        result = super(
            SaleAdvancePaymentInvoice, self)._prepare_advance_invoice_vals(
                cr, uid, ids, context)

        if context is None:
            context = {}

        sale_obj = self.pool['sale.order']
        wizard = self.browse(cr, uid, ids[0], context)
        # get invoice type
        payment_type = wizard.advance_payment_method

        for res in result:
            for sale in sale_obj.browse(cr, uid, [res[0]], context=context):

                # Verificamos o tipo de fatura
                if payment_type == 'percentage':
                    percent = wizard.amount / 100.0
                else:
                    percent = wizard.amount / sale.amount_gross

                # Aux list of invoice
                list_invoice = []

                # Invoice dict in res
                invoice_dict = res[1]['invoice_line'][0][2]

                for order_line in sale.order_line:

                    # Objeto sale.order.line
                    order_line_obj = self.pool.get('sale.order.line')

                    res_aux = {}
                    res_aux = (order_line_obj.
                               l10n_br_sale_prepare_order_line_invoice_line(
                                   cr, uid, order_line, res_aux, context))

                    res_aux = order_line_obj.\
                        l10n_br_sale_product_prepare_order_line_invoice_line(
                            cr, uid, order_line, res_aux, context)

                    tax_list_id = []

                    uosqty = self._get_line_qty(
                        cr, uid, order_line, context=context)
                    uos_id = self._get_line_uom(
                        cr, uid, order_line, context=context)

                    # Add the ID of tax of order line
                    for tax in order_line.tax_id:
                        tax_list_id.append(tax.id)

                    # create the invoice
                    inv_line_values = {
                        'name': order_line.name,
                        'origin': invoice_dict['origin'],
                        'account_id': invoice_dict['account_id'],
                        'price_unit': order_line.price_unit,
                        'quantity': (percent * uosqty) or 1.0,
                        'discount': invoice_dict['discount'],
                        'uos_id': uos_id,
                        'product_id': order_line.product_id.id,
                        'invoice_line_tax_id': [(6, 0, tax_list_id)],
                        'account_analytic_id': invoice_dict.get(
                            'account_analytic_id'),
                        'fiscal_category_id': res_aux.get(
                            'fiscal_category_id'),
                        'fiscal_position': res_aux.get('fiscal_position'),
                        'cfop_id': res_aux.get('cfop_id'),
                    }

                    list_invoice.append((0, 0, inv_line_values))

                res[1]['invoice_line'] = list_invoice
                res[1]['comment'] = invoice_dict['name']
        return result
<|endoftext|>


<commit_before># coding=utf-8

import re
import os
import pprint
from collections import OrderedDict

from bs4 import BeautifulSoup

TEMPLATE = """\
import re
from collections import OrderedDict
data = """

TEMPLATE_END = """\

for lang, grps in data.iteritems():
    for grp in grps.iterkeys():
        if data[lang][grp]["pattern"]:
            data[lang][grp]["pattern"] = re.compile(data[lang][grp]["pattern"])
"""

SZ_FIX_DATA = {
    "eng": {
        "PartialWordsAlways": {
            u"°x°": u"%",
            u"compiete": u"complete",
            u"Âs": u"'s",
            u"ÃÂs": u"'s",
            u"a/ion": u"ation",
            u"at/on": u"ation",
            u"l/an": u"lian",
            u"lljust": u"ll just",
            u". L ": u". I ",
            u"L know": u"I know",
            u"L should": u"I should",
            u"L do": u"I do",
            u"L would": u"I would",
            u"L could": u"I could",
            u"L can": u"I can",
            u"L happen": u"I happen",
            u"L might": u"I might",
            u"L have ": u"I have",
            u"L had": u"I had",
            u"L want": u"I want",
            u"L was": u"I was",
            u"L am": u"I am",
            u"L suggest": u"I suggest",
            u"L think": u"I think",
            u"L reckon": u"I reckon",
            u"L like": u"I like",
            u"L love": u"I love",
            u"L don't": u"I don't",
            u"L won't": u"I won't",
        },
        "WholeWords": {
            u"I'11": u"I'll",
            u"Tun": u"Run",
            u"pan'": u"part",
            u"al'": u"at",
            u"a re": u"are",
            u"wail'": u"wait",
            u"he)'": u"hey",
            u"he)\"": u"hey",
            u"He)'": u"Hey",
            u"He)\"": u"Hey",
            u"Yea h": u"Yeah",
            u"yea h": u"yeah",
            u"h is": u"his",
            u" 're ": u"'re ",
            u"LAst": u"Last",
            u"forthis": u"for this",
        }
    },
    "nld": {
        "PartialWordsAlways": {
            u"ט": u"è",
            u"י": u"é",
            u"כ": u"ë",
            u"צ": u"ë",
            u"ן": u"ï",
            u"ף": u"ó",
            u"א": u"à",
            u"Iֻ": u"I",
            u"č": u"è",
            u"פ": u"o",
            u"ם": u"i",
        },
    },
    "swe": {
        "PartialWordsAlways": {
            u"ĺ": u"å",
            u"Ĺ": u"Å",
        }
    }
}

SZ_FIX_DATA_GLOBAL = {
    "PartialWordsAlways": {
        u"¶¶": u"♫",
        u"¶": u"♪"
    }
}

if __name__ == "__main__":
    cur_dir = os.path.dirname(os.path.realpath(__file__))
    xml_dir = os.path.join(cur_dir, "xml")
    file_list = os.listdir(xml_dir)

    data = {}

    for fn in file_list:
        if fn.endswith("_OCRFixReplaceList.xml"):
            lang = fn.split("_")[0]
            soup = BeautifulSoup(open(os.path.join(xml_dir, fn)), "xml")

            fetch_data = (
                    # group, item_name, pattern
                    ("WholeLines", "Line", None),
                    ("WholeWords", "Word", lambda d: (ur"(?um)\b(?:" + u"|".join([re.escape(k) for k in d.keys()])
                                                      + ur')\b') if d else None),
                    ("PartialWordsAlways", "WordPart", None),
                    ("PartialLines", "LinePart", lambda d: (ur"(?um)(?:(?<=\s)|(?<=^)|(?<=\b))(?:" +
                                                            u"|".join([re.escape(k) for k in d.keys()]) +
                                                            ur")(?:(?=\s)|(?=$)|(?=\b))") if d else None),
                    ("BeginLines", "Beginning", lambda d: (ur"(?um)^(?:"+u"|".join([re.escape(k) for k in d.keys()])
                                                           + ur')') if d else None),
                    ("EndLines", "Ending", lambda d: (ur"(?um)(?:" + u"|".join([re.escape(k) for k in d.keys()]) +
                                                      ur")$") if d else None,),
            )

            data[lang] = dict((grp, {"data": OrderedDict(), "pattern": None}) for grp, item_name, pattern in fetch_data)

            for grp, item_name, pattern in fetch_data:
                for grp_data in soup.find_all(grp):
                    for line in grp_data.find_all(item_name):
                        data[lang][grp]["data"][line["from"]] = line["to"]

                # add our own dictionaries
                if lang in SZ_FIX_DATA and grp in SZ_FIX_DATA[lang]:
                    data[lang][grp]["data"].update(SZ_FIX_DATA[lang][grp])

                if grp in SZ_FIX_DATA_GLOBAL:
                    data[lang][grp]["data"].update(SZ_FIX_DATA_GLOBAL[grp])

                if pattern:
                    data[lang][grp]["pattern"] = pattern(data[lang][grp]["data"])

    f = open(os.path.join(cur_dir, "data.py"), "w+")
    f.write(TEMPLATE)
    f.write(pprint.pformat(data, width=1))
    f.write(TEMPLATE_END)
    f.close()
<commit_msg>add fixme<commit_after># coding=utf-8

import re
import os
import pprint
from collections import OrderedDict

from bs4 import BeautifulSoup

TEMPLATE = """\
import re
from collections import OrderedDict
data = """

TEMPLATE_END = """\

for lang, grps in data.iteritems():
    for grp in grps.iterkeys():
        if data[lang][grp]["pattern"]:
            data[lang][grp]["pattern"] = re.compile(data[lang][grp]["pattern"])
"""

SZ_FIX_DATA = {
    "eng": {
        "PartialWordsAlways": {
            u"°x°": u"%",
            u"compiete": u"complete",
            u"Âs": u"'s",
            u"ÃÂs": u"'s",
            u"a/ion": u"ation",
            u"at/on": u"ation",
            u"l/an": u"lian",
            u"lljust": u"ll just",
            u". L ": u". I ",

            # fixme: this is PartialLines
            u"L know": u"I know",
            u"L should": u"I should",
            u"L do": u"I do",
            u"L would": u"I would",
            u"L could": u"I could",
            u"L can": u"I can",
            u"L happen": u"I happen",
            u"L might": u"I might",
            u"L have ": u"I have",
            u"L had": u"I had",
            u"L want": u"I want",
            u"L was": u"I was",
            u"L am": u"I am",
            u"L suggest": u"I suggest",
            u"L think": u"I think",
            u"L reckon": u"I reckon",
            u"L like": u"I like",
            u"L love": u"I love",
            u"L don't": u"I don't",
            u"L won't": u"I won't",
        },
        "WholeWords": {
            u"I'11": u"I'll",
            u"Tun": u"Run",
            u"pan'": u"part",
            u"al'": u"at",
            u"a re": u"are",
            u"wail'": u"wait",
            u"he)'": u"hey",
            u"he)\"": u"hey",
            u"He)'": u"Hey",
            u"He)\"": u"Hey",
            u"Yea h": u"Yeah",
            u"yea h": u"yeah",
            u"h is": u"his",
            u" 're ": u"'re ",
            u"LAst": u"Last",
            u"forthis": u"for this",
        }
    },
    "nld": {
        "PartialWordsAlways": {
            u"ט": u"è",
            u"י": u"é",
            u"כ": u"ë",
            u"צ": u"ë",
            u"ן": u"ï",
            u"ף": u"ó",
            u"א": u"à",
            u"Iֻ": u"I",
            u"č": u"è",
            u"פ": u"o",
            u"ם": u"i",
        },
    },
    "swe": {
        "PartialWordsAlways": {
            u"ĺ": u"å",
            u"Ĺ": u"Å",
        }
    }
}

SZ_FIX_DATA_GLOBAL = {
    "PartialWordsAlways": {
        u"¶¶": u"♫",
        u"¶": u"♪"
    }
}

if __name__ == "__main__":
    cur_dir = os.path.dirname(os.path.realpath(__file__))
    xml_dir = os.path.join(cur_dir, "xml")
    file_list = os.listdir(xml_dir)

    data = {}

    for fn in file_list:
        if fn.endswith("_OCRFixReplaceList.xml"):
            lang = fn.split("_")[0]
            soup = BeautifulSoup(open(os.path.join(xml_dir, fn)), "xml")

            fetch_data = (
                    # group, item_name, pattern
                    ("WholeLines", "Line", None),
                    ("WholeWords", "Word", lambda d: (ur"(?um)\b(?:" + u"|".join([re.escape(k) for k in d.keys()])
                                                      + ur')\b') if d else None),
                    ("PartialWordsAlways", "WordPart", None),
                    ("PartialLines", "LinePart", lambda d: (ur"(?um)(?:(?<=\s)|(?<=^)|(?<=\b))(?:" +
                                                            u"|".join([re.escape(k) for k in d.keys()]) +
                                                            ur")(?:(?=\s)|(?=$)|(?=\b))") if d else None),
                    ("BeginLines", "Beginning", lambda d: (ur"(?um)^(?:"+u"|".join([re.escape(k) for k in d.keys()])
                                                           + ur')') if d else None),
                    ("EndLines", "Ending", lambda d: (ur"(?um)(?:" + u"|".join([re.escape(k) for k in d.keys()]) +
                                                      ur")$") if d else None,),
            )

            data[lang] = dict((grp, {"data": OrderedDict(), "pattern": None}) for grp, item_name, pattern in fetch_data)

            for grp, item_name, pattern in fetch_data:
                for grp_data in soup.find_all(grp):
                    for line in grp_data.find_all(item_name):
                        data[lang][grp]["data"][line["from"]] = line["to"]

                # add our own dictionaries
                if lang in SZ_FIX_DATA and grp in SZ_FIX_DATA[lang]:
                    data[lang][grp]["data"].update(SZ_FIX_DATA[lang][grp])

                if grp in SZ_FIX_DATA_GLOBAL:
                    data[lang][grp]["data"].update(SZ_FIX_DATA_GLOBAL[grp])

                if pattern:
                    data[lang][grp]["pattern"] = pattern(data[lang][grp]["data"])

    f = open(os.path.join(cur_dir, "data.py"), "w+")
    f.write(TEMPLATE)
    f.write(pprint.pformat(data, width=1))
    f.write(TEMPLATE_END)
    f.close()
<|endoftext|>


<commit_before><commit_msg>Slight modifications to pickleft routine to avoid a rare bug<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import rawdisk.scheme
import logging
from rawdisk.filesystems.detector import FilesystemDetector
from rawdisk.plugins.manager import Manager
from rawdisk.filesystems.unknown_volume import UnknownVolume
from rawdisk.scheme.mbr import SECTOR_SIZE

class Session(object):
    """Main class used to start filesystem analysis.

    Attributes:
        partitions (list): List of detected filesystems \
        (intialized :class:`Volume <rawdisk.filesystems.volume.Volume>` \
            objects)
        scheme (enum): One of \
        :attr:`SCHEME_MBR <rawdisk.scheme.common.SCHEME_MBR>` \
        or :attr:`SCHEME_GPT <rawdisk.scheme.common.SCHEME_GPT>`.
    """
    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.partitions = []
        self.scheme = None
        self.filename = None
        self.manager = Manager()

    def load_plugins(self):
        """Load filesystem detection plugins"""
        self.manager.load_plugins()

    def list_partitions(self):
        """Print a list of detected partitions."""

        for part in self.partitions:
            print(part)

    def load(self, filename, bs=512):
        """Starts filesystem analysis. Detects supported filesystems and \
        loads :attr:`partitions` array.

        Args:
            filename - Path to file or device for reading.

        Raises:
            IOError - File/device does not exist or is not readable.
        """
        self.filename = filename

        # Detect partitioning scheme
        self.scheme = rawdisk.scheme.common.detect_scheme(filename)
        detector = FilesystemDetector()

        if self.scheme == rawdisk.scheme.common.SCHEME_MBR:
            mbr = rawdisk.scheme.mbr.Mbr(filename)

            # Go through table entries and analyse ones that are supported
            for entry in mbr.partition_table.entries:
                volume = detector.detect_mbr(
                    filename,
                    entry.part_offset,
                    entry.part_type
                )

                if volume is not None:
                    volume.load(filename, entry.part_offset)
                    self.partitions.append(volume)
                else:
                    self.logger.warning(
                        'Were not able to detect MBR volume type')
                    self.partitions.append(
                        UnknownVolume(
                            entry.part_offset, entry.part_type,
                            entry.total_sectors * SECTOR_SIZE
                        )
                    )

        elif self.scheme == rawdisk.scheme.common.SCHEME_GPT:
            gpt = rawdisk.scheme.gpt.Gpt()
            gpt.load(filename)

            for entry in gpt.partition_entries:
                volume = detector.detect_gpt(
                    filename,
                    entry.first_lba * bs,
                    entry.type_guid
                )

                if volume is not None:
                    volume.load(filename, entry.first_lba * bs)
                    self.partitions.append(volume)
                else:
                    self.logger.warning(
                        'Were not able to detect GPT volume type')

                    self.partitions.append(
                        UnknownVolume(
                            entry.first_lba * bs, entry.type_guid,
                            (entry.last_lba - entry.first_lba) * bs
                        )
                    )

        elif self.scheme == rawdisk.scheme.common.SCHEME_UNKNOWN:
            self.logger.warning('Partitioning scheme is not supported.')
        else:
            self.logger.warning('Partitioning scheme could not be determined.')
<commit_msg>reset partition list on each load<commit_after># -*- coding: utf-8 -*-

import rawdisk.scheme
import logging
from rawdisk.filesystems.detector import FilesystemDetector
from rawdisk.plugins.manager import Manager
from rawdisk.filesystems.unknown_volume import UnknownVolume
from rawdisk.scheme.mbr import SECTOR_SIZE

class Session(object):
    """Main class used to start filesystem analysis.

    Attributes:
        partitions (list): List of detected filesystems \
        (intialized :class:`Volume <rawdisk.filesystems.volume.Volume>` \
            objects)
        scheme (enum): One of \
        :attr:`SCHEME_MBR <rawdisk.scheme.common.SCHEME_MBR>` \
        or :attr:`SCHEME_GPT <rawdisk.scheme.common.SCHEME_GPT>`.
    """
    def __init__(self):
        self.logger = logging.getLogger(__name__)
        self.partitions = []
        self.scheme = None
        self.filename = None
        self.manager = Manager()

    def load_plugins(self):
        """Load filesystem detection plugins"""
        self.manager.load_plugins()

    def list_partitions(self):
        """Print a list of detected partitions."""

        for part in self.partitions:
            print(part)

    def load(self, filename, bs=512):
        """Starts filesystem analysis. Detects supported filesystems and \
        loads :attr:`partitions` array.

        Args:
            filename - Path to file or device for reading.

        Raises:
            IOError - File/device does not exist or is not readable.
        """
        self.filename = filename
        self.partitions = []

        # Detect partitioning scheme
        self.scheme = rawdisk.scheme.common.detect_scheme(filename)
        detector = FilesystemDetector()

        if self.scheme == rawdisk.scheme.common.SCHEME_MBR:
            mbr = rawdisk.scheme.mbr.Mbr(filename)

            # Go through table entries and analyse ones that are supported
            for entry in mbr.partition_table.entries:
                volume = detector.detect_mbr(
                    filename,
                    entry.part_offset,
                    entry.part_type
                )

                if volume is not None:
                    volume.load(filename, entry.part_offset)
                    self.partitions.append(volume)
                else:
                    self.logger.warning(
                        'Were not able to detect MBR volume type')
                    self.partitions.append(
                        UnknownVolume(
                            entry.part_offset, entry.part_type,
                            entry.total_sectors * SECTOR_SIZE
                        )
                    )

        elif self.scheme == rawdisk.scheme.common.SCHEME_GPT:
            gpt = rawdisk.scheme.gpt.Gpt()
            gpt.load(filename)

            for entry in gpt.partition_entries:
                volume = detector.detect_gpt(
                    filename,
                    entry.first_lba * bs,
                    entry.type_guid
                )

                if volume is not None:
                    volume.load(filename, entry.first_lba * bs)
                    self.partitions.append(volume)
                else:
                    self.logger.warning(
                        'Were not able to detect GPT volume type')

                    self.partitions.append(
                        UnknownVolume(
                            entry.first_lba * bs, entry.type_guid,
                            (entry.last_lba - entry.first_lba) * bs
                        )
                    )

        elif self.scheme == rawdisk.scheme.common.SCHEME_UNKNOWN:
            self.logger.warning('Partitioning scheme is not supported.')
        else:
            self.logger.warning('Partitioning scheme could not be determined.')
<|endoftext|>


<commit_before>from django.conf import settings
import logging
import re

logger = logging.getLogger("registry_log")

def mongo_db_name(registry):
    return settings.MONGO_DB_PREFIX + registry

def mongo_db_name_reg_id(registry_id):
    from models import Registry
    reg = Registry.objects.get(id=registry_id)
    return settings.MONGO_DB_PREFIX + reg.code

def get_code(delimited_key):
    return delimited_key.split(settings.FORM_SECTION_DELIMITER)[-1]

def get_form_section_code(delimited_key):
    return delimited_key.split(settings.FORM_SECTION_DELIMITER)

def mongo_key(form_name, section_code, cde_code):
    return settings.FORM_SECTION_DELIMITER.join([form_name, section_code, cde_code])

def mongo_key_from_models(form_model, section_model, cde_model):
    return mongo_key(form_model.name, section_model.code, cde_model.code)

def id_on_page(registry_form_model, section_model, cde_model):
    return mongo_key(registry_form_model.name, section_model.code, cde_model.code)

def de_camelcase(s):
    value = s[0].upper() + s[1:]
    chunks = re.findall('[A-Z][^A-Z]*', value)
    return " ".join(chunks)

class FormLink(object):

    def __init__(self, patient_id, registry, registry_form, selected=False):
        self.registry = registry
        self.patient_id = patient_id
        self.form = registry_form
        self.selected = selected

    @property
    def url(self):
        from django.core.urlresolvers import reverse
        return reverse(
            'registry_form',
            args=(
                self.registry.code,
                self.form.pk,
                self.patient_id))

    @property
    def text(self):
        return de_camelcase(self.form.name)

def get_user(username):
    from registry.groups.models import CustomUser
    try:
        return CustomUser.objects.get(username=username)
    except CustomUser.DoesNotExist:
        return None

def get_users(usernames):
    return filter(lambda x: x is not None, [get_user(username) for username in usernames])

def has_feature(feature_name):
    if settings.FEATURES == "*":
        return True
    return feature_name in settings.FEATURES  # e.g. [ 'email_notification', 'adjudication' ]

def requires_feature(feature_name):
    def decorator(func):
        def wrapper(*args, **kwargs):
            if has_feature(feature_name):
                return func(*args, **kwargs)
            else:
                logger.info(
                    "%s will not be run with args %s kwargs %s as the site lacks feature %s" %
                    (func.__name__, args, kwargs, feature_name))
        return wrapper
    return decorator

def get_full_link(request, partial_link, login_link=False):
    if login_link:
        # return a redirect login    https://rdrf.ccgapps.com.au/ophg/login?next=/ophg/admin/
        login_url = "/login?next=" + partial_link
        return get_site_url(request, login_url)
    else:
        return get_site_url(request, partial_link)

def get_site_url(request, path="/"):
    # https://rdrf.ccgapps.com.au/ophg/admin/patients/patient/3/
    # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    # this part !
    return request.build_absolute_uri(path)

def location_name(registry_form):
    return de_camelcase(registry_form.name)
<commit_msg>RDR-979 Added some utility functions to make reporting easier<commit_after>from django.conf import settings
import logging
import re

logger = logging.getLogger("registry_log")

class BadKeyError(Exception):
    pass

def mongo_db_name(registry):
    return settings.MONGO_DB_PREFIX + registry

def mongo_db_name_reg_id(registry_id):
    from models import Registry
    reg = Registry.objects.get(id=registry_id)
    return settings.MONGO_DB_PREFIX + reg.code

def get_code(delimited_key):
    return delimited_key.split(settings.FORM_SECTION_DELIMITER)[-1]

def get_form_section_code(delimited_key):
    return delimited_key.split(settings.FORM_SECTION_DELIMITER)

def mongo_key(form_name, section_code, cde_code):
    return settings.FORM_SECTION_DELIMITER.join([form_name, section_code, cde_code])

def mongo_key_from_models(form_model, section_model, cde_model):
    return mongo_key(form_model.name, section_model.code, cde_model.code)

def models_from_mongo_key(registry_model, delimited_key):
    from rdrf.models import RegistryForm, Section, CommonDataElement
    form_name, section_code, cde_code = get_form_section_code(delimited_key)
    try:
        form_model = RegistryForm.objects.get(name=form_name, registry=registry_model)
    except RegistryForm.DoesNotExist:
        raise BadKeyError()

    try:
        section_model = Section.objects.get(code=section_code)
    except Section.DoesNotExist:
        raise BadKeyError()

    try:
        cde_model = CommonDataElement.objects.get(code=cde_code)
    except CommonDataElement.DoesNotExist:
        raise BadKeyError()

    return form_model, section_model, cde_model

def is_delimited_key(s):
    try:
        parts = s.split(settings.FORM_SECTION_DELIMITER)
        if len(parts) == 3:
            return True
    except Exception:
        pass
    return False

def id_on_page(registry_form_model, section_model, cde_model):
    return mongo_key(registry_form_model.name, section_model.code, cde_model.code)

def de_camelcase(s):
    value = s[0].upper() + s[1:]
    chunks = re.findall('[A-Z][^A-Z]*', value)
    return " ".join(chunks)

class FormLink(object):

    def __init__(self, patient_id, registry, registry_form, selected=False):
        self.registry = registry
        self.patient_id = patient_id
        self.form = registry_form
        self.selected = selected

    @property
    def url(self):
        from django.core.urlresolvers import reverse
        return reverse(
            'registry_form',
            args=(
                self.registry.code,
                self.form.pk,
                self.patient_id))

    @property
    def text(self):
        return de_camelcase(self.form.name)

def get_user(username):
    from registry.groups.models import CustomUser
    try:
        return CustomUser.objects.get(username=username)
    except CustomUser.DoesNotExist:
        return None

def get_users(usernames):
    return filter(lambda x: x is not None, [get_user(username) for username in usernames])

def has_feature(feature_name):
    if settings.FEATURES == "*":
        return True
    return feature_name in settings.FEATURES  # e.g. [ 'email_notification', 'adjudication' ]

def requires_feature(feature_name):
    def decorator(func):
        def wrapper(*args, **kwargs):
            if has_feature(feature_name):
                return func(*args, **kwargs)
            else:
                logger.info(
                    "%s will not be run with args %s kwargs %s as the site lacks feature %s" %
                    (func.__name__, args, kwargs, feature_name))
        return wrapper
    return decorator

def get_full_link(request, partial_link, login_link=False):
    if login_link:
        # return a redirect login    https://rdrf.ccgapps.com.au/ophg/login?next=/ophg/admin/
        login_url = "/login?next=" + partial_link
        return get_site_url(request, login_url)
    else:
        return get_site_url(request, partial_link)

def get_site_url(request, path="/"):
    # https://rdrf.ccgapps.com.au/ophg/admin/patients/patient/3/
    # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    # this part !
    return request.build_absolute_uri(path)

def location_name(registry_form):
    return de_camelcase(registry_form.name)
<|endoftext|>


<commit_before>#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from oslo_log import log as logging
from tempest.common import waiters
from tempest import test

from ironic_tempest_plugin.tests.scenario import baremetal_manager

LOG = logging.getLogger(__name__)

class BaremetalBasicOps(baremetal_manager.BaremetalScenarioTest):
    """This smoke test tests the pxe_ssh Ironic driver.

    It follows this basic set of operations:
        * Creates a keypair
        * Boots an instance using the keypair
        * Monitors the associated Ironic node for power and
          expected state transitions
        * Validates Ironic node's port data has been properly updated
        * Verifies SSH connectivity using created keypair via fixed IP
        * Associates a floating ip
        * Verifies SSH connectivity using created keypair via floating IP
        * Verifies instance rebuild with ephemeral partition preservation
        * Deletes instance
        * Monitors the associated Ironic node for power and
          expected state transitions
    """

    def rebuild_instance(self, preserve_ephemeral=False):
        self.rebuild_server(server_id=self.instance['id'],
                            preserve_ephemeral=preserve_ephemeral,
                            wait=False)

        node = self.get_node(instance_id=self.instance['id'])

        # We should remain on the same node
        self.assertEqual(self.node['uuid'], node['uuid'])
        self.node = node

        waiters.wait_for_server_status(
            self.servers_client,
            server_id=self.instance['id'],
            status='REBUILD',
            ready_wait=False)
        waiters.wait_for_server_status(
            self.servers_client,
            server_id=self.instance['id'],
            status='ACTIVE')

    def verify_partition(self, client, label, mount, gib_size):
        """Verify a labeled partition's mount point and size."""
        LOG.info("Looking for partition %s mounted on %s", label, mount)

        # Validate we have a device with the given partition label
        cmd = "/sbin/blkid | grep '%s' | cut -d':' -f1" % label
        device = client.exec_command(cmd).rstrip('\n')
        LOG.debug("Partition device is %s", device)
        self.assertNotEqual('', device)

        # Validate the mount point for the device
        cmd = "mount | grep '%s' | cut -d' ' -f3" % device
        actual_mount = client.exec_command(cmd).rstrip('\n')
        LOG.debug("Partition mount point is %s", actual_mount)
        self.assertEqual(actual_mount, mount)

        # Validate the partition size matches what we expect
        numbers = '0123456789'
        devnum = device.replace('/dev/', '')
        cmd = "cat /sys/block/%s/%s/size" % (devnum.rstrip(numbers), devnum)
        num_bytes = client.exec_command(cmd).rstrip('\n')
        num_bytes = int(num_bytes) * 512
        actual_gib_size = num_bytes / (1024 * 1024 * 1024)
        LOG.debug("Partition size is %d GiB", actual_gib_size)
        self.assertEqual(actual_gib_size, gib_size)

    def get_flavor_ephemeral_size(self):
        """Returns size of the ephemeral partition in GiB."""
        f_id = self.instance['flavor']['id']
        flavor = self.flavors_client.show_flavor(f_id)['flavor']
        ephemeral = flavor.get('OS-FLV-EXT-DATA:ephemeral')
        if not ephemeral or ephemeral == 'N/A':
            return None
        return int(ephemeral)

    def validate_ports(self):
        node_uuid = self.node['uuid']
        vifs = self.get_node_vifs(node_uuid)
        ir_ports = self.get_ports(node_uuid)
        ir_ports_addresses = [x['address'] for x in ir_ports]
        for vif in vifs:
            n_port_id = vif['id']
            body = self.ports_client.show_port(n_port_id)
            n_port = body['port']
            self.assertEqual(n_port['device_id'], self.instance['id'])
            self.assertIn(n_port['mac_address'], ir_ports_addresses)

    @test.idempotent_id('549173a5-38ec-42bb-b0e2-c8b9f4a08943')
    @test.services('compute', 'image', 'network')
    def test_baremetal_server_ops(self):
        self.add_keypair()
        self.instance, self.node = self.boot_instance()
        self.validate_ports()
        ip_address = self.get_server_ip(self.instance)
        self.get_remote_client(ip_address).validate_authentication()
        vm_client = self.get_remote_client(ip_address)

        # We expect the ephemeral partition to be mounted on /mnt and to have
        # the same size as our flavor definition.
        eph_size = self.get_flavor_ephemeral_size()
        if eph_size:
            self.verify_partition(vm_client, 'ephemeral0', '/mnt', eph_size)
            # Create the test file
            self.create_timestamp(
                ip_address, private_key=self.keypair['private_key'])

        self.terminate_instance(self.instance)
<commit_msg>Update validate_ports from BaremetalBasicOps<commit_after>#
# Copyright 2014 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from oslo_log import log as logging
from tempest.common import waiters
from tempest import config
from tempest.lib.common import api_version_request
from tempest import test

from ironic_tempest_plugin.tests.scenario import baremetal_manager

LOG = logging.getLogger(__name__)
CONF = config.CONF

class BaremetalBasicOps(baremetal_manager.BaremetalScenarioTest):
    """This smoke test tests the pxe_ssh Ironic driver.

    It follows this basic set of operations:
        * Creates a keypair
        * Boots an instance using the keypair
        * Monitors the associated Ironic node for power and
          expected state transitions
        * Validates Ironic node's port data has been properly updated
        * Verifies SSH connectivity using created keypair via fixed IP
        * Associates a floating ip
        * Verifies SSH connectivity using created keypair via floating IP
        * Verifies instance rebuild with ephemeral partition preservation
        * Deletes instance
        * Monitors the associated Ironic node for power and
          expected state transitions
    """

    def rebuild_instance(self, preserve_ephemeral=False):
        self.rebuild_server(server_id=self.instance['id'],
                            preserve_ephemeral=preserve_ephemeral,
                            wait=False)

        node = self.get_node(instance_id=self.instance['id'])

        # We should remain on the same node
        self.assertEqual(self.node['uuid'], node['uuid'])
        self.node = node

        waiters.wait_for_server_status(
            self.servers_client,
            server_id=self.instance['id'],
            status='REBUILD',
            ready_wait=False)
        waiters.wait_for_server_status(
            self.servers_client,
            server_id=self.instance['id'],
            status='ACTIVE')

    def verify_partition(self, client, label, mount, gib_size):
        """Verify a labeled partition's mount point and size."""
        LOG.info("Looking for partition %s mounted on %s", label, mount)

        # Validate we have a device with the given partition label
        cmd = "/sbin/blkid | grep '%s' | cut -d':' -f1" % label
        device = client.exec_command(cmd).rstrip('\n')
        LOG.debug("Partition device is %s", device)
        self.assertNotEqual('', device)

        # Validate the mount point for the device
        cmd = "mount | grep '%s' | cut -d' ' -f3" % device
        actual_mount = client.exec_command(cmd).rstrip('\n')
        LOG.debug("Partition mount point is %s", actual_mount)
        self.assertEqual(actual_mount, mount)

        # Validate the partition size matches what we expect
        numbers = '0123456789'
        devnum = device.replace('/dev/', '')
        cmd = "cat /sys/block/%s/%s/size" % (devnum.rstrip(numbers), devnum)
        num_bytes = client.exec_command(cmd).rstrip('\n')
        num_bytes = int(num_bytes) * 512
        actual_gib_size = num_bytes / (1024 * 1024 * 1024)
        LOG.debug("Partition size is %d GiB", actual_gib_size)
        self.assertEqual(actual_gib_size, gib_size)

    def get_flavor_ephemeral_size(self):
        """Returns size of the ephemeral partition in GiB."""
        f_id = self.instance['flavor']['id']
        flavor = self.flavors_client.show_flavor(f_id)['flavor']
        ephemeral = flavor.get('OS-FLV-EXT-DATA:ephemeral')
        if not ephemeral or ephemeral == 'N/A':
            return None
        return int(ephemeral)

    def validate_ports(self):
        node_uuid = self.node['uuid']
        vifs = []
        # TODO(vsaienko) switch to get_node_vifs() when all stable releases
        # supports Ironic API 1.28
        if (api_version_request.APIVersionRequest(
            CONF.baremetal.max_microversion) >=
                api_version_request.APIVersionRequest('1.28')):
            vifs = self.get_node_vifs(node_uuid)
        else:
            for port in self.get_ports(self.node['uuid']):
                vif = port['extra'].get('vif_port_id')
                if vif:
                    vifs.append({'id': vif})

        ir_ports = self.get_ports(node_uuid)
        ir_ports_addresses = [x['address'] for x in ir_ports]
        for vif in vifs:
            n_port_id = vif['id']
            body = self.ports_client.show_port(n_port_id)
            n_port = body['port']
            self.assertEqual(n_port['device_id'], self.instance['id'])
            self.assertIn(n_port['mac_address'], ir_ports_addresses)

    @test.idempotent_id('549173a5-38ec-42bb-b0e2-c8b9f4a08943')
    @test.services('compute', 'image', 'network')
    def test_baremetal_server_ops(self):
        self.add_keypair()
        self.instance, self.node = self.boot_instance()
        self.validate_ports()
        ip_address = self.get_server_ip(self.instance)
        self.get_remote_client(ip_address).validate_authentication()
        vm_client = self.get_remote_client(ip_address)

        # We expect the ephemeral partition to be mounted on /mnt and to have
        # the same size as our flavor definition.
        eph_size = self.get_flavor_ephemeral_size()
        if eph_size:
            self.verify_partition(vm_client, 'ephemeral0', '/mnt', eph_size)
            # Create the test file
            self.create_timestamp(
                ip_address, private_key=self.keypair['private_key'])

        self.terminate_instance(self.instance)
<|endoftext|>


<commit_before>import codecs
import string
import random
from openid import oidutil

def test_base64():
    allowed_s = string.letters + string.digits + '+/='
    allowed_d = {}
    for c in allowed_s:
        allowed_d[c] = None
    isAllowed = allowed_d.has_key

    def checkEncoded(s):
        for c in s:
            assert isAllowed(c), s

    cases = [
        '',
        'x',
        '\x00',
        '\x01',
        '\x00' * 100,
        ''.join(map(chr, range(256))),
        ]

    for s in cases:
        b64 = oidutil.toBase64(s)
        checkEncoded(b64)
        s_prime = oidutil.fromBase64(b64)
        assert s_prime == s, (s, b64, s_prime)

    # Randomized test
    for _ in xrange(50):
        n = random.randrange(2048)
        s = ''.join(map(chr, map(lambda _: random.randrange(256), range(n))))
        b64 = oidutil.toBase64(s)
        checkEncoded(b64)
        s_prime = oidutil.fromBase64(b64)
        assert s_prime == s, (s, b64, s_prime)

def test_normalizeUrl():
    n = oidutil.normalizeUrl

    assert 'http://foo.com/' == n('foo.com')

    assert 'http://foo.com/' == n('http://foo.com')
    assert 'https://foo.com/' == n('https://foo.com')
    assert 'http://foo.com/bar' == n('foo.com/bar')
    assert 'http://foo.com/bar' == n('http://foo.com/bar')

    assert 'http://foo.com/' == n('http://foo.com/')
    assert 'https://foo.com/' == n('https://foo.com/')
    assert 'https://foo.com/bar'  == n('https://foo.com/bar')

    assert 'http://foo.com/%E8%8D%89' == n(u'foo.com/\u8349')
    assert 'http://foo.com/%E8%8D%89' == n(u'http://foo.com/\u8349')

    non_ascii_domain_cases = [
        ('http://xn--vl1a.com/', u'\u8349.com'),
        ('http://xn--vl1a.com/', u'http://\u8349.com'),
        ('http://xn--vl1a.com/', u'\u8349.com/'),
        ('http://xn--vl1a.com/', u'http://\u8349.com/'),
        ('http://xn--vl1a.com/%E8%8D%89', u'\u8349.com/\u8349'),
        ('http://xn--vl1a.com/%E8%8D%89', u'http://\u8349.com/\u8349'),
        ]

    try:
        codecs.getencoder('idna')
    except LookupError:
        # If there is no idna codec, these cases with
        # non-ascii-representable domain names should fail.
        should_raise = True
    else:
        should_raise = False

    for expected, case in non_ascii_domain_cases:
        try:
            actual = n(case)
        except UnicodeError:
            assert should_raise
        else:
            assert not should_raise and actual == expected, case

    assert n(None) is None
    assert n('') is None
    assert n('http://') is None

# XXX: there are more functions that could benefit from being better
# specified and tested in oidutil.py These include, but are not
# limited to appendArgs

def test():
    test_base64()
    test_normalizeUrl()

if __name__ == '__main__':
    test()
<commit_msg>[project @ Added test case for oidutil.appendArgs]<commit_after>import unittest
import codecs
import string
import random
from openid import oidutil

def test_base64():
    allowed_s = string.letters + string.digits + '+/='
    allowed_d = {}
    for c in allowed_s:
        allowed_d[c] = None
    isAllowed = allowed_d.has_key

    def checkEncoded(s):
        for c in s:
            assert isAllowed(c), s

    cases = [
        '',
        'x',
        '\x00',
        '\x01',
        '\x00' * 100,
        ''.join(map(chr, range(256))),
        ]

    for s in cases:
        b64 = oidutil.toBase64(s)
        checkEncoded(b64)
        s_prime = oidutil.fromBase64(b64)
        assert s_prime == s, (s, b64, s_prime)

    # Randomized test
    for _ in xrange(50):
        n = random.randrange(2048)
        s = ''.join(map(chr, map(lambda _: random.randrange(256), range(n))))
        b64 = oidutil.toBase64(s)
        checkEncoded(b64)
        s_prime = oidutil.fromBase64(b64)
        assert s_prime == s, (s, b64, s_prime)

def test_normalizeUrl():
    n = oidutil.normalizeUrl

    assert 'http://foo.com/' == n('foo.com')

    assert 'http://foo.com/' == n('http://foo.com')
    assert 'https://foo.com/' == n('https://foo.com')
    assert 'http://foo.com/bar' == n('foo.com/bar')
    assert 'http://foo.com/bar' == n('http://foo.com/bar')

    assert 'http://foo.com/' == n('http://foo.com/')
    assert 'https://foo.com/' == n('https://foo.com/')
    assert 'https://foo.com/bar'  == n('https://foo.com/bar')

    assert 'http://foo.com/%E8%8D%89' == n(u'foo.com/\u8349')
    assert 'http://foo.com/%E8%8D%89' == n(u'http://foo.com/\u8349')

    non_ascii_domain_cases = [
        ('http://xn--vl1a.com/', u'\u8349.com'),
        ('http://xn--vl1a.com/', u'http://\u8349.com'),
        ('http://xn--vl1a.com/', u'\u8349.com/'),
        ('http://xn--vl1a.com/', u'http://\u8349.com/'),
        ('http://xn--vl1a.com/%E8%8D%89', u'\u8349.com/\u8349'),
        ('http://xn--vl1a.com/%E8%8D%89', u'http://\u8349.com/\u8349'),
        ]

    try:
        codecs.getencoder('idna')
    except LookupError:
        # If there is no idna codec, these cases with
        # non-ascii-representable domain names should fail.
        should_raise = True
    else:
        should_raise = False

    for expected, case in non_ascii_domain_cases:
        try:
            actual = n(case)
        except UnicodeError:
            assert should_raise
        else:
            assert not should_raise and actual == expected, case

    assert n(None) is None
    assert n('') is None
    assert n('http://') is None

class AppendArgsTest(unittest.TestCase):
    def __init__(self, desc, args, expected):
        unittest.TestCase.__init__(self)
        self.desc = desc
        self.args = args
        self.expected = expected

    def runTest(self):
        result = oidutil.appendArgs(*self.args)
        self.assertEqual(self.expected, result, self.args)

    def shortDescription(self):
        return self.desc

def buildAppendTests():
    simple = 'http://www.example.com/'
    cases = [
        ('empty list', (simple, []), simple),
        ('empty dict', (simple, {}), simple),
        ('one list', (simple, [('a', 'b')]), simple + '?a=b'),
        ('one dict', (simple, {'a':'b'}), simple + '?a=b'),
        ('two list (same)',
         (simple, [('a', 'b'), ('a', 'c')]), simple + '?a=b&a=c'),
        ('two list', (simple, [('a', 'b'), ('b', 'c')]), simple + '?a=b&b=c'),
        ('two list (order)',
         (simple, [('b', 'c'), ('a', 'b')]), simple + '?b=c&a=b'),
        ('two dict (order)',
         (simple, {'b':'c', 'a':'b'}), simple + '?a=b&b=c'),
        ]

    tests = []

    for name, args, expected in cases:
        test = AppendArgsTest(name, args, expected)
        tests.append(test)

    return unittest.TestSuite(tests)

def test_appendArgs():
    suite = buildAppendTests()
    runner = unittest.TextTestRunner()
    result = runner.run(suite)
    assert result.wasSuccessful()

# XXX: there are more functions that could benefit from being better
# specified and tested in oidutil.py These include, but are not
# limited to appendArgs

def test():
    test_base64()
    test_normalizeUrl()
    test_appendArgs()

if __name__ == '__main__':
    test()
<|endoftext|>