<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
from django.http import HttpResponse
from django.core.management import call_command
from django.core.cache import cache
from django.core.servers.basehttp import FileWrapper

def get_or_create(klass, _id=None, **kwargs):
    if _id is None:
        object = klass.objects.filter(**kwargs)
    else:
        object = klass.objects.filter(**{_id : kwargs[_id]})
    if object:
        return object[0]
    else:
        print "     add new", klass.__name__, kwargs
        return klass.objects.create(**kwargs)

def update_search_index():
    call_command("update_memopol_index")

def check_dir(filename):
    dirname = os.path.dirname(filename)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def send_file(request, filename, content_type='text/plain'):
    """
    Send a file through Django.
    """
    buffer = FileWrapper(file(filename))
    response = HttpResponse(buffer, content_type=content_type)
    response['Content-Length'] = os.path.getsize(filename)
    return response

def get_content_cache(request, filename, content_type='image/png'):
    """
    Return the cached image if exists and refresh is not forced
    Otherwise return False
    """
    if request.GET.get(u'force', u'0') != u'0':
        return False
    if not os.path.exists(filename):
        return False
    return send_file(request, filename, content_type=content_type)

COLORS =\
((208, 0, 0),
(222, 79, 17),
(246, 183, 36),
(253, 220, 11),
(253, 247, 11),
(232, 253, 11),
(162, 253, 11),
(114, 220, 44),
(58, 198, 9),
(55, 157, 0))

def color(score):
    return COLORS[int(score/10)]

def cached(expire):
    """cache the whole response for ``expire`` delay"""
    def wrapper(func):
        def wrapped(request, **kwargs):
            user = 'anon' if request.user.is_anonymous() else 'auth'
            path = '%s:%s' % (user, request.path)
            content = cache.get(path)
            if content is None:
                resp = func(request, **kwargs)
                if not resp.is_rendered:
                    resp.render()
                content = resp.content.replace('   ', '')
                cache.set(path, content, timeout=expire)
                resp = HttpResponse(content)
                resp['X-Cached'] = '0'
            else:
                resp = HttpResponse(content)
                resp['X-Cached'] = '1'
            return resp
        return wrapped
    return wrapper

# This one come from pyramid
# https://github.com/Pylons/pyramid/blob/master/pyramid/decorator.py
class reify(object):

    """ Put the result of a method which uses this (non-data)
    descriptor decorator in the instance dict after the first call,
    effectively replacing the decorator with an instance variable."""

    def __init__(self, wrapped):
        self.wrapped = wrapped
        try:
            self.__doc__ = wrapped.__doc__
        except: # pragma: no cover
            pass

    def __get__(self, inst, objtype=None):
        if inst is None:
            return self
        val = self.wrapped(inst)
        setattr(inst, self.wrapped.__name__, val)
        return val

# Code taken in dingus to fix south migration loading fixtures
# https://github.com/garybernhardt/dingus/blob/master/dingus.py

from functools import wraps

def patch(object_path, new_object):
    module_name, attribute_name = object_path.rsplit('.', 1)
    return _Patcher(module_name, attribute_name, new_object)

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _importer(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

class _Patcher:
    def __init__(self, module_name, attribute_name, new_object):
        self.module_name = module_name
        self.attribute_name = attribute_name
        self.module = _importer(self.module_name)
        self.new_object = new_object

    def __call__(self, fn):
        @wraps(fn)
        def new_fn(*args, **kwargs):
            self.patch_object()
            try:
                return fn(*args, **kwargs)
            finally:
                self.restore_object()
        return new_fn

    def __enter__(self):
        self.patch_object()

    def __exit__(self, exc_type, exc_value, traceback):
        self.restore_object()

    def patch_object(self):
        self.original_object = getattr(self.module, self.attribute_name)
        setattr(self.module, self.attribute_name, self.new_object)

    def restore_object(self):
        setattr(self.module, self.attribute_name, self.original_object)

def loaddata(orm, fixture_name):
        _get_model = lambda model_identifier: orm[model_identifier]

        with patch('django.core.serializers.python._get_model', _get_model):
            from django.core.management import call_command
            call_command("loaddata", fixture_name)

# end of code from dingus and more http://stackoverflow.com/questions/5472925/django-loading-data-from-fixture-after-backward-migration-loaddata-is-using-mod/5906258#5906258
<commit_msg>[fix] forgot to add an aditionnal value to COLORS for score == 100<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
from django.http import HttpResponse
from django.core.management import call_command
from django.core.cache import cache
from django.core.servers.basehttp import FileWrapper

def get_or_create(klass, _id=None, **kwargs):
    if _id is None:
        object = klass.objects.filter(**kwargs)
    else:
        object = klass.objects.filter(**{_id : kwargs[_id]})
    if object:
        return object[0]
    else:
        print "     add new", klass.__name__, kwargs
        return klass.objects.create(**kwargs)

def update_search_index():
    call_command("update_memopol_index")

def check_dir(filename):
    dirname = os.path.dirname(filename)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def send_file(request, filename, content_type='text/plain'):
    """
    Send a file through Django.
    """
    buffer = FileWrapper(file(filename))
    response = HttpResponse(buffer, content_type=content_type)
    response['Content-Length'] = os.path.getsize(filename)
    return response

def get_content_cache(request, filename, content_type='image/png'):
    """
    Return the cached image if exists and refresh is not forced
    Otherwise return False
    """
    if request.GET.get(u'force', u'0') != u'0':
        return False
    if not os.path.exists(filename):
        return False
    return send_file(request, filename, content_type=content_type)

COLORS =\
((208, 0, 0),
(222, 79, 17),
(246, 183, 36),
(253, 220, 11),
(253, 247, 11),
(232, 253, 11),
(162, 253, 11),
(114, 220, 44),
(58, 198, 9),
(55, 157, 0),
(55, 157, 0))

def color(score):
    return COLORS[int(score/10)]

def cached(expire):
    """cache the whole response for ``expire`` delay"""
    def wrapper(func):
        def wrapped(request, **kwargs):
            user = 'anon' if request.user.is_anonymous() else 'auth'
            path = '%s:%s' % (user, request.path)
            content = cache.get(path)
            if content is None:
                resp = func(request, **kwargs)
                if not resp.is_rendered:
                    resp.render()
                content = resp.content.replace('   ', '')
                cache.set(path, content, timeout=expire)
                resp = HttpResponse(content)
                resp['X-Cached'] = '0'
            else:
                resp = HttpResponse(content)
                resp['X-Cached'] = '1'
            return resp
        return wrapped
    return wrapper

# This one come from pyramid
# https://github.com/Pylons/pyramid/blob/master/pyramid/decorator.py
class reify(object):

    """ Put the result of a method which uses this (non-data)
    descriptor decorator in the instance dict after the first call,
    effectively replacing the decorator with an instance variable."""

    def __init__(self, wrapped):
        self.wrapped = wrapped
        try:
            self.__doc__ = wrapped.__doc__
        except: # pragma: no cover
            pass

    def __get__(self, inst, objtype=None):
        if inst is None:
            return self
        val = self.wrapped(inst)
        setattr(inst, self.wrapped.__name__, val)
        return val

# Code taken in dingus to fix south migration loading fixtures
# https://github.com/garybernhardt/dingus/blob/master/dingus.py

from functools import wraps

def patch(object_path, new_object):
    module_name, attribute_name = object_path.rsplit('.', 1)
    return _Patcher(module_name, attribute_name, new_object)

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _importer(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

class _Patcher:
    def __init__(self, module_name, attribute_name, new_object):
        self.module_name = module_name
        self.attribute_name = attribute_name
        self.module = _importer(self.module_name)
        self.new_object = new_object

    def __call__(self, fn):
        @wraps(fn)
        def new_fn(*args, **kwargs):
            self.patch_object()
            try:
                return fn(*args, **kwargs)
            finally:
                self.restore_object()
        return new_fn

    def __enter__(self):
        self.patch_object()

    def __exit__(self, exc_type, exc_value, traceback):
        self.restore_object()

    def patch_object(self):
        self.original_object = getattr(self.module, self.attribute_name)
        setattr(self.module, self.attribute_name, self.new_object)

    def restore_object(self):
        setattr(self.module, self.attribute_name, self.original_object)

def loaddata(orm, fixture_name):
        _get_model = lambda model_identifier: orm[model_identifier]

        with patch('django.core.serializers.python._get_model', _get_model):
            from django.core.management import call_command
            call_command("loaddata", fixture_name)

# end of code from dingus and more http://stackoverflow.com/questions/5472925/django-loading-data-from-fixture-after-backward-migration-loaddata-is-using-mod/5906258#5906258
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import time
from django.http import HttpResponse
from django.template.loader import render_to_string
from django.template import Context, Template
from django.template.base import TemplateSyntaxError
from django.conf import settings
from django.core.cache import cache

def check_dir(filename):
    dirname = os.path.dirname(filename)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def send_file(request, filename, content_type='text/plain'):
    """
    Send a file through Django.
    """
    buffer = open(filename, 'rb').read()
    response = HttpResponse(buffer, content_type=content_type)
    response['Content-Length'] = os.path.getsize(filename)
    return response

def get_content_cache(request, filename, content_type='image/png'):
    """
    Return the cached image if exists and refresh is not forced
    Otherwise return False
    """
    if request.GET.get(u'force', u'0') != u'0':
        return False
    if not os.path.exists(filename):
        return False
    return send_file(request, filename, content_type=content_type)

def color(score):
    colors = 255
    val = int(3 * colors * (score / 100.))
    red = green = colors
    if val < colors:
        green = int(2. / 3. * val)
    elif val < 2 * colors:
        green = int((2. / 3.) * colors + (1. / 3.) * (val / 2. - colors))
    else:
        red = 3 * colors - val
    return (red, green, 0)

global _cache
_cache = {}

def cached(expire):
    """cache the whole response for ``expire`` delay"""
    def wrapper(func):
        def wrapped(request, **kwargs):
            global _cache
            path = '%s:%s' % (request.user.is_anonymous() and 'anon' or 'auth', request.path)
            if path in _cache:
                ctime, resp = _cache[path]
                if ctime > int(time.time()):
                    return resp
            resp = func(request, **kwargs)
            _cache[path] = (int(time.time()) + expire, resp)
            return resp
        return wrapped
    return wrapper

# This one come from pyramid
# https://github.com/Pylons/pyramid/blob/master/pyramid/decorator.py
class reify(object):

    """ Put the result of a method which uses this (non-data)
    descriptor decorator in the instance dict after the first call,
    effectively replacing the decorator with an instance variable."""

    def __init__(self, wrapped):
        self.wrapped = wrapped
        try:
            self.__doc__ = wrapped.__doc__
        except: # pragma: no cover
            pass

    def __get__(self, inst, objtype=None):
        if inst is None:
            return self
        val = self.wrapped(inst)
        setattr(inst, self.wrapped.__name__, val)
        return val

# Code taken in dingus to fix south migration loading fixtures
# https://github.com/garybernhardt/dingus/blob/master/dingus.py

from functools import wraps

def patch(object_path, new_object):
    module_name, attribute_name = object_path.rsplit('.', 1)
    return _Patcher(module_name, attribute_name, new_object)

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _importer(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

class _Patcher:
    def __init__(self, module_name, attribute_name, new_object):
        self.module_name = module_name
        self.attribute_name = attribute_name
        self.module = _importer(self.module_name)
        self.new_object = new_object

    def __call__(self, fn):
        @wraps(fn)
        def new_fn(*args, **kwargs):
            self.patch_object()
            try:
                return fn(*args, **kwargs)
            finally:
                self.restore_object()
        return new_fn

    def __enter__(self):
        self.patch_object()

    def __exit__(self, exc_type, exc_value, traceback):
        self.restore_object()

    def patch_object(self):
        self.original_object = getattr(self.module, self.attribute_name)
        setattr(self.module, self.attribute_name, self.new_object)

    def restore_object(self):
        setattr(self.module, self.attribute_name, self.original_object)

def loaddata(orm, fixture_name):
        _get_model = lambda model_identifier: orm[model_identifier]

        with patch('django.core.serializers.python._get_model', _get_model):
            from django.core.management import call_command
            call_command("loaddata", fixture_name)

# end of code from dingus and more http://stackoverflow.com/questions/5472925/django-loading-data-from-fixture-after-backward-migration-loaddata-is-using-mod/5906258#5906258
<commit_msg>[mod] remove useless imports<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import time
from django.http import HttpResponse

def check_dir(filename):
    dirname = os.path.dirname(filename)
    if not os.path.isdir(dirname):
        os.makedirs(dirname)

def send_file(request, filename, content_type='text/plain'):
    """
    Send a file through Django.
    """
    buffer = open(filename, 'rb').read()
    response = HttpResponse(buffer, content_type=content_type)
    response['Content-Length'] = os.path.getsize(filename)
    return response

def get_content_cache(request, filename, content_type='image/png'):
    """
    Return the cached image if exists and refresh is not forced
    Otherwise return False
    """
    if request.GET.get(u'force', u'0') != u'0':
        return False
    if not os.path.exists(filename):
        return False
    return send_file(request, filename, content_type=content_type)

def color(score):
    colors = 255
    val = int(3 * colors * (score / 100.))
    red = green = colors
    if val < colors:
        green = int(2. / 3. * val)
    elif val < 2 * colors:
        green = int((2. / 3.) * colors + (1. / 3.) * (val / 2. - colors))
    else:
        red = 3 * colors - val
    return (red, green, 0)

global _cache
_cache = {}

def cached(expire):
    """cache the whole response for ``expire`` delay"""
    def wrapper(func):
        def wrapped(request, **kwargs):
            global _cache
            path = '%s:%s' % (request.user.is_anonymous() and 'anon' or 'auth', request.path)
            if path in _cache:
                ctime, resp = _cache[path]
                if ctime > int(time.time()):
                    return resp
            resp = func(request, **kwargs)
            _cache[path] = (int(time.time()) + expire, resp)
            return resp
        return wrapped
    return wrapper

# This one come from pyramid
# https://github.com/Pylons/pyramid/blob/master/pyramid/decorator.py
class reify(object):

    """ Put the result of a method which uses this (non-data)
    descriptor decorator in the instance dict after the first call,
    effectively replacing the decorator with an instance variable."""

    def __init__(self, wrapped):
        self.wrapped = wrapped
        try:
            self.__doc__ = wrapped.__doc__
        except: # pragma: no cover
            pass

    def __get__(self, inst, objtype=None):
        if inst is None:
            return self
        val = self.wrapped(inst)
        setattr(inst, self.wrapped.__name__, val)
        return val

# Code taken in dingus to fix south migration loading fixtures
# https://github.com/garybernhardt/dingus/blob/master/dingus.py

from functools import wraps

def patch(object_path, new_object):
    module_name, attribute_name = object_path.rsplit('.', 1)
    return _Patcher(module_name, attribute_name, new_object)

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _importer(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

class _Patcher:
    def __init__(self, module_name, attribute_name, new_object):
        self.module_name = module_name
        self.attribute_name = attribute_name
        self.module = _importer(self.module_name)
        self.new_object = new_object

    def __call__(self, fn):
        @wraps(fn)
        def new_fn(*args, **kwargs):
            self.patch_object()
            try:
                return fn(*args, **kwargs)
            finally:
                self.restore_object()
        return new_fn

    def __enter__(self):
        self.patch_object()

    def __exit__(self, exc_type, exc_value, traceback):
        self.restore_object()

    def patch_object(self):
        self.original_object = getattr(self.module, self.attribute_name)
        setattr(self.module, self.attribute_name, self.new_object)

    def restore_object(self):
        setattr(self.module, self.attribute_name, self.original_object)

def loaddata(orm, fixture_name):
        _get_model = lambda model_identifier: orm[model_identifier]

        with patch('django.core.serializers.python._get_model', _get_model):
            from django.core.management import call_command
            call_command("loaddata", fixture_name)

# end of code from dingus and more http://stackoverflow.com/questions/5472925/django-loading-data-from-fixture-after-backward-migration-loaddata-is-using-mod/5906258#5906258
<|endoftext|>


<commit_before># MIT License
#
# Copyright (c) 2015-2020 Iakiv Kramarenko
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import pytest
from selenium.common.exceptions import TimeoutException

from selene.support.conditions import have
from selene.support.jquery_style_selectors import ss
from tests_from_past.past.acceptance import get_test_driver
from tests_from_past.past.acceptance import given_active

def setup_module(m):
    set_driver(get_test_driver())

def teardown_module(m):
    driver().quit()

def test_should_passes():
    given_active("a", "b")
    ss("#todo-list>li").should(have.exact_texts("a", "b"))

def test_should_fails():
    given_active("a", "b")
    with pytest.raises(TimeoutException):
        ss("#todo-list>li").should(have.exact_texts("a.", "b."), timeout=0.1)
<commit_msg>remove tests: selene_collection_should_test.py<commit_after><|endoftext|>


<commit_before>"""
WSGI config for {{ project_name }} project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/
"""

from __future__ import absolute_import, unicode_literals

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings.dev")

application = get_wsgi_application()
<commit_msg>Adjust DJANGO_SETTINGS_MODULE to point to settings module<commit_after>"""
WSGI config for {{ project_name }} project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/{{ docs_version }}/howto/deployment/wsgi/
"""

from __future__ import absolute_import, unicode_literals

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "{{ project_name }}.settings")

application = get_wsgi_application()
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2017 Foursquare Labs Inc. All Rights Reserved.

from __future__ import (
  absolute_import,
  division,
  generators,
  nested_scopes,
  print_function,
  unicode_literals,
  with_statement,
)

import os

from pants.task.task import Task
from pants.util.dirutil import relative_symlink

from fsqio.pants.rpmbuild.subsystems.remote_source_fetcher import RemoteSourceFetcher
from fsqio.pants.rpmbuild.targets.remote_source import RemoteSource

class RemoteSourceTask(Task):
  """Fetch versioned files from a URL."""

  @classmethod
  def subsystem_dependencies(cls):
    return super(RemoteSourceTask, cls).subsystem_dependencies() + (RemoteSourceFetcher.Factory.scoped(cls),)

  @classmethod
  def product_types(cls):
    return ['remote_files']

  # Create results_dirs but do not cache them. This allows a stable file path for adding to classpath but each user
  # must bootstrap individually.
  @property
  def create_target_dirs(self):
    return True

  @property
  def cache_target_dirs(self):
    return False

  @staticmethod
  def _is_remote(target):
    return isinstance(target, RemoteSource)

  def execute(self):
    targets = self.context.targets(self._is_remote)
    with self.invalidated(targets, invalidate_dependents=True, topological_order=True) as invalidation_check:
      # The fetches are idempotent operations from the subsystem, invalidation only controls recreating the symlinks.
      for vt in invalidation_check.all_vts:
        remote_source = RemoteSourceFetcher.Factory.scoped_instance(self).create(vt.target)
        source_path = remote_source.path
        file_dir = source_path if remote_source.extracted else os.path.dirname(source_path)
        file_namez = os.listdir(source_path) if remote_source.extracted else [os.path.basename(source_path)]
        for filename in file_namez:
          symlink_file = os.path.join(vt.results_dir, filename)
          if not vt.valid or not os.path.isfile(symlink_file):
            relative_symlink(os.path.join(file_dir, filename), symlink_file)
        self.context.products.get('remote_files').add(vt.target, vt.results_dir).extend(file_namez)
<commit_msg>Fix crash that could happen if first bootstrap blob is extract=True (#1537)<commit_after># coding=utf-8
# Copyright 2017 Foursquare Labs Inc. All Rights Reserved.

from __future__ import (
  absolute_import,
  division,
  generators,
  nested_scopes,
  print_function,
  unicode_literals,
  with_statement,
)

import os

from pants.task.task import Task
from pants.util.dirutil import relative_symlink, safe_mkdir

from fsqio.pants.rpmbuild.subsystems.remote_source_fetcher import RemoteSourceFetcher
from fsqio.pants.rpmbuild.targets.remote_source import RemoteSource

class RemoteSourceTask(Task):
  """Fetch versioned files from a URL."""

  @classmethod
  def subsystem_dependencies(cls):
    return super(RemoteSourceTask, cls).subsystem_dependencies() + (RemoteSourceFetcher.Factory.scoped(cls),)

  @classmethod
  def product_types(cls):
    return ['remote_files']

  # Create results_dirs but do not cache them. This allows a stable file path for adding to classpath but each user
  # must bootstrap individually.
  @property
  def create_target_dirs(self):
    return True

  @property
  def cache_target_dirs(self):
    return False

  @staticmethod
  def _is_remote(target):
    return isinstance(target, RemoteSource)

  def execute(self):
    targets = self.context.targets(self._is_remote)
    with self.invalidated(targets, invalidate_dependents=True, topological_order=True) as invalidation_check:
      # The fetches are idempotent operations from the subsystem, invalidation only controls recreating the symlinks.
      for vt in invalidation_check.all_vts:
        remote_source = RemoteSourceFetcher.Factory.scoped_instance(self).create(vt.target)
        safe_mkdir(remote_source.path)
        file_dir = remote_source.path if remote_source.extracted else os.path.dirname(remote_source.path)
        file_namez = os.listdir(remote_source.path) if remote_source.extracted else [os.path.basename(remote_source.path)]
        for filename in file_namez:
          symlink_file = os.path.join(vt.results_dir, filename)
          if not vt.valid or not os.path.isfile(symlink_file):
            relative_symlink(os.path.join(file_dir, filename), symlink_file)
        self.context.products.get('remote_files').add(vt.target, vt.results_dir).extend(file_namez)
<|endoftext|>


<commit_before>import time
import threading

from dtest import Tester

class TestMetadata(Tester):

    def __init__(self, *args, **kwargs):
        kwargs['cluster_options'] = {'start_rpc': 'true'}
        Tester.__init__(self, *args, **kwargs)

    def force_compact(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()
        if cluster.version() < "2.1":
            node1.nodetool("compact Keyspace1 Standard1")
        else:
            node1.nodetool("compact keyspace1 standard1")

    def force_repair(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()
        if cluster.version() < "2.1":
            node1.nodetool('repair Keyspace1 Standard1')
        else:
            node1.nodetool('repair keyspace1 standard1')

    def do_read(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()

        if cluster.version() < "2.1":
            node1.stress(['-o', 'read', '-n', '30000', '-l', '2', '-t', '1', '-I', 'LZ4Compressor'])
        else:
            node1.stress(['read', 'no-warmup', 'n=30000', '-schema', 'replication(factor=2)', 'compression=LZ4Compressor',
                              '-rate', 'threads=1'])

    def metadata_reset_while_compact_test(self):
        """
        Resets the schema while a compact, read and repair happens.
        All kinds of glorious things can fail.
        """

        # while the schema is being reset, there will inevitably be some
        # queries that will error with this message
        self.ignore_log_patterns = '.*Unknown keyspace/cf pair.*'

        cluster = self.cluster
        cluster.populate(2).start(wait_other_notice=True)
        (node1, node2) = cluster.nodelist()

        node1.nodetool("disableautocompaction")
        node1.nodetool("setcompactionthroughput 1")

        for i in range(3):
            if cluster.version() < "2.1":
                node1.stress(['-o', 'insert', '-n', '30000', '-l', '2', '-t', '5', '-I', 'LZ4Compressor'])
            else:
                node1.stress(['write', 'no-warmup', 'n=30000', '-schema', 'replication(factor=2)', 'compression=LZ4Compressor',
                              '-rate', 'threads=5', '-pop', 'seq=1..30000'])
            node1.flush()

        thread = threading.Thread(target=self.force_compact)
        thread.start()
        time.sleep(1)

        thread2 = threading.Thread(target=self.force_repair)
        thread2.start()
        time.sleep(5)

        thread3 = threading.Thread(target=self.do_read)
        thread3.start()
        time.sleep(5)

        node1.nodetool("resetlocalschema")

        thread.join()
        thread2.join()
        thread3.join()
<commit_msg>Require 9831 in >=2.0 - hanging test<commit_after>import time
import threading

from dtest import Tester

class TestMetadata(Tester):

    def __init__(self, *args, **kwargs):
        kwargs['cluster_options'] = {'start_rpc': 'true'}
        Tester.__init__(self, *args, **kwargs)

    def force_compact(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()
        if cluster.version() < "2.1":
            node1.nodetool("compact Keyspace1 Standard1")
        else:
            node1.nodetool("compact keyspace1 standard1")

    def force_repair(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()
        if cluster.version() < "2.1":
            node1.nodetool('repair Keyspace1 Standard1')
        else:
            node1.nodetool('repair keyspace1 standard1')

    def do_read(self):
        cluster = self.cluster
        (node1, node2) = cluster.nodelist()

        if cluster.version() < "2.1":
            node1.stress(['-o', 'read', '-n', '30000', '-l', '2', '-t', '1', '-I', 'LZ4Compressor'])
        else:
            node1.stress(['read', 'no-warmup', 'n=30000', '-schema', 'replication(factor=2)', 'compression=LZ4Compressor',
                              '-rate', 'threads=1'])

    @require(9831, broken_in='2.0')
    def metadata_reset_while_compact_test(self):
        """
        Resets the schema while a compact, read and repair happens.
        All kinds of glorious things can fail.
        """

        # while the schema is being reset, there will inevitably be some
        # queries that will error with this message
        self.ignore_log_patterns = '.*Unknown keyspace/cf pair.*'

        cluster = self.cluster
        cluster.populate(2).start(wait_other_notice=True)
        (node1, node2) = cluster.nodelist()

        node1.nodetool("disableautocompaction")
        node1.nodetool("setcompactionthroughput 1")

        for i in range(3):
            if cluster.version() < "2.1":
                node1.stress(['-o', 'insert', '-n', '30000', '-l', '2', '-t', '5', '-I', 'LZ4Compressor'])
            else:
                node1.stress(['write', 'no-warmup', 'n=30000', '-schema', 'replication(factor=2)', 'compression=LZ4Compressor',
                              '-rate', 'threads=5', '-pop', 'seq=1..30000'])
            node1.flush()

        thread = threading.Thread(target=self.force_compact)
        thread.start()
        time.sleep(1)

        thread2 = threading.Thread(target=self.force_repair)
        thread2.start()
        time.sleep(5)

        thread3 = threading.Thread(target=self.do_read)
        thread3.start()
        time.sleep(5)

        node1.nodetool("resetlocalschema")

        thread.join()
        thread2.join()
        thread3.join()
<|endoftext|>


<commit_before>import reversion
import logging
import json

from django.contrib.gis.db import models as gis_models
from django.contrib.gis.db.models import Union
from rest_framework.exceptions import ValidationError
from common.models import AbstractBase, County, Constituency, Ward
from facilities.models import Facility

LOGGER = logging.getLogger(__name__)

class GISAbstractBase(AbstractBase, gis_models.Model):
    """
    We've intentionally duplicated the `AbstractBase` in the `common` app
    because we wanted to confine the impact of GIS ( Geographic ) stuff
    to this app.

    The GIS stuff should have only one touch-point with the rest of the
    models: the link to the Facility model.

    We've kept the fields that are in the `common` `AbstractBase` because
    we want to have the same kind of base behavior.
    """
    objects = gis_models.GeoManager()
    everything = gis_models.GeoManager()

    class Meta(AbstractBase.Meta):
        abstract = True

@reversion.register
class GeoCodeSource(GISAbstractBase):
    """
    Where the geo-code came from.

    This is the organization collecting the code.
    For example, DHMT, the Service Availability Mapping survey (SAM),
    Kenya Medical Research Institute (KEMRI), the Regional Center for
    Mapping of Resources for Development (RCMRD), the AIDS, Population
    and Health Integrated Assistance (APHIA) II, or another source.
    It is not the individual who collected the code
    """
    name = gis_models.CharField(
        max_length=100, unique=True,
        help_text="The name of the collecting organization")
    description = gis_models.TextField(
        help_text="A short summary of the collecting organization",
        null=True, blank=True)
    abbreviation = gis_models.CharField(
        max_length=10, null=True, blank=True,
        help_text="An acronym of the collecting or e.g SAM")

    def __unicode__(self):
        return self.name

@reversion.register
class GeoCodeMethod(GISAbstractBase):
    """
    Method used to capture the geo-code.

    Examples:
        1= Taken with GPS device,
        2= Calculated from proximity to school, village, markets
        3= Calculated from 1:50,000 scale topographic maps,
        4= Scanned from hand-drawn maps,
        5= Centroid calculation from sub-location
        8= No geo-code
        9= Other
    """
    name = gis_models.CharField(
        max_length=100, unique=True, help_text="The name of the method.")
    description = gis_models.TextField(
        help_text="A short description of the method",
        null=True, blank=True)

    def __unicode__(self):
        return self.name

@reversion.register
class FacilityCoordinates(GISAbstractBase):
    """
    Location derived by the use of GPS satellites and GPS device or receivers.

    It it three dimensional.
    The three-dimensional readings from a GPS device are latitude, longitude,
    and attitude. The date/time the reading is done is also important, as
    is the source and method of the reading.
    """
    facility = gis_models.OneToOneField(Facility)
    coordinates = gis_models.PointField()
    source = gis_models.ForeignKey(
        GeoCodeSource,
        help_text="where the geo code came from", on_delete=gis_models.PROTECT)
    method = gis_models.ForeignKey(
        GeoCodeMethod,
        help_text="Method used to obtain the geo codes. e.g"
        " taken with GPS device")
    collection_date = gis_models.DateTimeField(auto_now_add=True)

    def validate_longitude_and_latitude_within_kenya(self):
        try:
            boundary = WorldBorder.objects.get(code='KEN')
            if not boundary.mpoly.contains(self.coordinates):
                # This validation was relaxed ( temporarily? )
                # The Kenyan boundaries that we have loaded have low fidelity
                # at the edges, so that facilities that are, say, 100 meters
                # from the border are reported as not in Kenya
                # If higher fidelity map data is obtained, this validation
                # can be brought back
                LOGGER.error(
                    '{} is not within the Kenyan boundaries that we have'
                    .format(self.coordinates)
                )
        except WorldBorder.DoesNotExist:
            raise ValidationError('Setup error: Kenyan boundaries not loaded')

    def validate_longitude_and_latitude_within_constituency(self):
        try:
            boundary = ConstituencyBoundary.objects.get(
                area=self.facility.ward.constituency)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates,
                        self.facility.ward.constituency
                    )
                )
        except ConstituencyBoundary.DoesNotExist:
            raise ValidationError(
                'No boundary for {}'.format(
                    self.facility.ward.constituency
                )
            )

    def validate_longitude_and_latitude_within_county(self):
        try:
            boundary = CountyBoundary.objects.get(
                area=self.facility.ward.constituency.county)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates,
                        self.facility.ward.constituency.county
                    )
                )
        except CountyBoundary.DoesNotExist:
            raise ValidationError(
                'No boundary for {}'.format(
                    self.facility.ward.constituency.county
                )
            )

    def validate_longitude_and_latitude_within_ward(self):
        try:
            boundary = WardBoundary.objects.get(area=self.facility.ward)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates, self.facility.ward
                    )
                )
        except WardBoundary.DoesNotExist:
            LOGGER.error(
                'Ward {} does not have boundary info'.format(
                    self.facility.ward)
            )

    def clean(self):
        self.validate_longitude_and_latitude_within_kenya()
        self.validate_longitude_and_latitude_within_county()
        self.validate_longitude_and_latitude_within_constituency()
        self.validate_longitude_and_latitude_within_ward()
        super(FacilityCoordinates, self).clean()

    def __unicode__(self):
        return self.facility.name

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'facility coordinates'
        verbose_name = 'facility coordinates'

class AdministrativeUnitBoundary(GISAbstractBase):
    """Base class for the models that implement administrative boundaries

    All common operations and fields are here.
    We retain the default SRID ( 4326 - WGS84 ).
    """
    # These two fields should mirror the contents of the relevant admin
    # area model
    name = gis_models.CharField(max_length=100)
    code = gis_models.CharField(max_length=10, unique=True)

    # Making this field nullable is a temporary band-aid for a deficiency
    # in model_mommy ( a testing tool )
    # The impact of this is minimal; these models hold setup data that is
    # loaded and tested during each build
    mpoly = gis_models.MultiPolygonField(null=True, blank=True)

    @property
    def bound(self):
        return json.loads(self.mpoly.envelope.geojson) if self.mpoly else None

    @property
    def center(self):
        return json.loads(self.mpoly.centroid.geojson) if self.mpoly else None

    @property
    def surface_area(self):
        return self.mpoly.area if self.mpoly else 0

    @property
    def facility_count(self):
        return FacilityCoordinates.objects.filter(
            coordinates__contained=self.mpoly
        ).count() if self and self.mpoly else 0

    @property
    def density(self):
        """This is a synthetic value

        The units matter less than the relative density compared to other
        administrative units
        """
        return self.facility_count / (self.surface_area * 10000) \
            if self.surface_area else 0

    @property
    def facility_coordinates(self):
        from common.models.model_declarations import \
            _lookup_facility_coordinates
        return _lookup_facility_coordinates(self)

    def __unicode__(self):
        return self.name

    class Meta(GISAbstractBase.Meta):
        abstract = True

@reversion.register
class WorldBorder(AdministrativeUnitBoundary):
    """World boundaries

    Source: http://thematicmapping.org/downloads/TM_WORLD_BORDERS-0.3.zip
    """
    longitude = gis_models.FloatField()
    latitude = gis_models.FloatField()

    @property
    def geometry(self):
        """The world border data is unreliable, hence this; works for Kenya only"""
        return json.loads(
            CountyBoundary.objects.aggregate(
                Union('mpoly')
            )['mpoly__union'].geojson
        )

        # Fallback
        return json.loads(self.mpoly.geojson)

@reversion.register
class CountyBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(County)

    @property
    def constituency_ids(self):
        return Constituency.objects.filter(
            county=self.area).values_list('id', flat=True)

    @property
    def constituency_boundary_ids(self):
        constituency_boundary_ids = ConstituencyBoundary.objects.filter(
            area__id__in=self.constituency_ids
        ).values_list('id', flat=True)
        return constituency_boundary_ids

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'county boundaries'

@reversion.register
class ConstituencyBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(Constituency)

    @property
    def ward_ids(self):
        return Ward.objects.filter(
            constituency=self.area).values_list('id', flat=True)

    @property
    def ward_boundary_ids(self):
        ward_boundary_ids = WardBoundary.objects.filter(
            area__id__in=self.ward_ids
        ).values_list('id', flat=True)
        return ward_boundary_ids

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'constituency boundaries'

@reversion.register
class WardBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(Ward)

    @property
    def facility_ids(self):
        return FacilityCoordinates.objects.filter(
            coordinates__contained=self.mpoly
        ).values_list('id', flat=True) if self and self.mpoly else []

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'ward boundaries'
<commit_msg>Fix flake isue in docstring<commit_after>import reversion
import logging
import json

from django.contrib.gis.db import models as gis_models
from django.contrib.gis.db.models import Union
from rest_framework.exceptions import ValidationError
from common.models import AbstractBase, County, Constituency, Ward
from facilities.models import Facility

LOGGER = logging.getLogger(__name__)

class GISAbstractBase(AbstractBase, gis_models.Model):
    """
    We've intentionally duplicated the `AbstractBase` in the `common` app
    because we wanted to confine the impact of GIS ( Geographic ) stuff
    to this app.

    The GIS stuff should have only one touch-point with the rest of the
    models: the link to the Facility model.

    We've kept the fields that are in the `common` `AbstractBase` because
    we want to have the same kind of base behavior.
    """
    objects = gis_models.GeoManager()
    everything = gis_models.GeoManager()

    class Meta(AbstractBase.Meta):
        abstract = True

@reversion.register
class GeoCodeSource(GISAbstractBase):
    """
    Where the geo-code came from.

    This is the organization collecting the code.
    For example, DHMT, the Service Availability Mapping survey (SAM),
    Kenya Medical Research Institute (KEMRI), the Regional Center for
    Mapping of Resources for Development (RCMRD), the AIDS, Population
    and Health Integrated Assistance (APHIA) II, or another source.
    It is not the individual who collected the code
    """
    name = gis_models.CharField(
        max_length=100, unique=True,
        help_text="The name of the collecting organization")
    description = gis_models.TextField(
        help_text="A short summary of the collecting organization",
        null=True, blank=True)
    abbreviation = gis_models.CharField(
        max_length=10, null=True, blank=True,
        help_text="An acronym of the collecting or e.g SAM")

    def __unicode__(self):
        return self.name

@reversion.register
class GeoCodeMethod(GISAbstractBase):
    """
    Method used to capture the geo-code.

    Examples:
        1= Taken with GPS device,
        2= Calculated from proximity to school, village, markets
        3= Calculated from 1:50,000 scale topographic maps,
        4= Scanned from hand-drawn maps,
        5= Centroid calculation from sub-location
        8= No geo-code
        9= Other
    """
    name = gis_models.CharField(
        max_length=100, unique=True, help_text="The name of the method.")
    description = gis_models.TextField(
        help_text="A short description of the method",
        null=True, blank=True)

    def __unicode__(self):
        return self.name

@reversion.register
class FacilityCoordinates(GISAbstractBase):
    """
    Location derived by the use of GPS satellites and GPS device or receivers.

    It it three dimensional.
    The three-dimensional readings from a GPS device are latitude, longitude,
    and attitude. The date/time the reading is done is also important, as
    is the source and method of the reading.
    """
    facility = gis_models.OneToOneField(Facility)
    coordinates = gis_models.PointField()
    source = gis_models.ForeignKey(
        GeoCodeSource,
        help_text="where the geo code came from", on_delete=gis_models.PROTECT)
    method = gis_models.ForeignKey(
        GeoCodeMethod,
        help_text="Method used to obtain the geo codes. e.g"
        " taken with GPS device")
    collection_date = gis_models.DateTimeField(auto_now_add=True)

    def validate_longitude_and_latitude_within_kenya(self):
        try:
            boundary = WorldBorder.objects.get(code='KEN')
            if not boundary.mpoly.contains(self.coordinates):
                # This validation was relaxed ( temporarily? )
                # The Kenyan boundaries that we have loaded have low fidelity
                # at the edges, so that facilities that are, say, 100 meters
                # from the border are reported as not in Kenya
                # If higher fidelity map data is obtained, this validation
                # can be brought back
                LOGGER.error(
                    '{} is not within the Kenyan boundaries that we have'
                    .format(self.coordinates)
                )
        except WorldBorder.DoesNotExist:
            raise ValidationError('Setup error: Kenyan boundaries not loaded')

    def validate_longitude_and_latitude_within_constituency(self):
        try:
            boundary = ConstituencyBoundary.objects.get(
                area=self.facility.ward.constituency)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates,
                        self.facility.ward.constituency
                    )
                )
        except ConstituencyBoundary.DoesNotExist:
            raise ValidationError(
                'No boundary for {}'.format(
                    self.facility.ward.constituency
                )
            )

    def validate_longitude_and_latitude_within_county(self):
        try:
            boundary = CountyBoundary.objects.get(
                area=self.facility.ward.constituency.county)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates,
                        self.facility.ward.constituency.county
                    )
                )
        except CountyBoundary.DoesNotExist:
            raise ValidationError(
                'No boundary for {}'.format(
                    self.facility.ward.constituency.county
                )
            )

    def validate_longitude_and_latitude_within_ward(self):
        try:
            boundary = WardBoundary.objects.get(area=self.facility.ward)
            if not boundary.mpoly.contains(self.coordinates):
                raise ValidationError(
                    '{} not contained in boundary of {}'.format(
                        self.coordinates, self.facility.ward
                    )
                )
        except WardBoundary.DoesNotExist:
            LOGGER.error(
                'Ward {} does not have boundary info'.format(
                    self.facility.ward)
            )

    def clean(self):
        self.validate_longitude_and_latitude_within_kenya()
        self.validate_longitude_and_latitude_within_county()
        self.validate_longitude_and_latitude_within_constituency()
        self.validate_longitude_and_latitude_within_ward()
        super(FacilityCoordinates, self).clean()

    def __unicode__(self):
        return self.facility.name

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'facility coordinates'
        verbose_name = 'facility coordinates'

class AdministrativeUnitBoundary(GISAbstractBase):
    """Base class for the models that implement administrative boundaries

    All common operations and fields are here.
    We retain the default SRID ( 4326 - WGS84 ).
    """
    # These two fields should mirror the contents of the relevant admin
    # area model
    name = gis_models.CharField(max_length=100)
    code = gis_models.CharField(max_length=10, unique=True)

    # Making this field nullable is a temporary band-aid for a deficiency
    # in model_mommy ( a testing tool )
    # The impact of this is minimal; these models hold setup data that is
    # loaded and tested during each build
    mpoly = gis_models.MultiPolygonField(null=True, blank=True)

    @property
    def bound(self):
        return json.loads(self.mpoly.envelope.geojson) if self.mpoly else None

    @property
    def center(self):
        return json.loads(self.mpoly.centroid.geojson) if self.mpoly else None

    @property
    def surface_area(self):
        return self.mpoly.area if self.mpoly else 0

    @property
    def facility_count(self):
        return FacilityCoordinates.objects.filter(
            coordinates__contained=self.mpoly
        ).count() if self and self.mpoly else 0

    @property
    def density(self):
        """This is a synthetic value

        The units matter less than the relative density compared to other
        administrative units
        """
        return self.facility_count / (self.surface_area * 10000) \
            if self.surface_area else 0

    @property
    def facility_coordinates(self):
        from common.models.model_declarations import \
            _lookup_facility_coordinates
        return _lookup_facility_coordinates(self)

    def __unicode__(self):
        return self.name

    class Meta(GISAbstractBase.Meta):
        abstract = True

@reversion.register
class WorldBorder(AdministrativeUnitBoundary):
    """World boundaries

    Source: http://thematicmapping.org/downloads/TM_WORLD_BORDERS-0.3.zip
    """
    longitude = gis_models.FloatField()
    latitude = gis_models.FloatField()

    @property
    def geometry(self):
        """The world border data is unreliable, hence this; works for Kenya"""
        return json.loads(
            CountyBoundary.objects.aggregate(
                Union('mpoly')
            )['mpoly__union'].geojson
        )

        # Fallback
        return json.loads(self.mpoly.geojson)

@reversion.register
class CountyBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(County)

    @property
    def constituency_ids(self):
        return Constituency.objects.filter(
            county=self.area).values_list('id', flat=True)

    @property
    def constituency_boundary_ids(self):
        constituency_boundary_ids = ConstituencyBoundary.objects.filter(
            area__id__in=self.constituency_ids
        ).values_list('id', flat=True)
        return constituency_boundary_ids

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'county boundaries'

@reversion.register
class ConstituencyBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(Constituency)

    @property
    def ward_ids(self):
        return Ward.objects.filter(
            constituency=self.area).values_list('id', flat=True)

    @property
    def ward_boundary_ids(self):
        ward_boundary_ids = WardBoundary.objects.filter(
            area__id__in=self.ward_ids
        ).values_list('id', flat=True)
        return ward_boundary_ids

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'constituency boundaries'

@reversion.register
class WardBoundary(AdministrativeUnitBoundary):
    area = gis_models.OneToOneField(Ward)

    @property
    def facility_ids(self):
        return FacilityCoordinates.objects.filter(
            coordinates__contained=self.mpoly
        ).values_list('id', flat=True) if self and self.mpoly else []

    class Meta(GISAbstractBase.Meta):
        verbose_name_plural = 'ward boundaries'
<|endoftext|>


<commit_before>from __future__ import with_statement
import os
import pickle
from contextlib import closing

class Cache(object):
    def __init__(self):
        self._cache = {}

    def get(self, k):
        return self._cache.get(k)

    def set(self, k, v):
        self._cache[k] = v

class PickleCache(Cache):
    def __init__(self, filename='cache.db'):
        self.filename = filename
        self._cache = self.load()

    def load(self):
        if os.path.exists(self.filename):
            with closing(open(self.filename)) as fh:
                contents = fh.read()
            return pickle.loads(contents)
        return {}

    def save(self):
        with closing(open(self.filename, 'w')) as fh:
            fh.write(pickle.dumps(self._cache))
<commit_msg>Adding a redis cache<commit_after>from __future__ import with_statement
import os
import pickle
from contextlib import closing
try:
    from redis import Redis
except ImportError:
    Redis = None

class Cache(object):
    def __init__(self):
        self._cache = {}

    def get(self, k):
        return self._cache.get(k)

    def set(self, k, v):
        self._cache[k] = v

class PickleCache(Cache):
    def __init__(self, filename='cache.db'):
        self.filename = filename
        self._cache = self.load()

    def load(self):
        if os.path.exists(self.filename):
            with closing(open(self.filename)) as fh:
                contents = fh.read()
            return pickle.loads(contents)
        return {}

    def save(self):
        with closing(open(self.filename, 'w')) as fh:
            fh.write(pickle.dumps(self._cache))

if Redis:
    class RedisCache(Cache):
        def __init__(self, namespace='micawber', **conn):
            self.namespace = namespace
            self.key_fn = lambda self, k: '%s.%s' % (self.namespace, k)
            self.conn = Redis(**conn)

        def get(self, k):
            cached = self.conn.get(self.key_fn(k))
            if cached:
                return pickle.loads(cached)

        def set(self, k, v):
            self.conn.set(self.key_fn(k), pickle.dumps(v))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Common functions that are needed in most other submodules.
"""

import argparse
import datetime
import hashlib
import logging
import os.path
import sys

from midas.compat import ZipFile

SAMPLE_CONFIG = """
[DEFAULTS]
user_name = thbach
hadoop_home = hdfs://localhost:9000/user/%(user_name)s
local_home = /home/$(user_name)s

[job]
mappers = 16
reducers = 28

[alexa]
top1m_files = %(hadoop_home)s/alexa-files#alexa-files
crunchbase_db = sqlite:////$(local_home)s/ba_data/crunchbase_db.sql
cut_hash_key = 3

[hadoop]
home = %(local_home)s/opt/hadoop-1.0.3
exec = %(home)s/bin/hadoop
streaming = %(home)/contrib/streaming/hadoop-streaming-1.0.3.jar

[loggers]
keys = root

[handlers]
keys = stderr

[formatters]
keys = generic

[logger_root]
level = INFO
handlers = stderr

[handler_stderr]
class = StreamHandler
args = (sys.stderr, )
level = NOTSET
formatter = generic

[formatter_generic]
format = %(asctime)s %(levelname)-5.5s %(message)s
"""

#: How much letters of the hash key should be used to pre-sort the
#: entries. The larger this number the more and smaller files
#: will be produced. The default value is `2` which gives us `16 * 16 =
#: 256` files.
CUT_HASH_KEY = 3

class MDCommand(object):
    """ Initializes the :class:`MDCommand` instance by passing `argv`
    (usually :attr:`sys.argv`) to :meth:`parser` and configuring the
    :mod:`logging`.

    Subclass this to write an easy to use command line interface to
    your function. But, make sure to call this class's
    :meth:`__init__` method when over-writing it.
    """

    def __init__(self, argv):
        self.args = self.parser.parse_args(argv[1:])
        self.args.verbosity = sum(self.args.verbosity)
        logging.basicConfig(level=self.args.verbosity, stream=sys.stderr)

    @classmethod
    def cmd(cls, argv=sys.argv):
        """ Initiates the class and calls its :meth:`run`
        method. Returns whatever :meth:`run` returns, except for
        :class:`None` which is interpreted as `0`.
        """
        obj = cls(argv)
        ret_val = obj.run()
        if ret_val is None:  # nothing means everything's fine
            return 0
        return ret_val

    def run(self):
        """ Overwrite this method with the code you want your command
        to execute. Return an integer unequal 0 if you want to signal
        that something went wrong.
        """
        raise NotImplementedError()

    @property
    def parser(self):
        """ Returns a :class:`argparse.ArgumentParser` object having
        `--verbose`, `--quiet` flags and reads from `stdin`.
        """
        if not hasattr(self, '_parser'):
            parser = argparse.ArgumentParser(description=self.__doc__)
            parser.add_argument('-v', '--verbose', dest='verbosity',
                                action='append_const', const=-10,
                                help='be more verbose, can be given once',
                                default=[logging.getLevelName('INFO')])
            parser.add_argument('-q', '--quiet', dest='verbosity',
                                action='append_const', const=10,
                                help='be quieter, can be given up to three times')
            parser.add_argument('stream', nargs='*', metavar='RECORD',
                                default=sys.stdin, help='the records to read')
            self._parser = parser
            self.add_argument()
        return self._parser

    def add_argument(self):
        """ Overwrite this function to add further arguments to
        :attr:`self.parser`.
        """
        pass

class RankEntry(object):
    """ Returns an entry of a ranking for `site`.

    The `date` specifies the date of the entry and should be a
    :class:`datetime.datetime` instance.

    The `rank` is expected to be an integer.

    Methods are provided both, to format a :class:`RankEntry` in
    standardized manners and to parse these formats back.
    """
    #: The standard format we use to produce and parse time-stamps.
    TS_FORMAT = '%Y-%m-%d'
    #: The time-stamp format as encoded in Alexa's Top1M files.
    ALEXA_TS_FORMAT = 'top-1m-%Y-%m-%d.csv.zip'

    def __init__(self, name, date, rank):
        self.name = name
        self.date = date
        self.rank = rank
        self._key = None

    def __str__(self):
        return 'RankEntry({e.name}, {e.tstamp}, {e.rank})'.format(e=self)

    def __eq__(self, other):
        return (self.name == other.name
                and self.date == other.date
                and self.rank == other.rank)

    def __lt__(self, other):
        if self.name < other.name:
            return True
        elif self.name > other.name:
            return False
        elif self.date < other.date:
            return True
        elif self.date > other.date:
            return False
        elif self.rank < other.rank:
            return True
        else:
            return False

    def __le__(self, other):
        return self == other or self < other

    def __ge__(self, other):
        return not self < other

    def __gt__(self, other):
        return not self <= other

    @property
    def format_std(self):
        " `'name[TAB]tstamp,[SPACE]rank'`. "
        return '{e.name}\t{e.tstamp}, {e.rank}'.format(e=self)

    @property
    def tstamp(self):
        " Return the `date` formated as defined in :attr:`TS_FORMAT`. "
        return self.date.strftime(self.TS_FORMAT)

    @property
    def format_w_key(self):
        " `'key[TAB]format_std'` "
        return '{e.key}\t{e.format_std}'.format(e=self)

    @property
    def key(self):
        """ The first :attr:`CUT_HASH_KEY` digits of the hash produced
        by :func:`hashlib.sha1` on the `name`. The key is computed and
        cached even though the `name` changes.
        """
        if self._key is None:
            self._key = self.make_key(self.name)
        return self._key

    @classmethod
    def make_key(cls, s):
        sha1 = hashlib.sha1(s.encode()).hexdigest()
        return sha1[:CUT_HASH_KEY]

    @classmethod
    def parse_key(cls, s):
        key, std = s.strip().split('\t', 1)
        return cls.parse_std(std)

    @classmethod
    def parse_std(cls, s):
        name, tail = s.strip().split('\t')
        date, rank = tail.split(', ')
        date = datetime.datetime.strptime(date, cls.TS_FORMAT)
        return cls(name=name, date=date, rank=rank)

    @classmethod
    def iter_alexa_file(cls, fname):
        """ Returns an iterator yielding a :class:`RankEntry` for
        every line it finds in the file in the archive `fname`.

        The standard Alexa Top1M data is a
        :class:`zipfile.ZipFile`. The archive contains one file named
        `top-1m.csv`. The lines in this file have the form
        `rank,site_name` (no spaces!).

        As the time-stamp is encoded in the file name (defined by
        :attr:`ALEXA_TS_FORMAT`) and not in the line, parsing cannot be
        done line-by-line but needs to be done on a per file basis.
        """
        fname = fname.strip()
        fname_last = os.path.basename(fname)
        date = datetime.datetime.strptime(fname_last, cls.ALEXA_TS_FORMAT)
        with ZipFile(fname) as zf:
            # The archive contains one file named ``top-1m.csv``
            for line in zf.open('top-1m.csv'):
                rank, name = line.decode().strip().split(',', 1)
                yield cls(name, date, int(rank))
<commit_msg>gardening<commit_after># -*- coding: utf-8 -*-
""" Common functions that are needed in most other submodules.
"""

import argparse
import datetime
import hashlib
import logging
import os.path
import sys

from midas.compat import ZipFile

SAMPLE_CONFIG = """
[DEFAULTS]
user_name = thbach
hadoop_home = hdfs://localhost:9000/user/%(user_name)s
local_home = /home/$(user_name)s

[job]
mappers = 16
reducers = 28

[alexa]
top1m_files = %(hadoop_home)s/alexa-files#alexa-files
crunchbase_db = sqlite:////$(local_home)s/ba_data/crunchbase_db.sql
cut_hash_key = 3

[hadoop]
home = %(local_home)s/opt/hadoop-1.0.3
exec = %(home)s/bin/hadoop
streaming = %(home)/contrib/streaming/hadoop-streaming-1.0.3.jar

[loggers]
keys = root

[handlers]
keys = stderr

[formatters]
keys = generic

[logger_root]
level = INFO
handlers = stderr

[handler_stderr]
class = StreamHandler
args = (sys.stderr, )
level = NOTSET
formatter = generic

[formatter_generic]
format = %(asctime)s %(levelname)-5.5s %(message)s
"""

#: How much letters of the hash key should be used to pre-sort the
#: entries. The larger this number the more and smaller files
#: will be produced. The default value is `2` which gives us `16 * 16 =
#: 256` files.
CUT_HASH_KEY = 3

class MDCommand(object):
    """ Initializes the :class:`MDCommand` instance by passing `argv`
    (usually :attr:`sys.argv`) to :meth:`parser` and configuring the
    :mod:`logging`.

    Subclass this to write an easy to use command line interface to
    your function. But, make sure to call this class's
    :meth:`__init__` method when over-writing it.
    """

    def __init__(self, argv):
        self.args = self.parser.parse_args(argv[1:])
        self.args.verbosity = sum(self.args.verbosity)
        logging.basicConfig(level=self.args.verbosity, stream=sys.stderr)

    @classmethod
    def cmd(cls, argv=sys.argv):
        """ Initiates the class and calls its :meth:`run`
        method. Returns whatever :meth:`run` returns, except for
        :class:`None` which is interpreted as `0`.
        """
        obj = cls(argv)
        ret_val = obj.run()
        # `None` means everything's fine
        return 0 if ret_val is None else ret_val

    def run(self):
        """ Overwrite this method with the code you want your command
        to execute. Return an integer unequal `0` if you want to
        signal that something went wrong.
        """
        raise NotImplementedError()

    @property
    def parser(self):
        """ Returns a :class:`argparse.ArgumentParser` object having
        `--verbose`, `--quiet` flags and reads from `stdin`.
        """
        if not hasattr(self, '_parser'):
            parser = argparse.ArgumentParser(description=self.__doc__)
            parser.add_argument('-v', '--verbose', dest='verbosity',
                                action='append_const', const=-10,
                                help='be more verbose, can be given once',
                                default=[logging.getLevelName('INFO')])
            parser.add_argument('-q', '--quiet', dest='verbosity',
                                action='append_const', const=10,
                                help='be quieter, can be given up to three times')
            parser.add_argument('stream', nargs='*', metavar='RECORD',
                                default=sys.stdin, help='the records to read')
            self._parser = parser
            self.add_argument()
        return self._parser

    def add_argument(self):
        """ Overwrite this function to add further arguments to
        :attr:`self.parser`.
        """
        pass

class RankEntry(object):
    """ Returns an entry of a ranking for `site`.

    The `date` specifies the date of the entry and should be a
    :class:`datetime.datetime` instance.

    The `rank` is expected to be an integer.

    Methods are provided both, to format a :class:`RankEntry` in
    standardized manners and to parse these formats back.
    """
    #: The standard format we use to produce and parse time-stamps.
    TS_FORMAT = '%Y-%m-%d'
    #: The time-stamp format as encoded in Alexa's Top1M files.
    ALEXA_TS_FORMAT = 'top-1m-%Y-%m-%d.csv.zip'

    def __init__(self, name, date, rank):
        self.name = name
        self.date = date
        self.rank = rank
        self._key = None

    def __str__(self):
        return 'RankEntry({e.name}, {e.tstamp}, {e.rank})'.format(e=self)

    def __eq__(self, other):
        return (self.name == other.name
                and self.date == other.date
                and self.rank == other.rank)

    def __lt__(self, other):
        if self.name < other.name:
            return True
        elif self.name > other.name:
            return False
        elif self.date < other.date:
            return True
        elif self.date > other.date:
            return False
        elif self.rank < other.rank:
            return True
        else:
            return False

    def __le__(self, other):
        return self == other or self < other

    def __ge__(self, other):
        return not self < other

    def __gt__(self, other):
        return not self <= other

    @property
    def format_std(self):
        " `'name[TAB]tstamp,[SPACE]rank'`. "
        return '{e.name}\t{e.tstamp}, {e.rank}'.format(e=self)

    @property
    def tstamp(self):
        " Return the `date` formated as defined in :attr:`TS_FORMAT`. "
        return self.date.strftime(self.TS_FORMAT)

    @property
    def format_w_key(self):
        " `'key[TAB]format_std'` "
        return '{e.key}\t{e.format_std}'.format(e=self)

    @property
    def key(self):
        """ The first :attr:`CUT_HASH_KEY` digits of the hash produced
        by :func:`hashlib.sha1` on the `name`. The key is computed and
        cached even though the `name` changes.
        """
        if self._key is None:
            self._key = self.make_key(self.name)
        return self._key

    @classmethod
    def make_key(cls, s):
        sha1 = hashlib.sha1(s.encode()).hexdigest()
        return sha1[:CUT_HASH_KEY]

    @classmethod
    def parse_key(cls, s):
        key, std = s.strip().split('\t', 1)
        return cls.parse_std(std)

    @classmethod
    def parse_std(cls, s):
        name, tail = s.strip().split('\t')
        date, rank = tail.split(', ')
        date = datetime.datetime.strptime(date, cls.TS_FORMAT)
        return cls(name=name, date=date, rank=rank)

    @classmethod
    def iter_alexa_file(cls, fname):
        """ Returns an iterator yielding a :class:`RankEntry` for
        every line it finds in the file in the archive `fname`.

        The standard Alexa Top1M data is a
        :class:`zipfile.ZipFile`. The archive contains one file named
        `top-1m.csv`. The lines in this file have the form
        `rank,site_name` (no spaces!).

        As the time-stamp is encoded in the file name (defined by
        :attr:`ALEXA_TS_FORMAT`) and not in the line, parsing cannot be
        done line-by-line but needs to be done on a per file basis.
        """
        fname = fname.strip()
        fname_last = os.path.basename(fname)
        date = datetime.datetime.strptime(fname_last, cls.ALEXA_TS_FORMAT)
        with ZipFile(fname) as zf:
            # The archive contains one file named ``top-1m.csv``
            for line in zf.open('top-1m.csv'):
                rank, name = line.decode().strip().split(',', 1)
                yield cls(name, date, int(rank))
<|endoftext|>


<commit_before>import yaml
import time
import os
import shutil

from moulinette import m18n
from moulinette.core import init_authenticator
from yunohost.utils.error import YunohostError
from moulinette.utils.log import getActionLogger

from yunohost.tools import Migration
from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory
from yunohost.user import user_list, user_group_add, user_group_update
from yunohost.app import app_setting, app_list
from yunohost.service import service_regen_conf
from yunohost.permission import permission_add, permission_sync_to_user
from yunohost.user import user_permission_add

logger = getActionLogger('yunohost.migration')

###################################################
# Tools used also for restoration
###################################################

def migrate_LDAP_db(auth):
    logger.info(m18n.n("migration_0009_update_LDAP_database"))
    try:
        auth.remove('cn=sftpusers,ou=groups')
    except:
        logger.warn(m18n.n("error_when_removing_sftpuser_group"))

    with open('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') as f:
        ldap_map = yaml.load(f)

    try:
        attr_dict = ldap_map['parents']['ou=permission']
        auth.add('ou=permission', attr_dict)

        attr_dict = ldap_map['children']['cn=all_users,ou=groups']
        auth.add('cn=all_users,ou=groups', attr_dict)

        for rdn, attr_dict in ldap_map['depends_children'].items():
            auth.add(rdn, attr_dict)
    except Exception as e:
        raise YunohostError("migration_0009_LDAP_update_failed", error=e)

    logger.info(m18n.n("migration_0009_create_group"))

    #Create a group for each yunohost user
    user_list = auth.search('ou=users,dc=yunohost,dc=org',
                            '(&(objectclass=person)(!(uid=root))(!(uid=nobody)))',
                            ['uid', 'uidNumber'])
    for user_info in user_list:
        username = user_info['uid'][0]
        auth.update('uid=%s,ou=users' % username,
                    {'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh']})
        user_group_add(auth, username, gid=user_info['uidNumber'][0], sync_perm=False)
        user_group_update(auth, groupname=username, add_user=username, force=True, sync_perm=False)
        user_group_update(auth, groupname='all_users', add_user=username, force=True, sync_perm=False)

def migrate_app_permission(auth, app=None):
    logger.info(m18n.n("migration_0009_migrate_permission"))

    if app:
        apps = app_list(installed=True, filter=app)['apps']
    else:
        apps = app_list(installed=True)['apps']

    for app_info in apps:
        app = app_info['id']
        permission = app_setting(app, 'allowed_users')
        path = app_setting(app, 'path')
        domain = app_setting(app, 'domain')

        urls = [domain + path] if domain and path else None
        permission_add(auth, app, permission='main', urls=urls, default_allow=True, sync_perm=False)
        if permission:
            allowed_group = permission.split(',')
            user_permission_add(auth, [app], permission='main', group=allowed_group, sync_perm=False)
        app_setting(app, 'allowed_users', delete=True)

class MyMigration(Migration):
    """
        Update the LDAP DB to be able to store the permission
        Create a group for each yunohost user
        Migrate app permission from apps setting to LDAP
    """

    required = True

    def migrate(self):
        # Backup LDAP and the apps settings before to do the migration
        logger.info(m18n.n("migration_0009_backup_before_migration"))
        try:
            backup_folder = "/home/yunohost.backup/premigration/" + time.strftime('%Y%m%d-%H%M%S', time.gmtime())
            os.makedirs(backup_folder, 0o750)
            os.system("systemctl stop slapd")
            os.system("cp -r --preserve /etc/ldap %s/ldap_config" % backup_folder)
            os.system("cp -r --preserve /var/lib/ldap %s/ldap_db" % backup_folder)
            os.system("cp -r --preserve /etc/yunohost/apps %s/apps_settings" % backup_folder)
            os.system("systemctl start slapd")
        except Exception as e:
            raise YunohostError("migration_0009_can_not_backup_before_migration", error=e)

        # Update LDAP schema restart slapd
        logger.info(m18n.n("migration_0009_update_LDAP_schema"))
        service_regen_conf(names=['slapd'], force=True)

        # Do the authentication to LDAP after LDAP as been updated
        AUTH_IDENTIFIER = ('ldap', 'as-root')
        AUTH_PARAMETERS = {'uri': 'ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi',
                           'base_dn': 'dc=yunohost,dc=org',
                           'user_rdn': 'gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth'}
        auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS)

        try:
            #Update LDAP database
            migrate_LDAP_db(auth)

            # Migrate permission
            migrate_app_permission(auth)

            permission_sync_to_user(auth)
        except Exception as e:
            logger.warn(m18n.n("migration_0009_migration_failed_trying_to_rollback"))
            os.system("systemctl stop slapd")
            os.system("rm -r /etc/ldap/slapd.d") # To be sure that we don't keep some part of the old config
            os.system("cp -r --preserve %s/ldap_config/. /etc/ldap/" % backup_folder)
            os.system("cp -r --preserve %s/ldap_db/. /var/lib/ldap/" % backup_folder)
            os.system("cp -r --preserve %s/apps_settings/. /etc/yunohost/apps/" % backup_folder)
            os.system("systemctl start slapd")
            os.system("rm -r " + backup_folder)
            logger.info(m18n.n("migration_0009_rollback_success"))
            raise

        os.system("rm -r " + backup_folder)

        logger.info(m18n.n("migration_0009_done"))
<commit_msg>Let's be paranoid and include all that stuff in the try/except as well<commit_after>import yaml
import time
import os
import shutil

from moulinette import m18n
from moulinette.core import init_authenticator
from yunohost.utils.error import YunohostError
from moulinette.utils.log import getActionLogger

from yunohost.tools import Migration
from yunohost.utils.filesystem import free_space_in_directory, space_used_by_directory
from yunohost.user import user_list, user_group_add, user_group_update
from yunohost.app import app_setting, app_list
from yunohost.service import service_regen_conf
from yunohost.permission import permission_add, permission_sync_to_user
from yunohost.user import user_permission_add

logger = getActionLogger('yunohost.migration')

###################################################
# Tools used also for restoration
###################################################

def migrate_LDAP_db(auth):
    logger.info(m18n.n("migration_0009_update_LDAP_database"))
    try:
        auth.remove('cn=sftpusers,ou=groups')
    except:
        logger.warn(m18n.n("error_when_removing_sftpuser_group"))

    with open('/usr/share/yunohost/yunohost-config/moulinette/ldap_scheme.yml') as f:
        ldap_map = yaml.load(f)

    try:
        attr_dict = ldap_map['parents']['ou=permission']
        auth.add('ou=permission', attr_dict)

        attr_dict = ldap_map['children']['cn=all_users,ou=groups']
        auth.add('cn=all_users,ou=groups', attr_dict)

        for rdn, attr_dict in ldap_map['depends_children'].items():
            auth.add(rdn, attr_dict)
    except Exception as e:
        raise YunohostError("migration_0009_LDAP_update_failed", error=e)

    logger.info(m18n.n("migration_0009_create_group"))

    #Create a group for each yunohost user
    user_list = auth.search('ou=users,dc=yunohost,dc=org',
                            '(&(objectclass=person)(!(uid=root))(!(uid=nobody)))',
                            ['uid', 'uidNumber'])
    for user_info in user_list:
        username = user_info['uid'][0]
        auth.update('uid=%s,ou=users' % username,
                    {'objectClass': ['mailAccount', 'inetOrgPerson', 'posixAccount', 'userPermissionYnh']})
        user_group_add(auth, username, gid=user_info['uidNumber'][0], sync_perm=False)
        user_group_update(auth, groupname=username, add_user=username, force=True, sync_perm=False)
        user_group_update(auth, groupname='all_users', add_user=username, force=True, sync_perm=False)

def migrate_app_permission(auth, app=None):
    logger.info(m18n.n("migration_0009_migrate_permission"))

    if app:
        apps = app_list(installed=True, filter=app)['apps']
    else:
        apps = app_list(installed=True)['apps']

    for app_info in apps:
        app = app_info['id']
        permission = app_setting(app, 'allowed_users')
        path = app_setting(app, 'path')
        domain = app_setting(app, 'domain')

        urls = [domain + path] if domain and path else None
        permission_add(auth, app, permission='main', urls=urls, default_allow=True, sync_perm=False)
        if permission:
            allowed_group = permission.split(',')
            user_permission_add(auth, [app], permission='main', group=allowed_group, sync_perm=False)
        app_setting(app, 'allowed_users', delete=True)

class MyMigration(Migration):
    """
        Update the LDAP DB to be able to store the permission
        Create a group for each yunohost user
        Migrate app permission from apps setting to LDAP
    """

    required = True

    def migrate(self):
        # Backup LDAP and the apps settings before to do the migration
        logger.info(m18n.n("migration_0009_backup_before_migration"))
        try:
            backup_folder = "/home/yunohost.backup/premigration/" + time.strftime('%Y%m%d-%H%M%S', time.gmtime())
            os.makedirs(backup_folder, 0o750)
            os.system("systemctl stop slapd")
            os.system("cp -r --preserve /etc/ldap %s/ldap_config" % backup_folder)
            os.system("cp -r --preserve /var/lib/ldap %s/ldap_db" % backup_folder)
            os.system("cp -r --preserve /etc/yunohost/apps %s/apps_settings" % backup_folder)
            os.system("systemctl start slapd")
        except Exception as e:
            raise YunohostError("migration_0009_can_not_backup_before_migration", error=e)

        try:
            # Update LDAP schema restart slapd
            logger.info(m18n.n("migration_0009_update_LDAP_schema"))
            service_regen_conf(names=['slapd'], force=True)

            # Do the authentication to LDAP after LDAP as been updated
            AUTH_IDENTIFIER = ('ldap', 'as-root')
            AUTH_PARAMETERS = {'uri': 'ldapi://%2Fvar%2Frun%2Fslapd%2Fldapi',
                               'base_dn': 'dc=yunohost,dc=org',
                               'user_rdn': 'gidNumber=0+uidNumber=0,cn=peercred,cn=external,cn=auth'}
            auth = init_authenticator(AUTH_IDENTIFIER, AUTH_PARAMETERS)

            #Update LDAP database
            migrate_LDAP_db(auth)

            # Migrate permission
            migrate_app_permission(auth)

            permission_sync_to_user(auth)
        except Exception as e:
            logger.warn(m18n.n("migration_0009_migration_failed_trying_to_rollback"))
            os.system("systemctl stop slapd")
            os.system("rm -r /etc/ldap/slapd.d") # To be sure that we don't keep some part of the old config
            os.system("cp -r --preserve %s/ldap_config/. /etc/ldap/" % backup_folder)
            os.system("cp -r --preserve %s/ldap_db/. /var/lib/ldap/" % backup_folder)
            os.system("cp -r --preserve %s/apps_settings/. /etc/yunohost/apps/" % backup_folder)
            os.system("systemctl start slapd")
            os.system("rm -r " + backup_folder)
            logger.info(m18n.n("migration_0009_rollback_success"))
            raise
        else:
            os.system("rm -r " + backup_folder)

            logger.info(m18n.n("migration_0009_done"))
<|endoftext|>