<commit_before>from collections import deque
import datetime
import os
import re
import sys

from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.datastructures import SortedDict

from south import exceptions
from south.migration.utils import depends, dfs, flatten, get_app_name
from south.orm import LazyFakeORM, FakeORM
from south.utils import memoize

def all_migrations(applications=None):
    """
    Returns all Migrations for all `applications` that are migrated.
    """
    if applications is None:
        applications = models.get_apps()
    for app in applications:
        try:
            yield Migrations(app)
        except exceptions.NoMigrations:
            pass

def Migrations(application):
    if isinstance(application, basestring):
        app_name = application
    else:
        app_name = application.__name__
    if app_name not in Migrations.cache:
        Migrations.cache[app_name] = _Migrations(application)
    return Migrations.cache[app_name]
Migrations.cache = {}

class _Migrations(list):
    """
    Holds a list of Migration objects for a particular app.
    """

    MIGRATION_FILENAME = re.compile(r'(?!__init__)' # Don't match __init__.py
                                    r'[^.]*'        # Don't match dotfiles
                                    r'\.py$')       # Match only .py files

    def __new__(cls, application):
        if isinstance(application, basestring):
            return cls.from_name(application)
        return super(_Migrations, cls).__new__(cls)

    def __init__(self, application):
        if hasattr(application, '__name__'):
            self._cache = {}
            self.application = application

    @classmethod
    def from_name(cls, app_name):
        app = models.get_app(app_name)
        module_name = get_app_name(app)
        try:
            module = sys.modules[module_name]
        except KeyError:
            __import__(module_name, {}, {}, [''])
            module = sys.modules[module_name]
        return cls(module)

    def get_application(self):
        return self._application

    def set_application(self, application):
        self._application = application
        if not hasattr(application, 'migrations'):
            try:
                module = __import__(application.__name__ + '.migrations', {}, {})
                application.migrations = module.migrations
                self._migrations = application.migrations
            except ImportError:
                raise exceptions.NoMigrations(application)
        self._load_migrations_module(application.migrations)

    application = property(get_application, set_application)

    def _load_migrations_module(self, module):
        self._migrations = module
        filenames = []
        dirname = os.path.dirname(self._migrations.__file__)
        for f in os.listdir(dirname):
            if self.MIGRATION_FILENAME.match(os.path.basename(f)):
                filenames.append(f)
        filenames.sort()
        self.extend(self.migration(f) for f in filenames)

    def migration(self, filename):
        name = Migration.strip_filename(filename)
        if name not in self._cache:
            self._cache[name] = Migration(self, name)
        return self._cache[name]

    def __getitem__(self, value):
        if isinstance(value, basestring):
            return self.migration(value)
        return super(_Migrations, self).__getitem__(value)

    def _guess_migration(self, prefix):
        prefix = Migration.strip_filename(prefix)
        matches = [m for m in self if m.name().startswith(prefix)]
        if len(matches) == 1:
            return matches[0]
        elif len(matches) > 1:
            raise exceptions.MultiplePrefixMatches(prefix, matches)
        else:
            raise exceptions.UnknownMigration(prefix, None)

    def guess_migration(self, target_name):
        if target_name == 'zero' or not self:
            return
        elif target_name is None:
            return self[-1]
        else:
            return self._guess_migration(prefix=target_name)

    def app_name(self):
        return get_app_name(self._migrations)

    def full_name(self):
        return self._migrations.__name__

    def calculate_dependents(self):
        for migrations in all_migrations():
            for migration in migrations:
                migration.add_dependent(None)
                for dependency in migration.dependencies():
                    dependency.add_dependent(migration)

class Migration(object):
    def __init__(self, migrations, filename):
        """
        Returns the migration class implied by 'filename'.
        """
        self.migrations = migrations
        self.filename = filename

    def __str__(self):
        return self.app_name() + ':' + self.name()

    def __repr__(self):
        return u'<Migration: %s>' % unicode(self)

    def app_name(self):
        return self.migrations.app_name()

    @staticmethod
    def strip_filename(filename):
        return os.path.splitext(os.path.basename(filename))[0]

    def name(self):
        return self.strip_filename(os.path.basename(self.filename))

    def full_name(self):
        return self.migrations.full_name() + '.' + self.name()

    def migration(self):
        full_name = self.full_name()
        app_name = self.app_name()
        try:
            migration = sys.modules[full_name]
        except KeyError:
            try:
                migration = __import__(full_name, '', '', ['Migration'])
            except ImportError, e:
                raise exceptions.UnknownMigration(self, sys.exc_info())
            except Exception, e:
                raise exceptions.BrokenMigration(self, sys.exc_info())
        # Override some imports
        migration._ = lambda x: x  # Fake i18n
        migration.datetime = datetime
        return migration
    migration = memoize(migration)

    def migration_class(self):
        return self.migration().Migration

    def migration_instance(self):
        return self.migration_class()()
    migration_instance = memoize(migration_instance)

    def previous(self):
        index = self.migrations.index(self) - 1
        if index < 0:
            return None
        return self.migrations[index]
    previous = memoize(previous)

    def next(self):
        index = self.migrations.index(self) + 1
        if index >= len(self.migrations):
            return None
        return self.migrations[index]
    next = memoize(next)

    def dependencies(self):
        result = [self.previous()]
        if result[0] is None:
            result = []
        # Get forwards dependencies
        for app, name in getattr(self.migration_class(), 'depends_on', []):
            try:
                migrations = Migrations(app)
            except ImproperlyConfigured:
                raise exceptions.DependsOnUnmigratedApplication(self, app)
            migration = migrations.migration(name)
            try:
                migration.migration()
            except exceptions.UnknownMigration:
                raise exceptions.DependsOnUnknownMigration(self, migration)
            if migration.is_before(self) == False:
                raise exceptions.DependsOnHigherMigration(self, migration)
            result.append(migration)
        return result
    dependencies = memoize(dependencies)

    def add_dependent(self, migration):
        if not hasattr(self, '_dependents'):
            self._dependents = deque()
        if migration and migration not in self._dependents:
            self._dependents.appendleft(migration)

    def dependents(self):
        self.migrations.calculate_dependents()
        return self._dependents
    dependents = memoize(dependents)

    def forwards(self):
        return self.migration_instance().forwards

    def backwards(self):
        return self.migration_instance().backwards

    def _forwards_plan(self):
        result = SortedDict()
        # We need to apply all the migrations this one depends on
        for migration in self.dependencies():
            result.update(migration._forwards_plan())
        # Append ourselves to the result
        result[self] = None
        return result

    def forwards_plan(self):
        """
        Returns a list of Migration objects to be applied, in order.

        This list includes `self`, which will be applied last.
        """
        return depends(self, self.__class__.dependencies)

    def _backwards_plan(self):
        return depends(self, self.__class__.dependents)

    def backwards_plan(self):
        """
        Returns a list of Migration objects to be unapplied, in order.

        This list includes `self`, which will be unapplied last.
        """
        return list(self._backwards_plan())

    def is_before(self, other):
        if self.migrations == other.migrations:
            if self.filename < other.filename:
                return True
            return False

    def is_after(self, other):
        if self.migrations == other.migrations:
            if self.filename > other.filename:
                return True
            return False

    def prev_orm(self):
        previous = self.previous()
        if previous is None:
            # First migration? The 'previous ORM' is empty.
            return FakeORM(None, self.app_name())
        return previous.orm()
    prev_orm = memoize(prev_orm)

    def orm(self):
        return LazyFakeORM(self.migration().Migration, self.app_name())
    orm = memoize(orm)

    def no_dry_run(self):
        migration_class = self.migration_class()
        try:
            return migration_class.no_dry_run
        except AttributeError:
            return False
<commit_msg>Remove Migration._forwards_plan()<commit_after>from collections import deque
import datetime
import os
import re
import sys

from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.datastructures import SortedDict

from south import exceptions
from south.migration.utils import depends, dfs, flatten, get_app_name
from south.orm import LazyFakeORM, FakeORM
from south.utils import memoize

def all_migrations(applications=None):
    """
    Returns all Migrations for all `applications` that are migrated.
    """
    if applications is None:
        applications = models.get_apps()
    for app in applications:
        try:
            yield Migrations(app)
        except exceptions.NoMigrations:
            pass

def Migrations(application):
    if isinstance(application, basestring):
        app_name = application
    else:
        app_name = application.__name__
    if app_name not in Migrations.cache:
        Migrations.cache[app_name] = _Migrations(application)
    return Migrations.cache[app_name]
Migrations.cache = {}

class _Migrations(list):
    """
    Holds a list of Migration objects for a particular app.
    """

    MIGRATION_FILENAME = re.compile(r'(?!__init__)' # Don't match __init__.py
                                    r'[^.]*'        # Don't match dotfiles
                                    r'\.py$')       # Match only .py files

    def __new__(cls, application):
        if isinstance(application, basestring):
            return cls.from_name(application)
        return super(_Migrations, cls).__new__(cls)

    def __init__(self, application):
        if hasattr(application, '__name__'):
            self._cache = {}
            self.application = application

    @classmethod
    def from_name(cls, app_name):
        app = models.get_app(app_name)
        module_name = get_app_name(app)
        try:
            module = sys.modules[module_name]
        except KeyError:
            __import__(module_name, {}, {}, [''])
            module = sys.modules[module_name]
        return cls(module)

    def get_application(self):
        return self._application

    def set_application(self, application):
        self._application = application
        if not hasattr(application, 'migrations'):
            try:
                module = __import__(application.__name__ + '.migrations', {}, {})
                application.migrations = module.migrations
                self._migrations = application.migrations
            except ImportError:
                raise exceptions.NoMigrations(application)
        self._load_migrations_module(application.migrations)

    application = property(get_application, set_application)

    def _load_migrations_module(self, module):
        self._migrations = module
        filenames = []
        dirname = os.path.dirname(self._migrations.__file__)
        for f in os.listdir(dirname):
            if self.MIGRATION_FILENAME.match(os.path.basename(f)):
                filenames.append(f)
        filenames.sort()
        self.extend(self.migration(f) for f in filenames)

    def migration(self, filename):
        name = Migration.strip_filename(filename)
        if name not in self._cache:
            self._cache[name] = Migration(self, name)
        return self._cache[name]

    def __getitem__(self, value):
        if isinstance(value, basestring):
            return self.migration(value)
        return super(_Migrations, self).__getitem__(value)

    def _guess_migration(self, prefix):
        prefix = Migration.strip_filename(prefix)
        matches = [m for m in self if m.name().startswith(prefix)]
        if len(matches) == 1:
            return matches[0]
        elif len(matches) > 1:
            raise exceptions.MultiplePrefixMatches(prefix, matches)
        else:
            raise exceptions.UnknownMigration(prefix, None)

    def guess_migration(self, target_name):
        if target_name == 'zero' or not self:
            return
        elif target_name is None:
            return self[-1]
        else:
            return self._guess_migration(prefix=target_name)

    def app_name(self):
        return get_app_name(self._migrations)

    def full_name(self):
        return self._migrations.__name__

    def calculate_dependents(self):
        for migrations in all_migrations():
            for migration in migrations:
                migration.add_dependent(None)
                for dependency in migration.dependencies():
                    dependency.add_dependent(migration)

class Migration(object):
    def __init__(self, migrations, filename):
        """
        Returns the migration class implied by 'filename'.
        """
        self.migrations = migrations
        self.filename = filename

    def __str__(self):
        return self.app_name() + ':' + self.name()

    def __repr__(self):
        return u'<Migration: %s>' % unicode(self)

    def app_name(self):
        return self.migrations.app_name()

    @staticmethod
    def strip_filename(filename):
        return os.path.splitext(os.path.basename(filename))[0]

    def name(self):
        return self.strip_filename(os.path.basename(self.filename))

    def full_name(self):
        return self.migrations.full_name() + '.' + self.name()

    def migration(self):
        full_name = self.full_name()
        app_name = self.app_name()
        try:
            migration = sys.modules[full_name]
        except KeyError:
            try:
                migration = __import__(full_name, '', '', ['Migration'])
            except ImportError, e:
                raise exceptions.UnknownMigration(self, sys.exc_info())
            except Exception, e:
                raise exceptions.BrokenMigration(self, sys.exc_info())
        # Override some imports
        migration._ = lambda x: x  # Fake i18n
        migration.datetime = datetime
        return migration
    migration = memoize(migration)

    def migration_class(self):
        return self.migration().Migration

    def migration_instance(self):
        return self.migration_class()()
    migration_instance = memoize(migration_instance)

    def previous(self):
        index = self.migrations.index(self) - 1
        if index < 0:
            return None
        return self.migrations[index]
    previous = memoize(previous)

    def next(self):
        index = self.migrations.index(self) + 1
        if index >= len(self.migrations):
            return None
        return self.migrations[index]
    next = memoize(next)

    def dependencies(self):
        result = [self.previous()]
        if result[0] is None:
            result = []
        # Get forwards dependencies
        for app, name in getattr(self.migration_class(), 'depends_on', []):
            try:
                migrations = Migrations(app)
            except ImproperlyConfigured:
                raise exceptions.DependsOnUnmigratedApplication(self, app)
            migration = migrations.migration(name)
            try:
                migration.migration()
            except exceptions.UnknownMigration:
                raise exceptions.DependsOnUnknownMigration(self, migration)
            if migration.is_before(self) == False:
                raise exceptions.DependsOnHigherMigration(self, migration)
            result.append(migration)
        return result
    dependencies = memoize(dependencies)

    def add_dependent(self, migration):
        if not hasattr(self, '_dependents'):
            self._dependents = deque()
        if migration and migration not in self._dependents:
            self._dependents.appendleft(migration)

    def dependents(self):
        self.migrations.calculate_dependents()
        return self._dependents
    dependents = memoize(dependents)

    def forwards(self):
        return self.migration_instance().forwards

    def backwards(self):
        return self.migration_instance().backwards

    def forwards_plan(self):
        """
        Returns a list of Migration objects to be applied, in order.

        This list includes `self`, which will be applied last.
        """
        return depends(self, self.__class__.dependencies)

    def _backwards_plan(self):
        return depends(self, self.__class__.dependents)

    def backwards_plan(self):
        """
        Returns a list of Migration objects to be unapplied, in order.

        This list includes `self`, which will be unapplied last.
        """
        return list(self._backwards_plan())

    def is_before(self, other):
        if self.migrations == other.migrations:
            if self.filename < other.filename:
                return True
            return False

    def is_after(self, other):
        if self.migrations == other.migrations:
            if self.filename > other.filename:
                return True
            return False

    def prev_orm(self):
        previous = self.previous()
        if previous is None:
            # First migration? The 'previous ORM' is empty.
            return FakeORM(None, self.app_name())
        return previous.orm()
    prev_orm = memoize(prev_orm)

    def orm(self):
        return LazyFakeORM(self.migration().Migration, self.app_name())
    orm = memoize(orm)

    def no_dry_run(self):
        migration_class = self.migration_class()
        try:
            return migration_class.no_dry_run
        except AttributeError:
            return False
<|endoftext|>


<commit_before>"""VCS tests: Git."""

import os
import subprocess
from ..testing import COMMIT_USER_EMAIL
from ..testing import COMMIT_USER_NAME
from ..testing import VcsTestCase
from ..git import GitRepo
from ...utils import working_directory_keeper

class GitBaseTestCase(VcsTestCase):
    """Common utilities for Git test cases."""

    def create_src(self):
        os.chdir(self.src_dir)
        subprocess.call(['git', 'init', 'src-branch'])
        self.src_repo = os.path.join(self.src_dir, 'src-branch')
        os.chdir(self.src_repo)
        # repo configuration is local by default
        subprocess.call(['git', 'config', 'user.email', COMMIT_USER_EMAIL])
        subprocess.call(['git', 'config', 'user.name', COMMIT_USER_NAME])
        f = open('tracked', 'w')
        f.write("first" + os.linesep)
        f.close()
        subprocess.call(['git', 'add', 'tracked'])
        subprocess.call(['git', 'commit', '-m', 'initial commit'])
        f = open('tracked', 'w')
        f.write("last" + os.linesep)
        f.close()
        subprocess.call(['git', 'add', 'tracked'])
        subprocess.call(['git', 'commit', '-m', 'last version'])

class GitTestCase(GitBaseTestCase):

    def test_clone(self):
        """Git clone."""
        target_dir = os.path.join(self.dst_dir, "My clone")
        GitRepo(target_dir, self.src_repo)('master')

        self.assertTrue(os.path.isdir(target_dir))
        f = open(os.path.join(target_dir, 'tracked'))
        lines = f.readlines()
        f.close()
        self.assertEquals(lines[0].strip(), 'last')

    def test_clean(self):
        target_dir = os.path.join(self.dst_dir, "My clone")
        repo = GitRepo(target_dir, self.src_repo)
        try:
            repo.clean()
        except:
            self.fail("clean() should not fail if "
                      "clone not already done")

        repo('master')

        dirty_dir = os.path.join(repo.target_dir, 'dirty')
        os.mkdir(dirty_dir)
        dirty_files = (os.path.join(repo.target_dir, 'untracked'),
                       os.path.join(dirty_dir, 'untracked2'))
        for path in dirty_files:
            with open(path, 'w') as f:
                f.write('content')
        repo.clean()
        for path in dirty_files:
            self.failIf(os.path.exists(path),
                        "Untracked file should have been removed")
        self.failIf(os.path.exists(dirty_dir),
                    "Untracked dir should have been removed")

    def test_clone_on_sha(self):
        """Git clone."""
        target_dir = os.path.join(self.dst_dir, "My clone")
        repo = GitRepo(target_dir, self.src_repo)
        repo('master')
        sha = repo.parents()[0]
        target_dir = os.path.join(self.dst_dir, "My clone 2")
        repo = GitRepo(target_dir, self.src_repo)
        repo(sha)
        sha2 = repo.parents()[0]
        self.assert_(sha == sha2, 'Bad clone on SHA')

    def test_uncommitted_changes(self):
        """GitRepo can detect uncommitted changes."""
        # initial cloning
        target_dir = os.path.join(self.dst_dir, "clone to update")
        repo = GitRepo(target_dir, self.src_repo)
        repo('master')

        self.assertFalse(repo.uncommitted_changes())

        f = open(os.path.join(target_dir, 'tracked'), 'w')
        f.write('mod')
        f.close()

        self.assertTrue(repo.uncommitted_changes())

class GitBranchTestCase(GitBaseTestCase):

    def create_src(self):
        GitBaseTestCase.create_src(self)
        os.chdir(self.src_repo)
        self.make_branch(self.src_repo, 'somebranch')

    def make_branch(self, src_dir, name):
        """create a branch
        """
        subprocess.check_call(['git', 'branch', name], cwd=src_dir)

    def test_switch_local_branch(self):
        """Switch to a branch created before the clone.

        In this case, the branch already exists in local repo
        """

        # Testing starts here
        target_dir = os.path.join(self.dst_dir, "to_branch")
        branch = GitRepo(target_dir, self.src_repo)
        #update file from master after branching
        branch("master")
        with working_directory_keeper:
            os.chdir(target_dir)
            f = open('tracked', 'w')
            f.write("last after branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])

        # check that no changes exists when switching from one to other
        branch('somebranch')
        self.assertFalse(branch.uncommitted_changes())
        branch('master')
        self.assertFalse(branch.uncommitted_changes())

        #modify the branch
        branch('somebranch')
        self.assertFalse(branch.uncommitted_changes())
        self.assertFalse(branch.uncommitted_changes())
        with working_directory_keeper:
            os.chdir(target_dir)
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), 'last')
            f = open('tracked', 'w')
            f.write("last from branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), 'last from branch')

        # switch to master
        branch('master')
        self.assertFalse(branch.uncommitted_changes())
        f = open(os.path.join(target_dir, 'tracked'))
        lines = f.readlines()
        f.close()
        self.assertEquals(lines[0].strip(), 'last after branch')
        self.assertFalse(branch.uncommitted_changes())

    def test_switch_remote_branch(self):
        """Switch to a branch created after the clone.

        In this case, the branch doesn't exist in local repo
        """
        # init the local clone
        # Testing starts here
        target_dir = os.path.join(self.dst_dir, "to_branch")
        branch = GitRepo(target_dir, self.src_repo)
        #update file from master after branching
        branch("master")

        # create the remote branch with some modifications
        self.make_branch(self.src_repo, 'remotebranch')
        with working_directory_keeper:
            os.chdir(self.src_repo)
            subprocess.call(['git', 'checkout', 'remotebranch'])
            f = open('tracked', 'w')
            f.write("last after remote branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])

        # switch to the remote branch and check tracked file
        branch("remotebranch")

        with working_directory_keeper:
            os.chdir(target_dir)
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), "last after remote branch")
<commit_msg>Test failed on some buildslaves due to missing user.email&name<commit_after>"""VCS tests: Git."""

import os
import subprocess
from ..testing import COMMIT_USER_EMAIL
from ..testing import COMMIT_USER_NAME
from ..testing import VcsTestCase
from ..git import GitRepo
from ...utils import working_directory_keeper

class GitBaseTestCase(VcsTestCase):
    """Common utilities for Git test cases."""

    def create_src(self):
        os.chdir(self.src_dir)
        subprocess.call(['git', 'init', 'src-branch'])
        self.src_repo = os.path.join(self.src_dir, 'src-branch')
        os.chdir(self.src_repo)
        # repo configuration is local by default
        subprocess.call(['git', 'config', 'user.email', COMMIT_USER_EMAIL])
        subprocess.call(['git', 'config', 'user.name', COMMIT_USER_NAME])
        f = open('tracked', 'w')
        f.write("first" + os.linesep)
        f.close()
        subprocess.call(['git', 'add', 'tracked'])
        subprocess.call(['git', 'commit', '-m', 'initial commit'])
        f = open('tracked', 'w')
        f.write("last" + os.linesep)
        f.close()
        subprocess.call(['git', 'add', 'tracked'])
        subprocess.call(['git', 'commit', '-m', 'last version'])

class GitTestCase(GitBaseTestCase):

    def test_clone(self):
        """Git clone."""
        target_dir = os.path.join(self.dst_dir, "My clone")
        GitRepo(target_dir, self.src_repo)('master')

        self.assertTrue(os.path.isdir(target_dir))
        f = open(os.path.join(target_dir, 'tracked'))
        lines = f.readlines()
        f.close()
        self.assertEquals(lines[0].strip(), 'last')

    def test_clean(self):
        target_dir = os.path.join(self.dst_dir, "My clone")
        repo = GitRepo(target_dir, self.src_repo)
        try:
            repo.clean()
        except:
            self.fail("clean() should not fail if "
                      "clone not already done")

        repo('master')

        dirty_dir = os.path.join(repo.target_dir, 'dirty')
        os.mkdir(dirty_dir)
        dirty_files = (os.path.join(repo.target_dir, 'untracked'),
                       os.path.join(dirty_dir, 'untracked2'))
        for path in dirty_files:
            with open(path, 'w') as f:
                f.write('content')
        repo.clean()
        for path in dirty_files:
            self.failIf(os.path.exists(path),
                        "Untracked file should have been removed")
        self.failIf(os.path.exists(dirty_dir),
                    "Untracked dir should have been removed")

    def test_clone_on_sha(self):
        """Git clone."""
        target_dir = os.path.join(self.dst_dir, "My clone")
        repo = GitRepo(target_dir, self.src_repo)
        repo('master')
        sha = repo.parents()[0]
        target_dir = os.path.join(self.dst_dir, "My clone 2")
        repo = GitRepo(target_dir, self.src_repo)
        repo(sha)
        sha2 = repo.parents()[0]
        self.assert_(sha == sha2, 'Bad clone on SHA')

    def test_uncommitted_changes(self):
        """GitRepo can detect uncommitted changes."""
        # initial cloning
        target_dir = os.path.join(self.dst_dir, "clone to update")
        repo = GitRepo(target_dir, self.src_repo)
        repo('master')

        self.assertFalse(repo.uncommitted_changes())

        f = open(os.path.join(target_dir, 'tracked'), 'w')
        f.write('mod')
        f.close()

        self.assertTrue(repo.uncommitted_changes())

class GitBranchTestCase(GitBaseTestCase):

    def create_src(self):
        GitBaseTestCase.create_src(self)
        os.chdir(self.src_repo)
        self.make_branch(self.src_repo, 'somebranch')

    def make_branch(self, src_dir, name):
        """create a branch
        """
        subprocess.check_call(['git', 'branch', name], cwd=src_dir)

    def test_switch_local_branch(self):
        """Switch to a branch created before the clone.

        In this case, the branch already exists in local repo
        """

        # Testing starts here
        target_dir = os.path.join(self.dst_dir, "to_branch")
        branch = GitRepo(target_dir, self.src_repo)
        #update file from master after branching
        branch("master")
        with working_directory_keeper:
            os.chdir(target_dir)
            f = open('tracked', 'w')
            f.write("last after branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])

        # check that no changes exists when switching from one to other
        branch('somebranch')
        self.assertFalse(branch.uncommitted_changes())
        branch('master')
        self.assertFalse(branch.uncommitted_changes())

        #modify the branch
        branch('somebranch')
        self.assertFalse(branch.uncommitted_changes())
        self.assertFalse(branch.uncommitted_changes())
        with working_directory_keeper:
            os.chdir(target_dir)
            subprocess.call(['git', 'config', 'user.email', COMMIT_USER_EMAIL])
            subprocess.call(['git', 'config', 'user.name', COMMIT_USER_NAME])
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), 'last')
            f = open('tracked', 'w')
            f.write("last from branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), 'last from branch')

        # switch to master
        branch('master')
        self.assertFalse(branch.uncommitted_changes())
        f = open(os.path.join(target_dir, 'tracked'))
        lines = f.readlines()
        f.close()
        self.assertEquals(lines[0].strip(), 'last after branch')
        self.assertFalse(branch.uncommitted_changes())

    def test_switch_remote_branch(self):
        """Switch to a branch created after the clone.

        In this case, the branch doesn't exist in local repo
        """
        # init the local clone
        # Testing starts here
        target_dir = os.path.join(self.dst_dir, "to_branch")
        branch = GitRepo(target_dir, self.src_repo)
        #update file from master after branching
        branch("master")

        # create the remote branch with some modifications
        self.make_branch(self.src_repo, 'remotebranch')
        with working_directory_keeper:
            os.chdir(self.src_repo)
            subprocess.call(['git', 'checkout', 'remotebranch'])
            f = open('tracked', 'w')
            f.write("last after remote branch" + os.linesep)
            f.close()
            subprocess.call(['git', 'add', 'tracked'])
            subprocess.call(['git', 'commit', '-m', 'last version'])

        # switch to the remote branch and check tracked file
        branch("remotebranch")

        with working_directory_keeper:
            os.chdir(target_dir)
            f = open('tracked')
            lines = f.readlines()
            f.close()
            self.assertEquals(lines[0].strip(), "last after remote branch")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__version__ = '0.9.5'
<commit_msg>bump version<commit_after># -*- coding: utf-8 -*-

__version__ = '0.9.6'
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-

"""
@version: 1.0
@author: kevin
@license: Apache Licence
@contact: liujiezhang@bupt.edu.cn
@site:
@software: PyCharm Community Edition
@file: detail.py
@time: 16/11/25 上午10:57
"""

from myfunc import *
from scrapy.http import HtmlResponse
from collections import defaultdict
import time
import re
import chardet

def goodsOutline(url):
    '''
    获取三级目录详情
    :param url:网站主页
    :return:[{'url':'','first_grade':'',...},...{}]
    '''
    # 存储各级分类信息
    outline_data = []
    # 解析页面
    body = getHtml(url)
    html = HtmlResponse(url=url,body=body)
    # 一级类目名
    first_grade = [u'建筑 矿用 防爆',u'泵 阀门']
    #预存一部分XPATH的基址
    basepath = ['//*[@id="newpro101"]/div[2]/ul','//*[@id="newpro401"]/div[2]/ul']
    #//*[@id="newpro4"]/h3/a 泵 阀门
    for i in xrange(len(first_grade)):
        # 二级类目名
        if i == 0:
            #建筑 矿用 防爆
            second_grade = html.selector.xpath(basepath[i]+u'/li/b/a/text()').extract()[1:9]
        else:
            second_grade = html.selector.xpath(basepath[i]+u'/li/b/a/text()').extract()
        #print u'二级目录个数' + str(len(second_grade))
        for j in xrange(len(second_grade)):
            # 三级类目名和链接url
            # //*[@id="newpro101"]/div[2]/ul/li[3]/a[1]
            #//*[@id="newpro401"]/div[2]/ul/li[6]/a[1]
            if i == 0:
                third_grade_name = html.selector.xpath(basepath[i]+'/li[{0}]/a/text()'.format(j+2)).extract()
                third_grade_url = html.selector.xpath(basepath[i]+'/li[{0}]/a/@href'.format(j+2)).extract()
            else:
                third_grade_name = html.selector.xpath(basepath[i]+'/li[{0}]/a/text()'.format(j+1)).extract()
                third_grade_url = html.selector.xpath(basepath[i]+'/li[{0}]/a/@href'.format(j+1)).extract()
            print('三级目录个数：'+str(len(third_grade_name)))
            for k in xrange(len(third_grade_name)):
                # 格式化数据
                url_data = defaultdict()
                url_data['url'] ='http://www.runlian365.com/' + third_grade_url[k]
                url_data['third_grade'] = third_grade_name[k]
                url_data['second_grade'] = second_grade[j]
                url_data['first_grade'] = first_grade[i]
                url_data['created'] = int(time.time())
                url_data['updated'] = int(time.time())

                #print  url_data['first_grade'] + ':' + url_data['second_grade']+ ':'+url_data['third_grade']
                # 保存
                outline_data.append(url_data)

    return outline_data

def goodsUrlList(home_url):
    '''
    根据三级目录商品首页获取所有详情页url
    :param home_url: http://www.vipmro.com/search/?&categoryId=501110
    :return:url列表
    '''
    # 网站基址
    base = u'http://www.runlian365.com'
    # 所有条件下的列表
    all_group_list = parse(home_url)
    # 保存所有goods的详情页url
    url_list = []
    for url in all_group_list:
        # url = 'http://www.vipmro.com/search/?ram=0.9551325197768372&categoryId=501110&attrValueIds=509805,509801,509806,509807'
        # 解析html
        home_page = getHtml(url)
        html = HtmlResponse(url=url,body=str(home_page))
        urls = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[1]/div/div/h4/a/@href').extract()
        url_list.extend(urls)
    for x in xrange(0,len(url_list)):
        url_list[x] = base + url_list[x]
        print str(x) + url_list[x]

    #     print(len(urls))
    #     print(urls)
    #     exit()
    # print(len(url_list))
    # print(url_list)
    return url_list

def goodsDetail(detail_url):
    '''
    利用xpath解析关键字段
    :param detail_url: 详情页url
    :return: 各个字段信息 dict
    '''
    goods_data = defaultdict()
    # 详情页链接
    goods_data['source_url'] = detail_url
    # 解析html body必须是str类型
    body = getHtmlFromJs(detail_url)['content'].encode('utf-8')
    #print body
    html = HtmlResponse(url=detail_url, body=str(body))
    # 名称
    goods_data['name'] = html.xpath('//*[@id="comTitle"]/text()').extract()[0]
    print goods_data['name']
    # 价格
    goods_data['price'] = html.selector.xpath('//*[@id="oriPriceTop"]/text()').re(ur'[1-9]\d*\.?\d*|0\.\d*[1-9]\d*')[0]
    print goods_data['price']
    # 型号
    goods_data['type'] = ''
    #goods_data['type'] = html.selector.xpath('//*[@id="pdetail"]/div[3]/table/tbody/tr/th[contains(text(),u"型号")]').extract()[0]
    #//*[@id="pdetail"]/div[3]/table/tbody
    #xpath('..').xpath('./td/text()')
    #print goods_data['type']

    # 详情
    goods_data['detail'] = html.selector.xpath('//*[@id="pdetail"]').extract()[0]
    print goods_data['detail']
    exit(0)
    # 图片
    goods_data['pics'] = html.selector.xpath('//*[@id="proSmallImg"]').xpath('@src').extract()[0].replace('../','http://www.runlian365.com/')

    goods_data['storage'] = html.selector.xpath('/html/body/div[9]/div[1]/div[2]/div[2]/span[2]/text()').re(ur'[1-9]\d*\.?\d*|0\.\d*[1-9]\d*')[0]
    goods_data['lack_period'] = html.selector.xpath('/html/body/div[9]/div[1]/div[2]/div[3]/text()').extract()[0]
    goods_data['created'] = int(time.time())
    goods_data['updated'] = int(time.time())

    # print(goods_data['detail'])
    return goods_data

def parse(url):
    '''
    解析url下页面各种选择项组合的url
    :param url: http://www.vipmro.com/search/?&categoryId=501110
    :return:['http://www.vipmro.com/search/?categoryId=501110&attrValueIds=509801,512680,509807,509823']
    '''
    # 解析html
    home_page = getHtml(url)
    html = HtmlResponse(url=url,body=str(home_page))
    #总页数
    page_total = 1
    #三级列表下所有
    url_list = []
    if html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[@class = "pages_next"]'):
        page_total = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()-1]/text()').extract()[0].replace('...','')
    elif html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()]/text()'):
        page_total = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()]/text()').extract()[0]
    for x in xrange(1,int(page_total)+1):
        url_list.append(url.replace('.html','-'+ str(x) + '.html'))
    return url_list

if __name__ == '__main__':
    # url = 'http://www.vipmro.com/product/587879'
    url = 'http://b2b.hc360.com/supplyself/511911171.html'
    goodsDetail(url)

<commit_msg>hc360详情<commit_after># -*- coding:utf-8 -*-

"""
@version: 1.0
@author: kevin
@license: Apache Licence
@contact: liujiezhang@bupt.edu.cn
@site:
@software: PyCharm Community Edition
@file: detail.py
@time: 16/11/25 上午10:57
"""

from myfunc import *
from scrapy.http import HtmlResponse
from collections import defaultdict
import time
import re
import io
import chardet

def goodsOutline(url):
    '''
    获取三级目录详情
    :param url:网站主页
    :return:[{'url':'','first_grade':'',...},...{}]
    '''
    # 存储各级分类信息
    outline_data = []
    # 解析页面
    body = getHtml(url)
    html = HtmlResponse(url=url,body=body)
    # 一级类目名
    first_grade = [u'建筑 矿用 防爆',u'泵 阀门']
    #预存一部分XPATH的基址
    basepath = ['//*[@id="newpro101"]/div[2]/ul','//*[@id="newpro401"]/div[2]/ul']
    #//*[@id="newpro4"]/h3/a 泵 阀门
    for i in xrange(len(first_grade)):
        # 二级类目名
        if i == 0:
            #建筑 矿用 防爆
            second_grade = html.selector.xpath(basepath[i]+u'/li/b/a/text()').extract()[1:9]
        else:
            second_grade = html.selector.xpath(basepath[i]+u'/li/b/a/text()').extract()
        #print u'二级目录个数' + str(len(second_grade))
        for j in xrange(len(second_grade)):
            # 三级类目名和链接url
            # //*[@id="newpro101"]/div[2]/ul/li[3]/a[1]
            #//*[@id="newpro401"]/div[2]/ul/li[6]/a[1]
            if i == 0:
                third_grade_name = html.selector.xpath(basepath[i]+'/li[{0}]/a/text()'.format(j+2)).extract()
                third_grade_url = html.selector.xpath(basepath[i]+'/li[{0}]/a/@href'.format(j+2)).extract()
            else:
                third_grade_name = html.selector.xpath(basepath[i]+'/li[{0}]/a/text()'.format(j+1)).extract()
                third_grade_url = html.selector.xpath(basepath[i]+'/li[{0}]/a/@href'.format(j+1)).extract()
            print('三级目录个数：'+str(len(third_grade_name)))
            for k in xrange(len(third_grade_name)):
                # 格式化数据
                url_data = defaultdict()
                url_data['url'] ='http://www.runlian365.com/' + third_grade_url[k]
                url_data['third_grade'] = third_grade_name[k]
                url_data['second_grade'] = second_grade[j]
                url_data['first_grade'] = first_grade[i]
                url_data['created'] = int(time.time())
                url_data['updated'] = int(time.time())

                #print  url_data['first_grade'] + ':' + url_data['second_grade']+ ':'+url_data['third_grade']
                # 保存
                outline_data.append(url_data)

    return outline_data

def goodsUrlList(home_url):
    '''
    根据三级目录商品首页获取所有详情页url
    :param home_url: http://www.vipmro.com/search/?&categoryId=501110
    :return:url列表
    '''
    # 网站基址
    base = u'http://www.runlian365.com'
    # 所有条件下的列表
    all_group_list = parse(home_url)
    # 保存所有goods的详情页url
    url_list = []
    for url in all_group_list:
        # url = 'http://www.vipmro.com/search/?ram=0.9551325197768372&categoryId=501110&attrValueIds=509805,509801,509806,509807'
        # 解析html
        home_page = getHtml(url)
        html = HtmlResponse(url=url,body=str(home_page))
        urls = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[1]/div/div/h4/a/@href').extract()
        url_list.extend(urls)
    for x in xrange(0,len(url_list)):
        url_list[x] = base + url_list[x]
        print str(x) + url_list[x]

    #     print(len(urls))
    #     print(urls)
    #     exit()
    # print(len(url_list))
    # print(url_list)
    return url_list

def goodsDetail(detail_url):
    '''
    利用xpath解析关键字段
    :param detail_url: 详情页url
    :return: 各个字段信息 dict
    '''
    goods_data = defaultdict()
    # 详情页链接
    goods_data['source_url'] = detail_url
    # 解析html body必须是str类型
    body = getHtmlFromJs(detail_url)['content'].encode('utf-8')
    html = HtmlResponse(url=detail_url, body=str(body))
    # 名称
    goods_data['name'] = html.xpath('//*[@id="comTitle"]/text()').extract()[0]
    print goods_data['name']
    # 价格
    goods_data['price'] = html.selector.xpath('//*[@id="oriPriceTop"]/text()').re(ur'[1-9]\d*\.?\d*|0\.\d*[1-9]\d*')[0]
    print goods_data['price']
    # 型号
    goods_data['type'] = ''
    # 详情
    detaildiv = html.selector.xpath('//*[@id="pdetail"]')
    detail_table = '\n'.join(detaildiv.xpath('//table').extract())
    detail_p = ''.join(detaildiv.xpath('//p').extract())
    goods_data['detail'] = detail_table + detail_p
    print goods_data['detail']
    goods_data['detail'] = re.sub(ur'^<img.*>$', '', goods_data['detail'])
    f = open('detail.txt', 'w')
    f.write(goods_data['name'].encode('utf-8'))
    f.write(goods_data['detail'].encode('utf-8'))
    f.close()
    # 图片
    pics = []
    for pic in html.selector.xpath('//*[@id="thumblist"]/li/div/a/img/@src').extract():
        pics.append(pic.replace('100x100', '300x300'))
    goods_data['pics'] = ('|').join(pics)
    print goods_data['pics']
    #库存
    goods_data['storage'] = html.selector.xpath('//*[@id="supplyInfoNum"]/text()').re(ur'[1-9]\d*\.?\d*|0\.\d*[1-9]\d*')[0]
    print goods_data['storage']
    #供货时间
    goods_data['lack_period'] = ''
    goods_data['created'] = int(time.time())
    goods_data['updated'] = int(time.time())

    # print(goods_data['detail'])
    return goods_data

def parse(url):
    '''
    解析url下页面各种选择项组合的url
    :param url: http://www.vipmro.com/search/?&categoryId=501110
    :return:['http://www.vipmro.com/search/?categoryId=501110&attrValueIds=509801,512680,509807,509823']
    '''
    # 解析html
    home_page = getHtml(url)
    html = HtmlResponse(url=url,body=str(home_page))
    #总页数
    page_total = 1
    #三级列表下所有
    url_list = []
    if html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[@class = "pages_next"]'):
        page_total = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()-1]/text()').extract()[0].replace('...','')
    elif html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()]/text()'):
        page_total = html.selector.xpath('/html/body/div[5]/div/div[2]/div/div[2]/div/a[last()]/text()').extract()[0]
    for x in xrange(1,int(page_total)+1):
        url_list.append(url.replace('.html','-'+ str(x) + '.html'))
    return url_list

if __name__ == '__main__':
    # url = 'http://www.vipmro.com/product/587879'
    url = 'http://b2b.hc360.com/supplyself/605891383.html'
    goodsDetail(url)

<|endoftext|>


<commit_before>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.server.twisted`` module contains a server transport compatible
with the Twisted event loop. It uses the TwistedWebResource object as transport.

Also see the twisted examples in the examples directory of the source
distribution.

If you want to have a hard-coded URL in the wsdl document, this is how to do
it: ::

    resource = TwistedWebResource(...)
    resource.http_transport.doc.wsdl11.build_interface_document("http://example.com")

This is not strictly necessary -- if you don't do this, Spyne will get the
URL from the first request, build the wsdl on-the-fly and cache it as a
string in memory for later requests. However, if you want to make sure
you only have this url on the WSDL, this is how to do it. Note that if
your client takes the information in wsdl seriously, all requests will go
to the designated url above which can make testing a bit difficult. Use
in moderation.

This module is EXPERIMENTAL. Your mileage may vary. Patches are welcome.
"""

from __future__ import absolute_import

import logging
logger = logging.getLogger(__name__)

from inspect import isgenerator

from twisted.python.log import err
from twisted.internet.interfaces import IPullProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.web.iweb import UNKNOWN_LENGTH
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET

from zope.interface import implements

from spyne import MethodContext
from spyne import TransportContext
from spyne.auxproc import process_contexts
from spyne.const.ansi_color import LIGHT_GREEN
from spyne.const.ansi_color import END_COLOR
from spyne.const.http import HTTP_404
from spyne.model import PushBase
from spyne.model.fault import Fault
from spyne.server import ServerBase
from spyne.server.http import HttpBase
from spyne.server.http import HttpMethodContext

def _reconstruct_url(request):
    server_name = request.getRequestHostname()
    server_port = request.getHost().port
    if (bool(request.isSecure()), server_port) not in [(True, 443), (False, 80)]:
        server_name = '%s:%d' % (server_name, server_port)

    if request.isSecure():
        url_scheme = 'https'
    else:
        url_scheme = 'http'

    return ''.join([url_scheme, "://", server_name, request.uri])

class _Producer(object):
    implements(IPullProducer)

    deferred = None

    def __init__(self, body, consumer):
        """:param body: an iterable of strings"""

        # check to see if we can determine the length
        try:
            len(body) # iterator?
            self.length = sum([len(fragment) for fragment in body])
            self.body = iter(body)

        except TypeError:
            self.length = UNKNOWN_LENGTH
            self.body = body

        self.deferred = Deferred()

        self.consumer = consumer

    def resumeProducing(self):
        try:
            chunk = self.body.next()

        except StopIteration, e:
            self.consumer.unregisterProducer()
            if self.deferred is not None:
                self.deferred.callback(self.consumer)
                self.deferred = None
            return

        self.consumer.write(chunk)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        if self.deferred is not None:
            self.deferred.errback(
                               Exception("Consumer asked us to stop producing"))
        self.deferred = None

class TwistedHttpTransport(HttpBase):
    @staticmethod
    def decompose_incoming_envelope(prot, ctx, message):
        """This function is only called by the HttpRpc protocol to have the
        twisted web's Request object is parsed into ``ctx.in_body_doc`` and
        ``ctx.in_header_doc``.
        """

        request = ctx.in_document

        ctx.method_request_string = '{%s}%s' % (prot.app.interface.get_tns(),
                                                    request.path.split('/')[-1])

        logger.debug("%sMethod name: %r%s" % (LIGHT_GREEN,
                                          ctx.method_request_string, END_COLOR))

        ctx.in_header_doc = request.headers
        ctx.in_body_doc = request.args

class TwistedWebResource(Resource):
    """A server transport that exposes the application as a twisted web
    Resource.
    """

    isLeaf = True

    def __init__(self, app, chunked=False, max_content_length=2 * 1024 * 1024,
                                           block_length=8 * 1024):
        Resource.__init__(self)

        self.http_transport = TwistedHttpTransport(app, chunked,
                                            max_content_length, block_length)
        self._wsdl = None

    def render_GET(self, request):
        if request.uri.endswith('.wsdl') or request.uri.endswith('?wsdl'):
            return self.__handle_wsdl_request(request)

        else:
            return self.handle_rpc(request)

    def render_POST(self, request):
        return self.handle_rpc(request)

    def handle_rpc_error(self, p_ctx, others, error, request):
        resp_code = p_ctx.out_protocol.fault_to_http_response_code(error)

        request.setResponseCode(int(resp_code[:3]))

        p_ctx.out_object = error
        self.http_transport.get_out_string(p_ctx)

        process_contexts(self.http_transport, others, p_ctx, error=error)

        retval = ''.join(p_ctx.out_string)
        p_ctx.close()

        return retval

    def handle_rpc(self, request):
        initial_ctx = HttpMethodContext(self.http_transport, request,
                                 self.http_transport.app.out_protocol.mime_type)
        initial_ctx.in_string = [request.content.getvalue()]

        contexts = self.http_transport.generate_contexts(initial_ctx)
        p_ctx, others = contexts[0], contexts[1:]

        if p_ctx.in_error:
            return self.handle_rpc_error(p_ctx, others, p_ctx.in_error, request)

        else:
            self.http_transport.get_in_object(p_ctx)

            if p_ctx.in_error:
                return self.handle_rpc_error(p_ctx, others, p_ctx.in_error, request)

            else:
                self.http_transport.get_out_object(p_ctx)
                if p_ctx.out_error:
                    return self.handle_rpc_error(p_ctx, others, p_ctx.out_error,
                                                                        request)

        def _cb_request_finished(request):
            request.finish()
            p_ctx.close()

        def _eb_request_finished(request):
            err(request)
            p_ctx.close()
            request.finish()

        def _cb_deferred(retval, request, cb=True):
            if cb and len(p_ctx.descriptor.out_message._type_info) <= 1:
                p_ctx.out_object = [retval]
            else:
                p_ctx.out_object = retval

            self.http_transport.get_out_string(p_ctx)

            process_contexts(self.http_transport, others, p_ctx)

            producer = _Producer(p_ctx.out_string, request)
            producer.deferred.addCallbacks(_cb_request_finished,
                                                           _eb_request_finished)
            request.registerProducer(producer, False)

        def _eb_deferred(retval, request):
            p_ctx.out_error = retval.value
            if not issubclass(retval.type, Fault):
                retval.printTraceback()

            ret = self.handle_rpc_error(p_ctx, others, p_ctx.out_error, request)
            request.write(ret)
            request.finish()

        ret = p_ctx.out_object[0]
        if isinstance(ret, Deferred):
            ret.addCallback(_cb_deferred, request)
            ret.addErrback(_eb_deferred, request)

        elif isinstance(ret, PushBase):
            gen = self.http_transport.get_out_string(p_ctx)

            assert isgenerator(gen), "It looks like this protocol is not " \
                                     "async-compliant yet."

            def _cb_push():
                process_contexts(self.http_transport, others, p_ctx)

                producer = _Producer(p_ctx.out_string, request)
                producer.deferred.addCallbacks(_cb_request_finished,
                                                           _eb_request_finished)
                request.registerProducer(producer, False)

            ret.init(p_ctx, request, gen, _cb_push, None)

        else:
            _cb_deferred(p_ctx.out_object, request, cb=False)

        return NOT_DONE_YET

    def __handle_wsdl_request(self, request):
        ctx = HttpMethodContext(self.http_transport, request,
                                                      "text/xml; charset=utf-8")
        url = _reconstruct_url(request)

        if self.doc.wsdl11 is None:
            return HTTP_404

        if self._wsdl is None:
            self._wsdl = self.http_transport.doc.wsdl11.get_interface_document()

        ctx.transport.wsdl = self._wsdl

        try:
            if self._wsdl is None:
                self.http_transport.doc.wsdl11.build_interface_document(url)
                ctx.transport.wsdl = self._wsdl = \
                         self.http_transport.doc.wsdl11.get_interface_document()

            assert ctx.transport.wsdl is not None

            self.http_transport.event_manager.fire_event('wsdl', ctx)

            return ctx.transport.wsdl

        except Exception, e:
            ctx.transport.wsdl_error = e
            self.http_transport.event_manager.fire_event('wsdl_exception', ctx)
            raise

        finally:
            ctx.close()

try:
    from twisted.web.websockets import WebSocketsProtocol
    from twisted.web.websockets import WebSocketsResource
except ImportError:
    from spyne.util._twisted_ws import WebSocketsProtocol
    from spyne.util._twisted_ws import WebSocketsResource

class WebSocketTransportContext(TransportContext):
    def __init__(self, transport, type=None, client_handle=None):
        TransportContext.__init__(self, transport, type)

        self.client_handle = client_handle

class WebSocketMethodContext(MethodContext):
    def __init__(self, transport, client_handle):
        MethodContext.__init__(self, transport)

        self.transport = WebSocketTransportContext(transport,
                                                        'ws', client_handle)

class TwistedWebSocketProtocol(WebSocketsProtocol):
    def __init__(self, transport):
        self._spyne_transport = transport

    def frameReceived(self, opcode, data, fin):
        tpt = self._spyne_transport

        initial_ctx = WebSocketMethodContext(tpt, client_handle=self)
        initial_ctx.in_string = [data]

        contexts = tpt.generate_contexts(initial_ctx)
        p_ctx, others = contexts[0], contexts[1:]

        if p_ctx.in_error:
            p_ctx.out_object = p_ctx.in_error

        else:
            tpt.get_in_object(p_ctx)

            if p_ctx.in_error:
                p_ctx.out_object = p_ctx.in_error

            else:
                tpt.get_out_object(p_ctx)
                if p_ctx.out_error:
                    p_ctx.out_object = p_ctx.out_error

        def _cb_deferred(retval, cb=True):
            if cb and len(p_ctx.descriptor.out_message._type_info) <= 1:
                p_ctx.out_object = [retval]
            else:
                p_ctx.out_object = retval

            tpt.get_out_string(p_ctx)
            self.sendFrame(opcode, ''.join(p_ctx.out_string), fin)
            p_ctx.close()
            process_contexts(tpt, others, p_ctx)

        def _eb_deferred(retval):
            p_ctx.out_error = retval.value
            if not issubclass(retval.type, Fault):
                retval.printTraceback()

            tpt.get_out_string(p_ctx)
            self.sendFrame(opcode, ''.join(p_ctx.out_string), fin)
            p_ctx.close()

        ret = p_ctx.out_object[0]
        if isinstance(ret, Deferred):
            ret.addCallback(_cb_deferred)
            ret.addErrback(_eb_deferred)

        elif isinstance(ret, PushBase):
            raise NotImplementedError()

        else:
            _cb_deferred(p_ctx.out_object, cb=False)

    # FIXME: Something must happen here :)
    def connectionLost(self, reason):
        pass

class TwistedWebSocketFactory(Factory):
    def __init__(self, app):
        self.app = app
        self.transport = ServerBase(app)

    def buildProtocol(self, addr):
        return TwistedWebSocketProtocol(self.transport)

class TwistedWebSocketResource(WebSocketsResource):
    def __init__(self, app):
        WebSocketsResource.__init__(self, TwistedWebSocketFactory(app))
<commit_msg>add message propagation support to websockets<commit_after>
#
# spyne - Copyright (C) Spyne contributors.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301
#

"""The ``spyne.server.twisted`` module contains a server transport compatible
with the Twisted event loop. It uses the TwistedWebResource object as transport.

Also see the twisted examples in the examples directory of the source
distribution.

If you want to have a hard-coded URL in the wsdl document, this is how to do
it: ::

    resource = TwistedWebResource(...)
    resource.http_transport.doc.wsdl11.build_interface_document("http://example.com")

This is not strictly necessary -- if you don't do this, Spyne will get the
URL from the first request, build the wsdl on-the-fly and cache it as a
string in memory for later requests. However, if you want to make sure
you only have this url on the WSDL, this is how to do it. Note that if
your client takes the information in wsdl seriously, all requests will go
to the designated url above which can make testing a bit difficult. Use
in moderation.

This module is EXPERIMENTAL. Your mileage may vary. Patches are welcome.
"""

from __future__ import absolute_import

import logging
logger = logging.getLogger(__name__)

from inspect import isgenerator

from twisted.python.log import err
from twisted.internet.interfaces import IPullProducer
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.web.iweb import UNKNOWN_LENGTH
from twisted.web.resource import Resource
from twisted.web.server import NOT_DONE_YET
from twisted.python import log

from zope.interface import implements

from spyne import MethodContext
from spyne import TransportContext
from spyne.auxproc import process_contexts
from spyne.const.ansi_color import LIGHT_GREEN
from spyne.const.ansi_color import END_COLOR
from spyne.const.http import HTTP_404
from spyne.model import PushBase
from spyne.model.complex import ComplexModel
from spyne.model.fault import Fault
from spyne.server import ServerBase
from spyne.server.http import HttpBase
from spyne.server.http import HttpMethodContext

def _reconstruct_url(request):
    server_name = request.getRequestHostname()
    server_port = request.getHost().port
    if (bool(request.isSecure()), server_port) not in [(True, 443), (False, 80)]:
        server_name = '%s:%d' % (server_name, server_port)

    if request.isSecure():
        url_scheme = 'https'
    else:
        url_scheme = 'http'

    return ''.join([url_scheme, "://", server_name, request.uri])

class _Producer(object):
    implements(IPullProducer)

    deferred = None

    def __init__(self, body, consumer):
        """:param body: an iterable of strings"""

        # check to see if we can determine the length
        try:
            len(body) # iterator?
            self.length = sum([len(fragment) for fragment in body])
            self.body = iter(body)

        except TypeError:
            self.length = UNKNOWN_LENGTH
            self.body = body

        self.deferred = Deferred()

        self.consumer = consumer

    def resumeProducing(self):
        try:
            chunk = self.body.next()

        except StopIteration, e:
            self.consumer.unregisterProducer()
            if self.deferred is not None:
                self.deferred.callback(self.consumer)
                self.deferred = None
            return

        self.consumer.write(chunk)

    def pauseProducing(self):
        pass

    def stopProducing(self):
        if self.deferred is not None:
            self.deferred.errback(
                               Exception("Consumer asked us to stop producing"))
        self.deferred = None

class TwistedHttpTransport(HttpBase):
    @staticmethod
    def decompose_incoming_envelope(prot, ctx, message):
        """This function is only called by the HttpRpc protocol to have the
        twisted web's Request object is parsed into ``ctx.in_body_doc`` and
        ``ctx.in_header_doc``.
        """

        request = ctx.in_document

        ctx.method_request_string = '{%s}%s' % (prot.app.interface.get_tns(),
                                                    request.path.split('/')[-1])

        logger.debug("%sMethod name: %r%s" % (LIGHT_GREEN,
                                          ctx.method_request_string, END_COLOR))

        ctx.in_header_doc = request.headers
        ctx.in_body_doc = request.args

class TwistedWebResource(Resource):
    """A server transport that exposes the application as a twisted web
    Resource.
    """

    isLeaf = True

    def __init__(self, app, chunked=False, max_content_length=2 * 1024 * 1024,
                                           block_length=8 * 1024):
        Resource.__init__(self)

        self.http_transport = TwistedHttpTransport(app, chunked,
                                            max_content_length, block_length)
        self._wsdl = None

    def render_GET(self, request):
        if request.uri.endswith('.wsdl') or request.uri.endswith('?wsdl'):
            return self.__handle_wsdl_request(request)

        else:
            return self.handle_rpc(request)

    def render_POST(self, request):
        return self.handle_rpc(request)

    def handle_rpc_error(self, p_ctx, others, error, request):
        resp_code = p_ctx.out_protocol.fault_to_http_response_code(error)

        request.setResponseCode(int(resp_code[:3]))

        p_ctx.out_object = error
        self.http_transport.get_out_string(p_ctx)

        process_contexts(self.http_transport, others, p_ctx, error=error)

        retval = ''.join(p_ctx.out_string)
        p_ctx.close()

        return retval

    def handle_rpc(self, request):
        initial_ctx = HttpMethodContext(self.http_transport, request,
                                 self.http_transport.app.out_protocol.mime_type)
        initial_ctx.in_string = [request.content.getvalue()]

        contexts = self.http_transport.generate_contexts(initial_ctx)
        p_ctx, others = contexts[0], contexts[1:]

        if p_ctx.in_error:
            return self.handle_rpc_error(p_ctx, others, p_ctx.in_error, request)

        else:
            self.http_transport.get_in_object(p_ctx)

            if p_ctx.in_error:
                return self.handle_rpc_error(p_ctx, others, p_ctx.in_error, request)

            else:
                self.http_transport.get_out_object(p_ctx)
                if p_ctx.out_error:
                    return self.handle_rpc_error(p_ctx, others, p_ctx.out_error,
                                                                        request)

        def _cb_request_finished(request):
            request.finish()
            p_ctx.close()

        def _eb_request_finished(request):
            err(request)
            p_ctx.close()
            request.finish()

        def _cb_deferred(retval, request, cb=True):
            if cb and len(p_ctx.descriptor.out_message._type_info) <= 1:
                p_ctx.out_object = [retval]
            else:
                p_ctx.out_object = retval

            self.http_transport.get_out_string(p_ctx)

            process_contexts(self.http_transport, others, p_ctx)

            producer = _Producer(p_ctx.out_string, request)
            producer.deferred.addCallbacks(_cb_request_finished,
                                                           _eb_request_finished)
            request.registerProducer(producer, False)

        def _eb_deferred(retval, request):
            p_ctx.out_error = retval.value
            if not issubclass(retval.type, Fault):
                retval.printTraceback()

            ret = self.handle_rpc_error(p_ctx, others, p_ctx.out_error, request)
            request.write(ret)
            request.finish()

        ret = p_ctx.out_object[0]
        if isinstance(ret, Deferred):
            ret.addCallback(_cb_deferred, request)
            ret.addErrback(_eb_deferred, request)

        elif isinstance(ret, PushBase):
            gen = self.http_transport.get_out_string(p_ctx)

            assert isgenerator(gen), "It looks like this protocol is not " \
                                     "async-compliant yet."

            def _cb_push():
                process_contexts(self.http_transport, others, p_ctx)

                producer = _Producer(p_ctx.out_string, request)
                producer.deferred.addCallbacks(_cb_request_finished,
                                                           _eb_request_finished)
                request.registerProducer(producer, False)

            ret.init(p_ctx, request, gen, _cb_push, None)

        else:
            _cb_deferred(p_ctx.out_object, request, cb=False)

        return NOT_DONE_YET

    def __handle_wsdl_request(self, request):
        ctx = HttpMethodContext(self.http_transport, request,
                                                      "text/xml; charset=utf-8")
        url = _reconstruct_url(request)

        if self.doc.wsdl11 is None:
            return HTTP_404

        if self._wsdl is None:
            self._wsdl = self.http_transport.doc.wsdl11.get_interface_document()

        ctx.transport.wsdl = self._wsdl

        try:
            if self._wsdl is None:
                self.http_transport.doc.wsdl11.build_interface_document(url)
                ctx.transport.wsdl = self._wsdl = \
                         self.http_transport.doc.wsdl11.get_interface_document()

            assert ctx.transport.wsdl is not None

            self.http_transport.event_manager.fire_event('wsdl', ctx)

            return ctx.transport.wsdl

        except Exception, e:
            ctx.transport.wsdl_error = e
            self.http_transport.event_manager.fire_event('wsdl_exception', ctx)
            raise

        finally:
            ctx.close()

try:
    from twisted.web.websockets import WebSocketsProtocol
    from twisted.web.websockets import WebSocketsResource
    from twisted.web.websockets import CONTROLS

except ImportError:
    from spyne.util._twisted_ws import WebSocketsProtocol
    from spyne.util._twisted_ws import WebSocketsResource
    from spyne.util._twisted_ws import CONTROLS

class WebSocketTransportContext(TransportContext):
    def __init__(self, transport, type, client_handle, parent):
        TransportContext.__init__(self, transport, type)

        self.client_handle = client_handle
        self.parent = parent

class WebSocketMethodContext(MethodContext):
    def __init__(self, transport, client_handle):
        MethodContext.__init__(self, transport)

        self.transport = WebSocketTransportContext(transport, 'ws',
                                                            client_handle, self)

class TwistedWebSocketProtocol(WebSocketsProtocol):
    def __init__(self, transport, bookkeep=False, _clients=None):
        self._spyne_transport = transport
        self._clients = _clients
        if bookkeep:
            self.connectionMade = self._connectionMade
            self.connectionLost = self._connectionLost
        if _clients is None:
            self._clients = {}

    def _connectionMade(self):
        WebSocketsProtocol.connectionMade(self)

        self._clients[id(self)] = self

    def _connectionLost(self, reason):
        del self._clients[id(self)]

    def frameReceived(self, opcode, data, fin):
        tpt = self._spyne_transport

        initial_ctx = WebSocketMethodContext(tpt, client_handle=self)
        initial_ctx.in_string = [data]

        contexts = tpt.generate_contexts(initial_ctx)
        p_ctx, others = contexts[0], contexts[1:]

        if p_ctx.in_error:
            p_ctx.out_object = p_ctx.in_error

        else:
            tpt.get_in_object(p_ctx)

            if p_ctx.in_error:
                p_ctx.out_object = p_ctx.in_error

            else:
                tpt.get_out_object(p_ctx)
                if p_ctx.out_error:
                    p_ctx.out_object = p_ctx.out_error

        def _cb_deferred(retval, cb=True):
            if cb and len(p_ctx.descriptor.out_message._type_info) <= 1:
                p_ctx.out_object = [retval]
            else:
                p_ctx.out_object = retval

            tpt.get_out_string(p_ctx)
            self.sendFrame(opcode, ''.join(p_ctx.out_string), fin)
            p_ctx.close()
            process_contexts(tpt, others, p_ctx)

        def _eb_deferred(retval):
            p_ctx.out_error = retval.value
            if not issubclass(retval.type, Fault):
                retval.printTraceback()

            tpt.get_out_string(p_ctx)
            self.sendFrame(opcode, ''.join(p_ctx.out_string), fin)
            p_ctx.close()

        ret = p_ctx.out_object[0]
        if isinstance(ret, Deferred):
            ret.addCallback(_cb_deferred)
            ret.addErrback(_eb_deferred)

        elif isinstance(ret, PushBase):
            raise NotImplementedError()

        else:
            _cb_deferred(p_ctx.out_object, cb=False)

class TwistedWebSocketFactory(Factory):
    def __init__(self, app, bookkeep=False, _clients=None):
        self.app = app
        self.transport = ServerBase(app)
        self.bookkeep = bookkeep
        self._clients = _clients
        if _clients is None:
            self._clients = {}

    def buildProtocol(self, addr):
        return TwistedWebSocketProtocol(self.transport, self.bookkeep,
                                                                self._clients)

class _Fake(object):
    pass

def _FakeWrap(cls):
    class _Ret(ComplexModel):
        _type_info = {"ugh ": cls}

    return _Ret

class _FakeCtx(object):
    def __init__(self, obj, cls):
        self.out_object = obj
        self.out_error = None
        self.descriptor = _Fake()
        self.descriptor.out_message = cls

class InvalidRequestError(Exception):
    pass

class TwistedWebSocketResource(WebSocketsResource):
    def __init__(self, app, bookkeep=False):
        self.app = app
        self._clients = {}
        if bookkeep:
            self.propagate = self._propagate

        WebSocketsResource.__init__(self, TwistedWebSocketFactory(app,
                                                       bookkeep, self._clients))

    def propagate(self):
        raise InvalidRequestError("You must enable bookkeeping to have "
                                  "message propagation work.")

    def _propagate(self, obj, cls=None):
        if cls is None:
            cls = obj.__class__

        op = self.app.out_protocol
        ctx = _FakeCtx(obj, cls)
        op.serialize(ctx, op.RESPONSE)
        op.create_out_string(ctx)
        doc = ''.join(ctx.out_string)

        for c in self._clients.itervalues():
            c.sendFrame(CONTROLS.TEXT, doc, True)
<|endoftext|>


<commit_before>class History:
    def __init__(self, framerate):

        self.Handlist = []
        self.framerate = framerate
        self.i = 0
        while self.i <= 2*framerate:
            self.Handlist.append([None, None, None, None, None])
            self.i += 1

    def add_information(self, position_of_hand_x, position_of_hand_y, time, number_of_fingers, operator):

        self.Handlist[0:0] = [[position_of_hand_x, position_of_hand_y, time, number_of_fingers, operator]]
        del self.Handlist[2*self.framerate+1]
        #print(self.Handlist)

    def get_position_of_hand_x(self, index):
        return self.Handlist[index][0]

    def get_position_of_hand_y(self, index):
        return self.Handlist[index][1]

    def get_time(self, index):
        return self.Handlist[index][2]

    def get_number_of_fingers(self, index):
        return self.Handlist[index][3]

    def get_operator(self, index):
        return self.Handlist[index][4]

    def confirm_fingers(self):
        if self.get_number_of_fingers(2*self.framerate - 1) == self.get_number_of_fingers(0):
            return self.get_number_of_fingers(0)
        else:
            return None

test = History(3)
test.add_information(61, 5, 1, 8, None)
test.add_information(54, 6, 2, 8, "+")
test.add_information(1, 7, 3, 3, "-")
test.add_information(61, 5, 1, 8, None)
test.add_information(54, 6, 2, 8, "+")
test.add_information(61, 5, 1, 8, None)

print(test.Handlist)
print(test.confirm_fingers())
if __name__ == "__main__":
    print("Please run Main.")
<commit_msg>confirm_newest_finger<commit_after>import datetime

class History:
    def __init__(self):
        self.Handlist = []

    def add_information(self, position_of_hand_x, position_of_hand_y, time, number_of_fingers, operator):

        self.Handlist.insert(0,[position_of_hand_x, position_of_hand_y, time, number_of_fingers, operator])

    def get_position_of_hand_x(self, index):
        return self.Handlist[index][0]

    def get_position_of_hand_y(self, index):
        return self.Handlist[index][1]

    def get_time(self, index):
        return self.Handlist[index][2]

    def get_number_of_fingers(self, index):
        return self.Handlist[index][3]

    def get_operator(self, index):
        return self.Handlist[index][4]

    def confirm_fingers(self):
        return self.confirm_newest_finger()

    def confirm_newest_finger(self):
        self.get_number_of_fingers(0)
        time_now = self.get_time(0)
        time_old = time_now
        i = 1
        while self.get_number_of_fingers(i) == self.get_number_of_fingers(0):
            time_old = self.get_time(i)
            i += 1
        time_difference = time_now - time_old
        return time_difference > datetime.timedelta(0, 2, 0, 0, 0, 0, 0)

test = History()
test.add_information(61, 5, datetime.datetime(2017, 4, 6, 12, 45, 10), 6, None)
test.add_information(54, 6, datetime.datetime(2017, 4, 6, 12, 45, 11), 6, "+")
test.add_information(1, 7, datetime.datetime(2017, 4, 6, 12, 45, 12), 6, "-")
test.add_information(61, 5, datetime.datetime(2017, 4, 6, 12, 45, 13), 5, None)
test.add_information(54, 6, datetime.datetime(2017, 4, 6, 12, 45, 14), 8, "+")
test.add_information(61, 5, datetime.datetime(2017, 4, 6, 12, 45, 19), 5, None)
test.add_information(61, 5, datetime.datetime(2017, 4, 6, 12, 45, 20), 8, None)

print(test.Handlist)
print(test.confirm_fingers())
#if __name__ == "__main__":
    #print("Please run Main.")
<|endoftext|>


<commit_before>'''
This file contains the lookup logic for ajax-based editor search widgets.
'''

from selectable.base import ModelLookup
from selectable.registry import registry
from metashare.repository.models import personInfoType_model, \
    actorInfoType_model, documentInfoType_model, documentationInfoType_model,\
    projectInfoType_model, organizationInfoType_model,\
    membershipInfoType_model, \
    targetResourceInfoType_model, languageVarietyInfoType_model, \
    sizeInfoType_model, annotationInfoType_model, videoFormatInfoType_model, \
    imageFormatInfoType_model, resolutionInfoType_model, \
    audioSizeInfoType_model

class PersonLookup(ModelLookup):
    model = personInfoType_model

    def get_query(self, request, term):
        #results = super(PersonLookup, self).get_query(request, term)
        # Since MultiTextFields cannot be searched using query sets (they are base64-encoded and pickled),
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(person):
            'Helper function to group the search code for a person'
            for multifield in (person.surname, person.givenName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False
        persons = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = persons
        else:
            results = [p for p in persons if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class GenericUnicodeLookup(ModelLookup):
    '''
    A reusable base class for lookups that have to be carried out by hand
    (rather than using querysets to do an efficient database search),
    doing string matching on the unicode string representing a database item.
    '''
    def get_query(self, request, term):
        # Since Subclassables and some other classes cannot be searched using query sets,
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(item):
            if (isinstance(ModelLookup, actorInfoType_model) and item.as_subclass().copy_status != "m"):
                return False
            'Helper function to group the search code for a database item'
            return lcterm in unicode(item).lower()

        items = ''
        if isinstance(ModelLookup, documentInfoType_model):
            items = self.get_queryset().filter(copy_status="m")
        else:
            items = self.get_queryset()
        if term == '*':
            results = items
        else:
            results = [p for p in items if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class ActorLookup(GenericUnicodeLookup):
    model = actorInfoType_model

class DocumentationLookup(ModelLookup):
    '''
    A special lookup which can represent values of both
    the (structured) documentInfo type and the documentUnstructured text-only type,
    but which performes lookup only on the structured entries.
    '''
    model = documentationInfoType_model

    def get_query(self, request, term):
        # Since Subclassables and some other classes cannot be searched using query sets,
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(item):
            if item.as_subclass().copy_status != "m":
                return False
            'Helper function to group the search code for a database item'
            return lcterm in unicode(item).lower()
        items = documentInfoType_model.objects.get_query_set()
        if term == '*':
            results = items
        else:
            results = [p for p in items if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class MembershipDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = membershipInfoType_model

class AnnotationLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = annotationInfoType_model

class VideoFormatLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = videoFormatInfoType_model

class ImageFormatLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = imageFormatInfoType_model

class ResolutionDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = resolutionInfoType_model

class LanguageVarietyDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = languageVarietyInfoType_model

class SizeDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = sizeInfoType_model

class AudioSizeDummyLookup(ModelLookup):
    model = audioSizeInfoType_model

class ProjectLookup(ModelLookup):
    model = projectInfoType_model
    search_fields = ('projectName__contains', )
    filters = {}

    def get_query(self, request, term):
        lcterm = term.lower()

        def matches(project):
            'Helper function to group the search code for a project'
            for multifield in (project.projectName, project.projectShortName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False

        projects = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = projects
        else:
            results = [p for p in projects if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

    def get_item_label(self, item):
        short_names = ''.join(item.projectShortName.itervalues())
        names = ''.join(item.projectName.itervalues())
        res = u'%s: %s' % (short_names, names)
        return res

    def get_item_id(self, item):
        return item.id

class OrganizationLookup(ModelLookup):
    model = organizationInfoType_model

    def get_query(self, request, term):
        # Since MultiTextFields cannot be searched using query sets (they are base64-encoded and pickled),
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(org):
            'Helper function to group the search code for a person'
            for multifield in (org.organizationShortName, org.organizationName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False
        orgs = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = orgs
        else:
            results = [p for p in orgs if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

    def get_item_label(self, item):
        short_names = ''.join(item.organizationShortName.itervalues())
        names = ''.join(item.organizationName.itervalues())
        res = u'%s: %s' % (short_names, names)
        return res

class DocumentLookup(GenericUnicodeLookup):
    model = documentInfoType_model

class TargetResourceLookup(GenericUnicodeLookup):
    model = targetResourceInfoType_model

registry.register(PersonLookup)
registry.register(ActorLookup)
registry.register(DocumentationLookup)
registry.register(DocumentLookup)
registry.register(ProjectLookup)
registry.register(OrganizationLookup)
registry.register(TargetResourceLookup)

<commit_msg>fixed pylint violations<commit_after>'''
This file contains the lookup logic for ajax-based editor search widgets.
'''

from selectable.base import ModelLookup
from selectable.registry import registry
from metashare.repository.models import personInfoType_model, \
    actorInfoType_model, documentInfoType_model, documentationInfoType_model,\
    projectInfoType_model, organizationInfoType_model,\
    membershipInfoType_model, \
    targetResourceInfoType_model, languageVarietyInfoType_model, \
    sizeInfoType_model, annotationInfoType_model, videoFormatInfoType_model, \
    imageFormatInfoType_model, resolutionInfoType_model, \
    audioSizeInfoType_model

class PersonLookup(ModelLookup):
    model = personInfoType_model

    def get_query(self, request, term):
        #results = super(PersonLookup, self).get_query(request, term)
        # Since MultiTextFields cannot be searched using query sets (they are base64-encoded and pickled),
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(person):
            'Helper function to group the search code for a person'
            for multifield in (person.surname, person.givenName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False
        persons = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = persons
        else:
            results = [p for p in persons if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class GenericUnicodeLookup(ModelLookup):
    '''
    A reusable base class for lookups that have to be carried out by hand
    (rather than using querysets to do an efficient database search),
    doing string matching on the unicode string representing a database item.
    '''
    def get_query(self, request, term):
        # Since Subclassables and some other classes cannot be searched using query sets,
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(item):
            'Helper function to group the search code for a database item'
            if (isinstance(ModelLookup, actorInfoType_model) and item.as_subclass().copy_status != "m"):
                return False
            return lcterm in unicode(item).lower()

        items = ''
        if isinstance(ModelLookup, documentInfoType_model):
            items = self.get_queryset().filter(copy_status="m")
        else:
            items = self.get_queryset()
        if term == '*':
            results = items
        else:
            results = [p for p in items if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class ActorLookup(GenericUnicodeLookup):
    model = actorInfoType_model

class DocumentationLookup(ModelLookup):
    '''
    A special lookup which can represent values of both
    the (structured) documentInfo type and the documentUnstructured text-only type,
    but which performes lookup only on the structured entries.
    '''
    model = documentationInfoType_model

    def get_query(self, request, term):
        # Since Subclassables and some other classes cannot be searched using query sets,
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(item):
            'Helper function to group the search code for a database item'
            if item.as_subclass().copy_status != "m":
                return False

            return lcterm in unicode(item).lower()
        items = documentInfoType_model.objects.get_query_set()
        if term == '*':
            results = items
        else:
            results = [p for p in items if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

class MembershipDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = membershipInfoType_model

class AnnotationLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = annotationInfoType_model

class VideoFormatLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = videoFormatInfoType_model

class ImageFormatLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = imageFormatInfoType_model

class ResolutionDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = resolutionInfoType_model

class LanguageVarietyDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = languageVarietyInfoType_model

class SizeDummyLookup(ModelLookup):
    '''
        Dummy class for use with OneToOneWidget.
        Should be removed when unnecessary dependencies are
        removed from OneToOneWidget
    '''
    model = sizeInfoType_model

class AudioSizeDummyLookup(ModelLookup):
    model = audioSizeInfoType_model

class ProjectLookup(ModelLookup):
    model = projectInfoType_model
    search_fields = ('projectName__contains', )
    filters = {}

    def get_query(self, request, term):
        lcterm = term.lower()

        def matches(project):
            'Helper function to group the search code for a project'
            for multifield in (project.projectName, project.projectShortName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False

        projects = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = projects
        else:
            results = [p for p in projects if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

    def get_item_label(self, item):
        short_names = ''.join(item.projectShortName.itervalues())
        names = ''.join(item.projectName.itervalues())
        res = u'%s: %s' % (short_names, names)
        return res

    def get_item_id(self, item):
        return item.id

class OrganizationLookup(ModelLookup):
    model = organizationInfoType_model

    def get_query(self, request, term):
        # Since MultiTextFields cannot be searched using query sets (they are base64-encoded and pickled),
        # we must do the searching by hand.
        # Note: this is inefficient, but in practice fast enough it seems (tested with >1000 resources)
        lcterm = term.lower()
        def matches(org):
            'Helper function to group the search code for a person'
            for multifield in (org.organizationShortName, org.organizationName):
                for field in multifield.itervalues():
                    if lcterm in field.lower():
                        return True
            return False
        orgs = self.get_queryset().filter(copy_status="m")
        if term == '*':
            results = orgs
        else:
            results = [p for p in orgs if matches(p)]
        if results is not None:
            print u'{} results'.format(results.__len__())
        else:
            print u'No results'
        return results

    def get_item_label(self, item):
        short_names = ''.join(item.organizationShortName.itervalues())
        names = ''.join(item.organizationName.itervalues())
        res = u'%s: %s' % (short_names, names)
        return res

class DocumentLookup(GenericUnicodeLookup):
    model = documentInfoType_model

class TargetResourceLookup(GenericUnicodeLookup):
    model = targetResourceInfoType_model

registry.register(PersonLookup)
registry.register(ActorLookup)
registry.register(DocumentationLookup)
registry.register(DocumentLookup)
registry.register(ProjectLookup)
registry.register(OrganizationLookup)
registry.register(TargetResourceLookup)

<|endoftext|>


<commit_before>from django.conf import settings

from sumo.redis_utils import redis_client
from customercare.cron import get_customercare_stats

print "Removing old data"
redis = redis_client(name='default')
redis.delete(settings.CC_TOP_CONTRIB_CACHE_KEY)

print "Collecting new data."
get_customercare_stats()

print "Done"
<commit_msg>Skip the migration if Redis is not installed.<commit_after>from django.conf import settings

from sumo.redis_utils import redis_client, RedisError
from customercare.cron import get_customercare_stats

try:
    print "Removing old data"
    redis = redis_client(name='default')
    redis.delete(settings.CC_TOP_CONTRIB_CACHE_KEY)

    print "Collecting new data."
    get_customercare_stats()

    print "Done"
except RedisError:
    print "This migration needs Redis to be done."
<|endoftext|>