<commit_before>#!/usr/bin/python2.7
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Run tests in parallel."""

import argparse
import glob
import itertools
import json
import multiprocessing
import os
import sys
import time

import jobset
import watch_dirs

# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class SimpleConfig(object):

  def __init__(self, config, environ={}):
    self.build_config = config
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = (config != 'gcov')
    self.environ = environ

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=[binary],
                          environ=self.environ,
                          hash_targets=hash_targets
                              if self.allow_hashing else None)

# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
class ValgrindConfig(object):

  def __init__(self, config, tool):
    self.build_config = config
    self.tool = tool
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = False

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool, binary],
                   hash_targets=None)

class CLanguage(object):

  def __init__(self, make_target, test_lang):
    self.make_target = make_target
    with open('tools/run_tests/tests.json') as f:
      js = json.load(f)
      self.binaries = [tgt['name']
                       for tgt in js
                       if tgt['language'] == test_lang]

  def test_specs(self, config):
    out = []
    for name in self.binaries:
      binary = 'bins/%s/%s' % (config.build_config, name)
      out.append(config.job_spec(binary, [binary]))
    return out

  def make_targets(self):
    return ['buildtests_%s' % self.make_target]

  def build_steps(self):
    return []

class NodeLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_node.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_node.sh']]

class PhpLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('src/php/bin/run_tests.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_php.sh']]

class PythonLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_python.sh', None)]

  def make_targets(self):
    return[]

  def build_steps(self):
    return [['tools/run_tests/build_python.sh']]

# different configurations we can run under
_CONFIGS = {
    'dbg': SimpleConfig('dbg'),
    'opt': SimpleConfig('opt'),
    'tsan': SimpleConfig('tsan', environ={
        'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt'}),
    'msan': SimpleConfig('msan'),
    'ubsan': SimpleConfig('ubsan'),
    'asan': SimpleConfig('asan', environ={
        'ASAN_OPTIONS': 'detect_leaks=1:color=always:suppressions=tools/tsan_suppressions.txt'}),
    'gcov': SimpleConfig('gcov'),
    'memcheck': ValgrindConfig('valgrind', 'memcheck'),
    'helgrind': ValgrindConfig('dbg', 'helgrind')
    }

_DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'php': PhpLanguage(),
    'python': PythonLanguage(),
    }

# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=['all'] + sorted(_CONFIGS.keys()),
                  nargs='+',
                  default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=int)
argp.add_argument('-j', '--jobs', default=1000, type=int)
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=sorted(_LANGUAGES.keys()))
args = argp.parse_args()

# grab config
run_configs = set(_CONFIGS[cfg]
                  for cfg in itertools.chain.from_iterable(
                      _CONFIGS.iterkeys() if x == 'all' else [x]
                      for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)

make_targets = []
languages = set(_LANGUAGES[l] for l in args.language)
build_steps = [jobset.JobSpec(['make',
                               '-j', '%d' % (multiprocessing.cpu_count() + 1),
                               'CONFIG=%s' % cfg] + list(set(
                                   itertools.chain.from_iterable(
                                       l.make_targets() for l in languages))))
               for cfg in build_configs] + list(set(
                   jobset.JobSpec(cmdline)
                   for l in languages
                   for cmdline in l.build_steps()))
one_run = set(
    spec
    for config in run_configs
    for language in args.language
    for spec in _LANGUAGES[language].test_specs(config))

runs_per_test = args.runs_per_test
forever = args.forever

class TestCache(object):
  """Cache for running tests."""

  def __init__(self, use_cache_results):
    self._last_successful_run = {}
    self._use_cache_results = use_cache_results

  def should_run(self, cmdline, bin_hash):
    if cmdline not in self._last_successful_run:
      return True
    if self._last_successful_run[cmdline] != bin_hash:
      return True
    if not self._use_cache_results:
      return True
    return False

  def finished(self, cmdline, bin_hash):
    self._last_successful_run[cmdline] = bin_hash

  def dump(self):
    return [{'cmdline': k, 'hash': v}
            for k, v in self._last_successful_run.iteritems()]

  def parse(self, exdump):
    self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump)

  def save(self):
    with open('.run_tests_cache', 'w') as f:
      f.write(json.dumps(self.dump()))

  def maybe_load(self):
    if os.path.exists('.run_tests_cache'):
      with open('.run_tests_cache') as f:
        self.parse(json.loads(f.read()))

def _build_and_run(check_cancelled, newline_on_success, cache):
  """Do one pass of building & running tests."""
  # build latest sequentially
  if not jobset.run(build_steps, maxjobs=1):
    return 1

  # run all the tests
  all_runs = itertools.chain.from_iterable(
      itertools.repeat(one_run, runs_per_test))
  if not jobset.run(all_runs, check_cancelled,
                    newline_on_success=newline_on_success,
                    maxjobs=min(args.jobs, min(c.maxjobs for c in run_configs)),
                    cache=cache):
    return 2

  return 0

test_cache = TestCache(runs_per_test == 1)
test_cache.maybe_load()

if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    success = _build_and_run(check_cancelled=have_files_changed,
                             newline_on_success=False,
                             cache=test_cache) == 0
    if not previous_success and success:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    test_cache.save()
    while not have_files_changed():
      time.sleep(1)
else:
  result = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          cache=test_cache)
  if result == 0:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  test_cache.save()
  sys.exit(result)
<commit_msg>Make valgrind a bit more useful<commit_after>#!/usr/bin/python2.7
# Copyright 2015, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Run tests in parallel."""

import argparse
import glob
import itertools
import json
import multiprocessing
import os
import sys
import time

import jobset
import watch_dirs

# SimpleConfig: just compile with CONFIG=config, and run the binary to test
class SimpleConfig(object):

  def __init__(self, config, environ={}):
    self.build_config = config
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = (config != 'gcov')
    self.environ = environ

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=[binary],
                          environ=self.environ,
                          hash_targets=hash_targets
                              if self.allow_hashing else None)

# ValgrindConfig: compile with some CONFIG=config, but use valgrind to run
class ValgrindConfig(object):

  def __init__(self, config, tool, args=[]):
    self.build_config = config
    self.tool = tool
    self.args = args
    self.maxjobs = 2 * multiprocessing.cpu_count()
    self.allow_hashing = False

  def job_spec(self, binary, hash_targets):
    return jobset.JobSpec(cmdline=['valgrind', '--tool=%s' % self.tool] +
                          self.args + [binary],
                          shortname='valgrind %s' % binary,
                          hash_targets=None)

class CLanguage(object):

  def __init__(self, make_target, test_lang):
    self.make_target = make_target
    with open('tools/run_tests/tests.json') as f:
      js = json.load(f)
      self.binaries = [tgt['name']
                       for tgt in js
                       if tgt['language'] == test_lang]

  def test_specs(self, config):
    out = []
    for name in self.binaries:
      binary = 'bins/%s/%s' % (config.build_config, name)
      out.append(config.job_spec(binary, [binary]))
    return out

  def make_targets(self):
    return ['buildtests_%s' % self.make_target]

  def build_steps(self):
    return []

class NodeLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_node.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_node.sh']]

class PhpLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('src/php/bin/run_tests.sh', None)]

  def make_targets(self):
    return ['static_c']

  def build_steps(self):
    return [['tools/run_tests/build_php.sh']]

class PythonLanguage(object):

  def test_specs(self, config):
    return [config.job_spec('tools/run_tests/run_python.sh', None)]

  def make_targets(self):
    return[]

  def build_steps(self):
    return [['tools/run_tests/build_python.sh']]

# different configurations we can run under
_CONFIGS = {
    'dbg': SimpleConfig('dbg'),
    'opt': SimpleConfig('opt'),
    'tsan': SimpleConfig('tsan', environ={
        'TSAN_OPTIONS': 'suppressions=tools/tsan_suppressions.txt'}),
    'msan': SimpleConfig('msan'),
    'ubsan': SimpleConfig('ubsan'),
    'asan': SimpleConfig('asan', environ={
        'ASAN_OPTIONS': 'detect_leaks=1:color=always:suppressions=tools/tsan_suppressions.txt'}),
    'gcov': SimpleConfig('gcov'),
    'memcheck': ValgrindConfig('valgrind', 'memcheck', ['--leak-check=full']),
    'helgrind': ValgrindConfig('dbg', 'helgrind')
    }

_DEFAULT = ['dbg', 'opt']
_LANGUAGES = {
    'c++': CLanguage('cxx', 'c++'),
    'c': CLanguage('c', 'c'),
    'node': NodeLanguage(),
    'php': PhpLanguage(),
    'python': PythonLanguage(),
    }

# parse command line
argp = argparse.ArgumentParser(description='Run grpc tests.')
argp.add_argument('-c', '--config',
                  choices=['all'] + sorted(_CONFIGS.keys()),
                  nargs='+',
                  default=_DEFAULT)
argp.add_argument('-n', '--runs_per_test', default=1, type=int)
argp.add_argument('-j', '--jobs', default=1000, type=int)
argp.add_argument('-f', '--forever',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('--newline_on_success',
                  default=False,
                  action='store_const',
                  const=True)
argp.add_argument('-l', '--language',
                  choices=sorted(_LANGUAGES.keys()),
                  nargs='+',
                  default=sorted(_LANGUAGES.keys()))
args = argp.parse_args()

# grab config
run_configs = set(_CONFIGS[cfg]
                  for cfg in itertools.chain.from_iterable(
                      _CONFIGS.iterkeys() if x == 'all' else [x]
                      for x in args.config))
build_configs = set(cfg.build_config for cfg in run_configs)

make_targets = []
languages = set(_LANGUAGES[l] for l in args.language)
build_steps = [jobset.JobSpec(['make',
                               '-j', '%d' % (multiprocessing.cpu_count() + 1),
                               'CONFIG=%s' % cfg] + list(set(
                                   itertools.chain.from_iterable(
                                       l.make_targets() for l in languages))))
               for cfg in build_configs] + list(set(
                   jobset.JobSpec(cmdline)
                   for l in languages
                   for cmdline in l.build_steps()))
one_run = set(
    spec
    for config in run_configs
    for language in args.language
    for spec in _LANGUAGES[language].test_specs(config))

runs_per_test = args.runs_per_test
forever = args.forever

class TestCache(object):
  """Cache for running tests."""

  def __init__(self, use_cache_results):
    self._last_successful_run = {}
    self._use_cache_results = use_cache_results

  def should_run(self, cmdline, bin_hash):
    if cmdline not in self._last_successful_run:
      return True
    if self._last_successful_run[cmdline] != bin_hash:
      return True
    if not self._use_cache_results:
      return True
    return False

  def finished(self, cmdline, bin_hash):
    self._last_successful_run[cmdline] = bin_hash

  def dump(self):
    return [{'cmdline': k, 'hash': v}
            for k, v in self._last_successful_run.iteritems()]

  def parse(self, exdump):
    self._last_successful_run = dict((o['cmdline'], o['hash']) for o in exdump)

  def save(self):
    with open('.run_tests_cache', 'w') as f:
      f.write(json.dumps(self.dump()))

  def maybe_load(self):
    if os.path.exists('.run_tests_cache'):
      with open('.run_tests_cache') as f:
        self.parse(json.loads(f.read()))

def _build_and_run(check_cancelled, newline_on_success, cache):
  """Do one pass of building & running tests."""
  # build latest sequentially
  if not jobset.run(build_steps, maxjobs=1):
    return 1

  # run all the tests
  all_runs = itertools.chain.from_iterable(
      itertools.repeat(one_run, runs_per_test))
  if not jobset.run(all_runs, check_cancelled,
                    newline_on_success=newline_on_success,
                    maxjobs=min(args.jobs, min(c.maxjobs for c in run_configs)),
                    cache=cache):
    return 2

  return 0

test_cache = TestCache(runs_per_test == 1)
test_cache.maybe_load()

if forever:
  success = True
  while True:
    dw = watch_dirs.DirWatcher(['src', 'include', 'test'])
    initial_time = dw.most_recent_change()
    have_files_changed = lambda: dw.most_recent_change() != initial_time
    previous_success = success
    success = _build_and_run(check_cancelled=have_files_changed,
                             newline_on_success=False,
                             cache=test_cache) == 0
    if not previous_success and success:
      jobset.message('SUCCESS',
                     'All tests are now passing properly',
                     do_newline=True)
    jobset.message('IDLE', 'No change detected')
    test_cache.save()
    while not have_files_changed():
      time.sleep(1)
else:
  result = _build_and_run(check_cancelled=lambda: False,
                          newline_on_success=args.newline_on_success,
                          cache=test_cache)
  if result == 0:
    jobset.message('SUCCESS', 'All tests passed', do_newline=True)
  else:
    jobset.message('FAILED', 'Some tests failed', do_newline=True)
  test_cache.save()
  sys.exit(result)
<|endoftext|>


<commit_before>from django.core.management.base import BaseCommand, CommandError

from core.models import Host, Plugin, Data_Source as Source
from configuration.forms.plugin import Plugin_Form
from core.management.commands._utils import CommandDispatcher

from optparse import make_option
from os import devnull
import sys
import re

class Command(CommandDispatcher):
    """Storage management base command."""
    actions = ('list','add','delete','del')
    def _subcommand_names(self):
        return ('list','add','delete','del')

    def _subcommand(self, *args, **opts):
        """Dispatch in a Command by reading first argv."""
        if not args or args[0] not in self.actions:
            self.stdout.write(self.usage('host'))
        elif args[0] == 'list':
            return List_Command()
        elif args[0] == 'add':
            return Add_Command()
        elif args[0] in ('delete','del'):
            return Delete_Command()

ROW_FORMAT = '{id:5} | {name:35} | {plugin:25} | {host:30}'
class List_Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-i', '--ids', action='store', help="Select plugins by ID separated by comma"),
        make_option('-s', '--saved', action='store_true', default=False, help="Only list plugins saved in db"),
    )

    def handle(self, *args, **opts):
        # Select plugin by id or ids
        if opts['ids']:
            ids = [ i.strip() for i in opts['ids'].split(',') ]
            plugins = Plugin.objects.filter(id__in=ids)
        else:
            plugins = Plugin.objects.all()
        # Stop if no given id
        if not plugins.exists():
            self.stdout.write("There's no plugin with given ID: '%s'" % opts['ids'] )
            sys.exit(1)
        # Walk on host and list plugins
        for p in plugins:
            self.stdout.write("* %s sources:" % p)
            self.stdout.write(ROW_FORMAT.format(**{u'id': 'ID', 'name': 'Name', 'plugin': 'Plugin', 'host': 'Host'}))
            # List plugins
            for s in p.get_data_sources():
                if Source.objects.filter(plugin=p, name=s).exists():
                    s = Source.objects.create(plugin=p, name=s).__dict__
                    s.update({'plugin': p.name, 'host': p.host.name})
                    self.stdout.write(ROW_FORMAT.format(**s))
                elif not opts['saved']:
                    s = {'plugin': p.name, 'host': p.host.name, 'name': s, 'id': 'None'}
                    self.stdout.write(ROW_FORMAT.format(**s))
            #self.stdout.write('Saved cound: %i' % sources

# class Delete_Command(BaseCommand):
#     option_list = BaseCommand.option_list + (
#         make_option('-i', '--ids', action='store', default=None, help="Select plugin by ID separated by comma"),
#         make_option('-q', '--quiet', action='store_true', help="Don't print info"),
#     )
#
#     def handle(self, *args, **opts):
#         if opts['quiet']: self.stdout = open(devnull, 'w')
#         # Select group by ids
#         if opts['ids']:
#             ids = [ i.strip() for i in opts['ids'].split(',') ]
#             plugins = Plugin.objects.filter(id__in=ids)
#         else:
#             self.stdout.write("You must give one or more ID.")
#             self.print_help('plugin', 'help')
#             sys.exit(1)
#         # Stop if no given id
#         if not plugins.exists():
#             self.stdout.write("There's no plugin with given ID: '%s'" % opts['ids'] )
#             sys.exit(1)
#         for p in plugins:
#             p.delete()
#             self.stdout.write('Delete plugin: %s' % p)
#
#
class Add_Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-i', '--ids', action='store', help="Select plugins by ID separated by comma"),
        make_option('-p', '--pattern', action='store', default='.*', help="Select sources by pattern"),
        make_option('-q', '--quiet', action='store_true', help="Don't print info"),
    )

    def handle(self, *args, **opts):
        if opts['quiet']: self.stdout = open(devnull, 'w')
        if opts['ids']:
            ids = [ i.strip() for i in opts['ids'].split(',') ]
            plugins = Plugin.objects.filter(id__in=ids)
        else:
            plugins = Plugin.objects.all()
        # Skip if plugins don't exists
        if not plugins.exists():
            self.stdout.write("There's no plugin with given ID: '%s'" % opts['ids'] )
            sys.exit(1)

        # Walk on plugin and save matching sources
        REG_PLUGIN = re.compile(opts['pattern'])
        self.stdout.write(ROW_FORMAT.format(**{u'id': 'ID', 'name': 'Name', 'plugin': 'Plugin', 'host': 'Host'}))
        for p in plugins:
            sources = p.get_data_sources()
            for s in sources:
                if REG_PLUGIN.search(s) and not Source.objects.filter(name=s, plugin=p).exists():
                    s = Source.objects.create(name=s, plugin=p)
                    s_data = s.__dict__
                    s_data.update({'plugin': p.name, 'host': p.host})
                    self.stdout.write(ROW_FORMAT.format(**s_data))
<commit_msg>Add source delete cmd<commit_after>from django.core.management.base import BaseCommand, CommandError

from core.models import Host, Plugin, Data_Source as Source
from configuration.forms.plugin import Plugin_Form
from core.management.commands._utils import CommandDispatcher

from optparse import make_option
from os import devnull
import sys
import re

class Command(CommandDispatcher):
    """Storage management base command."""
    actions = ('list','add','delete','del')
    def _subcommand_names(self):
        return ('list','add','delete','del')

    def _subcommand(self, *args, **opts):
        """Dispatch in a Command by reading first argv."""
        if not args or args[0] not in self.actions:
            self.stdout.write(self.usage('host'))
        elif args[0] == 'list':
            return List_Command()
        elif args[0] == 'add':
            return Add_Command()
        elif args[0] in ('delete','del'):
            return Delete_Command()

ROW_FORMAT = '{id:5} | {name:35} | {plugin:25} | {host:30}'
class List_Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-i', '--ids', action='store', help="Select plugins by ID separated by comma"),
        make_option('-s', '--saved', action='store_true', default=False, help="Only list plugins saved in db"),
    )

    def handle(self, *args, **opts):
        # Select plugin by id or ids
        if opts['ids']:
            ids = [ i.strip() for i in opts['ids'].split(',') ]
            plugins = Plugin.objects.filter(id__in=ids)
        else:
            plugins = Plugin.objects.all()
        # Stop if no given id
        if not plugins.exists():
            self.stdout.write("There's no plugin with given ID: '%s'" % opts['ids'] )
            sys.exit(1)
        # Walk on host and list plugins
        for p in plugins:
            self.stdout.write("* %s sources:" % p)
            self.stdout.write(ROW_FORMAT.format(**{u'id': 'ID', 'name': 'Name', 'plugin': 'Plugin', 'host': 'Host'}))
            # List plugins
            for s in p.get_data_sources():
                if Source.objects.filter(plugin=p, name=s).exists():
                    s = Source.objects.create(plugin=p, name=s).__dict__
                    s.update({'plugin': p.name, 'host': p.host.name})
                    self.stdout.write(ROW_FORMAT.format(**s))
                elif not opts['saved']:
                    s = {'plugin': p.name, 'host': p.host.name, 'name': s, 'id': 'None'}
                    self.stdout.write(ROW_FORMAT.format(**s))
            #self.stdout.write('Saved cound: %i' % sources

class Delete_Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-i', '--ids', action='store', default=None, help="Select source by ID separated by comma"),
        make_option('-q', '--quiet', action='store_true', help="Don't print info"),
    )

    def handle(self, *args, **opts):
        if opts['quiet']: self.stdout = open(devnull, 'w')
        # Select group by ids
        if opts['ids']:
            ids = [ i.strip() for i in opts['ids'].split(',') ]
            sources = Source.objects.filter(id__in=ids)
        else:
            self.stdout.write("You must give one or more ID.")
            self.print_help('source', 'help')
            sys.exit(1)
        # Stop if no given id
        if not sources.exists():
            self.stdout.write("There's no source with given ID: '%s'" % opts['ids'] )
            sys.exit(1)
        for s in sources:
            s.delete()
            self.stdout.write('Delete source: %s' % s)

class Add_Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('-i', '--ids', action='store', help="Select plugins by ID separated by comma"),
        make_option('-p', '--pattern', action='store', default='.*', help="Select sources by pattern"),
        make_option('-q', '--quiet', action='store_true', help="Don't print info"),
    )

    def handle(self, *args, **opts):
        if opts['quiet']: self.stdout = open(devnull, 'w')
        if opts['ids']:
            ids = [ i.strip() for i in opts['ids'].split(',') ]
            plugins = Plugin.objects.filter(id__in=ids)
        else:
            plugins = Plugin.objects.all()
        # Skip if plugins don't exists
        if not plugins.exists():
            self.stdout.write("There's no plugin with given ID: '%s'" % opts['ids'] )
            sys.exit(1)

        # Walk on plugin and save matching sources
        REG_PLUGIN = re.compile(opts['pattern'])
        self.stdout.write(ROW_FORMAT.format(**{u'id': 'ID', 'name': 'Name', 'plugin': 'Plugin', 'host': 'Host'}))
        for p in plugins:
            sources = p.get_data_sources()
            for s in sources:
                if REG_PLUGIN.search(s) and not Source.objects.filter(name=s, plugin=p).exists():
                    s = Source.objects.create(name=s, plugin=p)
                    s_data = s.__dict__
                    s_data.update({'plugin': p.name, 'host': p.host})
                    self.stdout.write(ROW_FORMAT.format(**s_data))
<|endoftext|>


<commit_before>import logging
import time
import threading
from typing import List, Tuple, Any, Dict
from unittest import TestCase
from lib.client.core import Connection, RecvTimeout
from lib.client.communication import Communication
from lib.client.util import MsgBuilder

class SimConnection(Connection):

    def __init__(self,
                 send_msg_queue: List[str],
                 send_lock: threading.Lock,
                 recv_msg_queue: List[str],
                 recv_lock: threading.Lock,
                 tag: str):

        self._send_msg_queue = send_msg_queue
        self._send_lock = send_lock
        self._recv_msg_queue = recv_msg_queue
        self._recv_lock = recv_lock
        self._tag = tag

    def send(self,
             data: str):

        logging.debug("[%s]: Sending: %s" % (self._tag, data))
        with self._send_lock:
            self._send_msg_queue.append(data)

    def recv(self,
             buffsize: int,
             timeout: float = 20.0) -> str:

        logging.debug("[%s]: Start receiving." % self._tag)
        start_time = time.time()
        while True:
            # Check if our received timed out.
            if (time.time() - start_time) > timeout:
                logging.debug("[%s]: Timeout while receiving." % self._tag)
                raise RecvTimeout

            with self._recv_lock:
                if self._recv_msg_queue:
                    data = self._recv_msg_queue.pop(0)
                    logging.debug("[%s]: Received: %s" % (self._tag, data))
                    return data
            time.sleep(0.2)

class TestCommunicationBasic(TestCase):

    def _config_logging(self):
        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                            datefmt='%m/%d/%Y %H:%M:%S',
                            level=logging.DEBUG)

    def _create_communication(self) -> Tuple[Communication, Communication]:
        lock_send_client = threading.Lock()
        lock_send_server = threading.Lock()
        msg_queue_send_client = []
        msg_queue_send_server = []

        conn_client = SimConnection(msg_queue_send_client,
                                    lock_send_client,
                                    msg_queue_send_server,
                                    lock_send_server,
                                    "client")

        conn_server = SimConnection(msg_queue_send_server,
                                    lock_send_server,
                                    msg_queue_send_client,
                                    lock_send_client,
                                    "server")

        comm_client = Communication(conn_client)
        comm_server = Communication(conn_server, is_server=True)

        comm_client._log_tag = "client"
        comm_server._log_tag = "server"

        comm_client.set_connected()
        comm_server.set_connected()

        return comm_client, comm_server

    def _msg_receiver(self,
                      **kwargs):

        count = kwargs["count"]  # type: int
        comm = kwargs["comm"]  # type: Communication
        recv_msgs = kwargs["recv_msgs"]  # type: List[Dict[str, Any]]
        sync = kwargs["sync"]  # type: threading.Event

        # Wait until we are clear to receive messages.
        sync.wait()
        logging.debug("[%s]: Starting receiver loop." % comm._log_tag)

        for _ in range(count):
            recv_msg = comm.recv_request()
            recv_msgs.append(recv_msg)

    def test_single_communication(self):

        self._config_logging()

        comm_client, comm_server = self._create_communication()

        ping_msg = MsgBuilder.build_ping_msg()
        promise = comm_client.send_request("ping", ping_msg)

        recv_msg = comm_server.recv_request()
        if recv_msg is None:
            self.fail("Receiving message failed.")

        if "ping" != recv_msg["message"]:
            self.fail("Expected 'ping' message.")

        if not promise.is_finished(timeout=5.0):
            self.fail("Expected message to be sent.")

        if not promise.was_successful():
            self.fail("Sending message was not successful.")

    def test_stress_communication(self):

        count = 1

        self._config_logging()

        comm_client, comm_server = self._create_communication()

        receiving_sync = threading.Event()
        receiving_sync.clear()

        msgs_recv_server = []
        kwargs = {"count": count,
                  "comm": comm_server,
                  "recv_msgs": msgs_recv_server,
                  "sync": receiving_sync}
        server_receiver = threading.Thread(target=self._msg_receiver,
                                           kwargs=kwargs,
                                           daemon=True)
        server_receiver.start()

        msgs_recv_client = []
        kwargs = {"count": count,
                  "comm": comm_client,
                  "recv_msgs": msgs_recv_client,
                  "sync": receiving_sync}
        client_receiver = threading.Thread(target=self._msg_receiver,
                                           kwargs=kwargs,
                                           daemon=True)
        client_receiver.start()

        receiving_sync.set()

        # Send requests from client to server.
        requests_client = []
        for _ in range(count):
            ping_msg = MsgBuilder.build_ping_msg()
            promise = comm_client.send_request("ping", ping_msg)
            requests_client.append(promise)

        # Send requests from server to client.
        requests_server = []
        for _ in range(count):
            ping_msg = MsgBuilder.build_ping_msg()
            ping_msg = ping_msg.replace("clientTime", "serverTime")
            promise = comm_server.send_request("ping", ping_msg)
            requests_server.append(promise)

        # Give each received message 2 seconds time.
        client_receiver.join(timeout=(count * 10.0)) # TODO
        if client_receiver.isAlive():
            self.fail("Client timed out while receiving messages.")

        # Give each received message 2 seconds time.
        server_receiver.join(timeout=(count * 10.0)) # TODO
        if server_receiver.isAlive():
            self.fail("Server timed out while receiving messages.")

        if len(requests_client) != len(msgs_recv_server):
            self.fail("Client requests differ from messages received by server.")

        if len(requests_server) != len(msgs_recv_client):
            self.fail("Server requests differ from messages received by client.")

        for i in range(len(requests_client)):
            promise = requests_client[i]
            recv_msg = msgs_recv_server[i]

            if promise.msg_type != recv_msg["message"]:
                self.fail("Message type from send and receive different.")

            # TODO check if correct order

        print("HIER")<commit_msg>finished communication channel stress test<commit_after>import logging
import time
import threading
import json
from typing import List, Tuple, Any, Dict
from unittest import TestCase
from lib.client.core import Connection, RecvTimeout
from lib.client.communication import Communication
from lib.client.util import MsgBuilder

class SimConnection(Connection):

    def __init__(self,
                 send_msg_queue: List[str],
                 send_lock: threading.Lock,
                 recv_msg_queue: List[str],
                 recv_lock: threading.Lock,
                 tag: str):

        self._send_msg_queue = send_msg_queue
        self._send_lock = send_lock
        self._recv_msg_queue = recv_msg_queue
        self._recv_lock = recv_lock
        self._tag = tag

    def send(self,
             data: str):

        logging.debug("[%s]: Sending: %s" % (self._tag, data))
        with self._send_lock:
            self._send_msg_queue.append(data)

    def recv(self,
             buffsize: int,
             timeout: float = 20.0) -> str:

        logging.debug("[%s]: Start receiving." % self._tag)
        start_time = time.time()
        while True:
            # Check if our received timed out.
            if (time.time() - start_time) > timeout:
                logging.debug("[%s]: Timeout while receiving." % self._tag)
                raise RecvTimeout

            with self._recv_lock:
                if self._recv_msg_queue:
                    data = self._recv_msg_queue.pop(0)
                    logging.debug("[%s]: Received: %s" % (self._tag, data))
                    return data
            time.sleep(0.2)

class TestCommunicationBasic(TestCase):

    def _config_logging(self):
        logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                            datefmt='%m/%d/%Y %H:%M:%S',
                            level=logging.ERROR)

    def _create_communication(self) -> Tuple[Communication, Communication]:
        lock_send_client = threading.Lock()
        lock_send_server = threading.Lock()
        msg_queue_send_client = []
        msg_queue_send_server = []

        conn_client = SimConnection(msg_queue_send_client,
                                    lock_send_client,
                                    msg_queue_send_server,
                                    lock_send_server,
                                    "client")

        conn_server = SimConnection(msg_queue_send_server,
                                    lock_send_server,
                                    msg_queue_send_client,
                                    lock_send_client,
                                    "server")

        comm_client = Communication(conn_client)
        comm_server = Communication(conn_server, is_server=True)

        comm_client._log_tag = "client"
        comm_server._log_tag = "server"

        comm_client.set_connected()
        comm_server.set_connected()

        return comm_client, comm_server

    def _msg_receiver(self,
                      **kwargs):

        count = kwargs["count"]  # type: int
        comm = kwargs["comm"]  # type: Communication
        recv_msgs = kwargs["recv_msgs"]  # type: List[Dict[str, Any]]
        sync = kwargs["sync"]  # type: threading.Event

        # Wait until we are clear to receive messages.
        sync.wait()
        logging.debug("[%s]: Starting receiver loop." % comm._log_tag)

        for _ in range(count):
            recv_msg = comm.recv_request()
            recv_msgs.append(recv_msg)

    def test_single_communication(self):
        """
        Tests single request sending through the communication channel from the client to the server.
        """
        self._config_logging()

        comm_client, comm_server = self._create_communication()

        ping_msg = MsgBuilder.build_ping_msg()
        promise = comm_client.send_request("ping", ping_msg)

        recv_msg = comm_server.recv_request()
        if recv_msg is None:
            self.fail("Receiving message failed.")

        if "ping" != recv_msg["message"]:
            self.fail("Expected 'ping' message.")

        if not promise.is_finished(timeout=5.0):
            self.fail("Expected message to be sent.")

        if not promise.was_successful():
            self.fail("Sending message was not successful.")

    def test_stress_communication(self):
        """
        Stress tests communication by letting client and server trying to send
        X messages to each other at the same time. Checks order of the send/received messages
        as well as not to take too long to send messages.
        """

        count = 30

        self._config_logging()

        comm_client, comm_server = self._create_communication()

        receiving_sync = threading.Event()
        receiving_sync.clear()

        msgs_recv_server = []
        kwargs = {"count": count,
                  "comm": comm_server,
                  "recv_msgs": msgs_recv_server,
                  "sync": receiving_sync}
        server_receiver = threading.Thread(target=self._msg_receiver,
                                           kwargs=kwargs,
                                           daemon=True)
        server_receiver.start()

        msgs_recv_client = []
        kwargs = {"count": count,
                  "comm": comm_client,
                  "recv_msgs": msgs_recv_client,
                  "sync": receiving_sync}
        client_receiver = threading.Thread(target=self._msg_receiver,
                                           kwargs=kwargs,
                                           daemon=True)
        client_receiver.start()

        start_timer = time.time()
        receiving_sync.set()

        # Send requests from client to server.
        requests_client = []
        for i in range(count):
            ping_msg = MsgBuilder.build_ping_msg()
            ping_dict = json.loads(ping_msg)
            # Insert bogus field to uniquely identify messages on the other side.
            ping_dict["test_msg_name"] = "client_" + str(i)
            ping_msg = json.dumps(ping_dict)
            promise = comm_client.send_request("ping", ping_msg)
            requests_client.append(promise)

        # Send requests from server to client.
        requests_server = []
        for i in range(count):
            ping_msg = MsgBuilder.build_ping_msg()
            ping_dict = json.loads(ping_msg)
            # Since the server is sending the message, swap clientTime with serverTime
            ping_dict["serverTime"] = ping_dict["clientTime"]
            del ping_dict["clientTime"]
            # Insert bogus field to uniquely identify messages on the other side.
            ping_dict["test_msg_name"] = "server_" + str(i)
            ping_msg = json.dumps(ping_dict)
            promise = comm_server.send_request("ping", ping_msg)
            requests_server.append(promise)

        # Give each each message 10 seconds time
        # ("count" messages send by client and "count" messages send by server).
        for _ in range(count * 2 * 10):
            if client_receiver.isAlive():
                client_receiver.join(timeout=1.0)
            elif server_receiver.isAlive():
                server_receiver.join(timeout=1.0)
            else:
                break
        if client_receiver.isAlive():
            self.fail("Client timed out while receiving messages.")
        if server_receiver.isAlive():
            self.fail("Server timed out while receiving messages.")

        if len(requests_client) != len(msgs_recv_server):
            self.fail("Client requests differ from messages received by server.")

        if len(requests_server) != len(msgs_recv_client):
            self.fail("Server requests differ from messages received by client.")

        # Check requests send by the client.
        for i in range(len(requests_client)):
            promise = requests_client[i]
            send_msg = json.loads(promise.msg)
            recv_msg = msgs_recv_server[i]

            if promise.msg_type != recv_msg["message"]:
                self.fail("Message type from send and receive different (client -> server).")

            if send_msg["test_msg_name"] != recv_msg["test_msg_name"]:
                self.fail("Messages sent and received different or different order (client -> server).")

        # Check requests send by the server.
        for i in range(len(requests_client)):
            promise = requests_server[i]
            send_msg = json.loads(promise.msg)
            recv_msg = msgs_recv_client[i]

            if promise.msg_type != recv_msg["message"]:
                self.fail("Message type from send and receive different (server -> client).")

            if send_msg["test_msg_name"] != recv_msg["test_msg_name"]:
                self.fail("Messages sent and received different or different order (server -> client).")

        time_elapsed = time.time() - start_timer
        logging.info("Needed %.2f seconds to send/receive messages." % time_elapsed)
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for updating different kinds of models in datastore.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <rijk0214@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.ext import db

from django.utils.translation import ugettext_lazy

from soc.logic import dicts
from soc.logic import out_of_band

class Error(Exception):
  """Base class for all exceptions raised by this module."""
  pass

class Logic:
  """Base logic for entity classes.

  The BaseLogic class functions specific to Entity classes by relying
  on the the child-classes to implement _model, _name and _key_name
  """

  def _updateField(self, model, name, value):
    """Hook called when a field is updated.

    Base classes should override if any special actions need to be
    taken when a field is updated. The field is not updated if the
    method does not return a True value.
    """

    return True

  def _keyName(self, **kwargs):
    """Returns the KeyName constructed from kwargs for this type of entity

    The KeyName is in the following format:
    entity.name:<key_value1>:<key_value2>:...:<key_valueN>
    """

    # get the KeyFieldNames for this entity
    key_field_names = self.getKeyFieldNames()

    # check if all given KeyFieldNames are valid for this entity
    if not all(key in key_field_names for key in kwargs.keys()):
      raise Error("Some of the provided arguments are not key fields")

    # check if all key_field_names for this entity are present in kwargs
    if not all(field in kwargs.keys() for field in key_field_names):
        raise Error("Not all the required key fields are present")

    # check if all kwargs.values() are non-false
    if not all(kwargs.values()):
        raise Error("Not all KeyValues are non-false")

    # construct the KeyValues in the order given by getKeyFieldNames()
    keyvalues = []
    for key_field_name in key_field_names:
        keyvalues.append(kwargs[key_field_name])

    # construct the KeyName in the appropriate format
    return ":".join([self._name] + keyvalues)

  def getKeyValues(self, entity):
    """Exctracts the key values from entity and returns them

    Args:
      entity: the entity from which to extract the key values
    """

    raise NotImplementedError

  def getKeyValuesFromFields(self, fields):
    """Exctracts the key values from a dict and returns them

    Args:
      fields: the dict from which to extract the key values
    """

    raise NotImplementedError

  def getKeyFieldNames(self):
    """Returns an array with the names of the Key Fields
    """

    raise NotImplementedError

  def getKeySuffix(self, entity):
    """Returns a suffix for the specified entity or None if no entity specified

    Args:
      entity: the entity for which to get the suffix
    """

    if not entity:
      return None

    key_values = self.getKeyValues(entity)
    suffix = '/'.join(key_values)

    return suffix

  def getKeyFieldsFromDict(self, dictionary):
    """Does any required massaging and filtering of dictionary

    The resulting dictionary contains just the key names, and has any
    required translations/modifications performed.

    Args:
      dictionary: The arguments to massage
    """

    keys = self.getKeyFieldNames()
    values = self.getKeyValuesFromFields(dictionary)
    key_fields = dicts.zip(keys, values)

    return key_fields

  def getFromKeyName(self, key_name):
    """"Returns User entity for key_name or None if not found.
-
-    Args:
-      key_name: key name of entity
    """

    return self._model.get_by_key_name(key_name)

  def getFromFields(self, **kwargs):
    """Returns the entity for the specified key names, or None if not found.

    Args:
      **kwargs: the fields of the entity that uniquely identifies it
    """

    key_name = self.getKeyNameForFields(kwargs)

    if key_name:
      entity = self._model.get_by_key_name(key_name)
    else:
      entity = None

    return entity

  def getIfFields(self, fields):
    """Returns entity for supplied link name if one exists.

    Args:
      fields: the fields of the entity that uniquely identifies it

    Returns:
      * None if a field is false.
      * Entity for supplied fields

    Raises:
      out_of_band.ErrorResponse if link name is not false, but no entity
      with the supplied link name exists in the Datastore
    """

    if not all(fields.values()):
      # exit without error, to let view know that link_name was not supplied
      return None

    entity = self.getFromFields(**fields)

    if entity:
      # an entity exist for this link_name, so return that entity
      return entity

    format_text = ugettext_lazy('"%(key)s" is "%(value)s"')

    msg_pairs = [format_text % {'key': key, 'value': value}
      for key, value in fields.iteritems()]

    joined_pairs = ' and '.join(msg_pairs)

    msg = ugettext_lazy(
      'There is no "%(name)s" where %(pairs)s.') % {
        'name': self._name, 'pairs': joined_pairs}

    # else: fields were supplied, but there is no Entity that has it
    raise out_of_band.ErrorResponse(msg, status=404)

  def getKeyNameForFields(self, fields):
    """Return a Datastore key_name for a Entity from the specified fields.

    Args:
      fields: the fields of the entity that uniquely identifies it
    """

    if not all(fields.values()):
      return None

    return self._keyName(**fields)

  def getForLimitAndOffset(self, limit, offset=0):
    """Returns entities for given offset and limit or None if not found.

    Args:
      limit: max amount of entities to return
      offset: optional number of results to skip first; default zero.
    """

    query = self._model.all()
    return query.fetch(limit, offset)

  def getForFields(self, properties, unique=False, limit=1000, offset=0):
    """Returns all entities that have the specified properties

    Args:
      properties: the properties that the entity should have
      unique: if set, only the first item from the resultset will be returned
      limit: max amount of entities to return
      offset: optional number of results to skip first; default zero.
    """

    if not properties:
      raise Error("Properties did not contain any values")

    format_text = '%(key)s = :%(key)s'
    msg_pairs = [format_text % {'key': key} for key in properties.iterkeys()]
    joined_pairs = ' AND '.join(msg_pairs)
    condition = 'WHERE %s' % joined_pairs

    query = self._model.gql(condition, **properties)

    if unique:
      return query.get()

    result = query.fetch(limit, offset)
    return result

  def updateModelProperties(self, model, model_properties):
    """Update existing model entity using supplied model properties.

    Args:
      model: a model entity
      model_properties: keyword arguments that correspond to model entity
        properties and their values

    Returns:
      the original model entity with any supplied properties changed
    """

    def update():
      return self._unsafeUpdateModelProperties(model, model_properties)

    return db.run_in_transaction(update)

  def _unsafeUpdateModelProperties(self, model, model_properties):
    """(see updateModelProperties)

    Like updateModelProperties(), but not run within a transaction.
    """

    properties = model.properties()

    for prop in properties.values():
      name = prop.name

      if not name in self._skip_properties and name in model_properties:
        value = model_properties[prop.name]

        if self._updateField(model, name, value):
          prop.__set__(model, value)

    model.put()
    return model

  def updateOrCreateFromKeyName(self, properties, key_name):
    """Update existing entity, or create new one with supplied properties.

    Args:
      properties: dict with entity properties and their values
      key_name: the key_name of the entity that uniquely identifies it

    Returns:
      the entity corresponding to the key_name, with any supplied
      properties changed, or a new entity now associated with the
      supplied key_name and properties.
    """

    entity = self.getFromKeyName(key_name)

    if not entity:
      # entity did not exist, so create one in a transaction
      entity = self._model.get_or_insert(key_name, **properties)

    # there is no way to be sure if get_or_insert() returned a new entity or
    # got an existing one due to a race, so update with properties anyway,
    # in a transaction
    return self.updateModelProperties(entity, properties)

  def updateOrCreateFromFields(self, properties, fields):
    """Like updateOrCreateFromKeyName, but resolves fields to a key_name first.
    """

    # attempt to retrieve the existing entity
    key_name  = self.getKeyNameForFields(fields)

    return self.updateOrCreateFromKeyName(properties, key_name)

  def isDeletable(self, entity):
    """Returns whether the specified entity can be deleted.

    Args:
      entity: an existing entity in datastore
    """

    return True

  def delete(self, entity):
    """Delete existing entity from datastore.

    Args:
      entity: an existing entity in datastore
    """

    entity.delete()
<commit_msg>Some indention fixes and remove not needed variable in soc.logic.models.base module.<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for updating different kinds of models in datastore.
"""

__authors__ = [
  '"Todd Larsen" <tlarsen@google.com>',
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  '"Lennard de Rijk" <rijk0214@gmail.com>',
  '"Pawel Solyga" <pawel.solyga@gmail.com>',
  ]

from google.appengine.ext import db

from django.utils.translation import ugettext_lazy

from soc.logic import dicts
from soc.logic import out_of_band

class Error(Exception):
  """Base class for all exceptions raised by this module."""
  pass

class Logic:
  """Base logic for entity classes.

  The BaseLogic class functions specific to Entity classes by relying
  on the the child-classes to implement _model, _name and _key_name
  """

  def _updateField(self, model, name, value):
    """Hook called when a field is updated.

    Base classes should override if any special actions need to be
    taken when a field is updated. The field is not updated if the
    method does not return a True value.
    """

    return True

  def _keyName(self, **kwargs):
    """Returns the KeyName constructed from kwargs for this type of entity

    The KeyName is in the following format:
    entity.name:<key_value1>:<key_value2>:...:<key_valueN>
    """

    # get the KeyFieldNames for this entity
    key_field_names = self.getKeyFieldNames()

    # check if all given KeyFieldNames are valid for this entity
    if not all(key in key_field_names for key in kwargs.keys()):
      raise Error("Some of the provided arguments are not key fields")

    # check if all key_field_names for this entity are present in kwargs
    if not all(field in kwargs.keys() for field in key_field_names):
      raise Error("Not all the required key fields are present")

    # check if all kwargs.values() are non-false
    if not all(kwargs.values()):
      raise Error("Not all KeyValues are non-false")

    # construct the KeyValues in the order given by getKeyFieldNames()
    keyvalues = []
    for key_field_name in key_field_names:
      keyvalues.append(kwargs[key_field_name])

    # construct the KeyName in the appropriate format
    return ":".join([self._name] + keyvalues)

  def getKeyValues(self, entity):
    """Exctracts the key values from entity and returns them

    Args:
      entity: the entity from which to extract the key values
    """

    raise NotImplementedError

  def getKeyValuesFromFields(self, fields):
    """Exctracts the key values from a dict and returns them

    Args:
      fields: the dict from which to extract the key values
    """

    raise NotImplementedError

  def getKeyFieldNames(self):
    """Returns an array with the names of the Key Fields
    """

    raise NotImplementedError

  def getKeySuffix(self, entity):
    """Returns a suffix for the specified entity or None if no entity specified

    Args:
      entity: the entity for which to get the suffix
    """

    if not entity:
      return None

    key_values = self.getKeyValues(entity)
    suffix = '/'.join(key_values)

    return suffix

  def getKeyFieldsFromDict(self, dictionary):
    """Does any required massaging and filtering of dictionary

    The resulting dictionary contains just the key names, and has any
    required translations/modifications performed.

    Args:
      dictionary: The arguments to massage
    """

    keys = self.getKeyFieldNames()
    values = self.getKeyValuesFromFields(dictionary)
    key_fields = dicts.zip(keys, values)

    return key_fields

  def getFromKeyName(self, key_name):
    """"Returns User entity for key_name or None if not found.
-
-    Args:
-      key_name: key name of entity
    """

    return self._model.get_by_key_name(key_name)

  def getFromFields(self, **kwargs):
    """Returns the entity for the specified key names, or None if not found.

    Args:
      **kwargs: the fields of the entity that uniquely identifies it
    """

    key_name = self.getKeyNameForFields(kwargs)

    if key_name:
      entity = self._model.get_by_key_name(key_name)
    else:
      entity = None

    return entity

  def getIfFields(self, fields):
    """Returns entity for supplied link name if one exists.

    Args:
      fields: the fields of the entity that uniquely identifies it

    Returns:
      * None if a field is false.
      * Entity for supplied fields

    Raises:
      out_of_band.ErrorResponse if link name is not false, but no entity
      with the supplied link name exists in the Datastore
    """

    if not all(fields.values()):
      # exit without error, to let view know that link_name was not supplied
      return None

    entity = self.getFromFields(**fields)

    if entity:
      # an entity exist for this link_name, so return that entity
      return entity

    format_text = ugettext_lazy('"%(key)s" is "%(value)s"')

    msg_pairs = [format_text % {'key': key, 'value': value}
      for key, value in fields.iteritems()]

    joined_pairs = ' and '.join(msg_pairs)

    msg = ugettext_lazy(
      'There is no "%(name)s" where %(pairs)s.') % {
        'name': self._name, 'pairs': joined_pairs}

    # else: fields were supplied, but there is no Entity that has it
    raise out_of_band.ErrorResponse(msg, status=404)

  def getKeyNameForFields(self, fields):
    """Return a Datastore key_name for a Entity from the specified fields.

    Args:
      fields: the fields of the entity that uniquely identifies it
    """

    if not all(fields.values()):
      return None

    return self._keyName(**fields)

  def getForLimitAndOffset(self, limit, offset=0):
    """Returns entities for given offset and limit or None if not found.

    Args:
      limit: max amount of entities to return
      offset: optional number of results to skip first; default zero.
    """

    query = self._model.all()
    return query.fetch(limit, offset)

  def getForFields(self, properties, unique=False, limit=1000, offset=0):
    """Returns all entities that have the specified properties

    Args:
      properties: the properties that the entity should have
      unique: if set, only the first item from the resultset will be returned
      limit: max amount of entities to return
      offset: optional number of results to skip first; default zero.
    """

    if not properties:
      raise Error("Properties did not contain any values")

    format_text = '%(key)s = :%(key)s'
    msg_pairs = [format_text % {'key': key} for key in properties.iterkeys()]
    joined_pairs = ' AND '.join(msg_pairs)
    condition = 'WHERE %s' % joined_pairs

    query = self._model.gql(condition, **properties)

    if unique:
      return query.get()

    return query.fetch(limit, offset)

  def updateModelProperties(self, model, model_properties):
    """Update existing model entity using supplied model properties.

    Args:
      model: a model entity
      model_properties: keyword arguments that correspond to model entity
        properties and their values

    Returns:
      the original model entity with any supplied properties changed
    """

    def update():
      return self._unsafeUpdateModelProperties(model, model_properties)

    return db.run_in_transaction(update)

  def _unsafeUpdateModelProperties(self, model, model_properties):
    """(see updateModelProperties)

    Like updateModelProperties(), but not run within a transaction.
    """

    properties = model.properties()

    for prop in properties.values():
      name = prop.name

      if not name in self._skip_properties and name in model_properties:
        value = model_properties[prop.name]

        if self._updateField(model, name, value):
          prop.__set__(model, value)

    model.put()
    return model

  def updateOrCreateFromKeyName(self, properties, key_name):
    """Update existing entity, or create new one with supplied properties.

    Args:
      properties: dict with entity properties and their values
      key_name: the key_name of the entity that uniquely identifies it

    Returns:
      the entity corresponding to the key_name, with any supplied
      properties changed, or a new entity now associated with the
      supplied key_name and properties.
    """

    entity = self.getFromKeyName(key_name)

    if not entity:
      # entity did not exist, so create one in a transaction
      entity = self._model.get_or_insert(key_name, **properties)

    # there is no way to be sure if get_or_insert() returned a new entity or
    # got an existing one due to a race, so update with properties anyway,
    # in a transaction
    return self.updateModelProperties(entity, properties)

  def updateOrCreateFromFields(self, properties, fields):
    """Like updateOrCreateFromKeyName, but resolves fields to a key_name first.
    """

    # attempt to retrieve the existing entity
    key_name  = self.getKeyNameForFields(fields)

    return self.updateOrCreateFromKeyName(properties, key_name)

  def isDeletable(self, entity):
    """Returns whether the specified entity can be deleted.

    Args:
      entity: an existing entity in datastore
    """

    return True

  def delete(self, entity):
    """Delete existing entity from datastore.

    Args:
      entity: an existing entity in datastore
    """

    entity.delete()
<|endoftext|>


<commit_before>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for displaying views.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import http
from django.utils.translation import ugettext_lazy

import soc.logic
import soc.logic.out_of_band
import soc.views.helper.lists
import soc.views.helper.responses
import soc.views.out_of_band

from soc.logic import dicts
from soc.logic import models
from soc.views import simple
from soc.views import helper
from soc.views.helper import access

class View:
  """Views for entity classes.

  The View class functions specific to Entity classes by relying
  on the the child-classes to define the following fields:

  self._logic: the logic singleton for this entity
  """

  DEF_SUBMIT_MSG_PARAM_NAME = 's'
  DEF_SUBMIT_MSG_PROFILE_SAVED = 0

  DEF_CREATE_NEW_ENTITY_MSG = ugettext_lazy(
      ' You can create a new %(entity_type_lower)s by visiting'
      ' Create '
      'a New %(entity_type)s page.')

  def __init__(self, params=None, rights=None, stop=False):
    """

    Args:
      rights: This dictionary should be filled with the access check
        functions that should be called, it will be modified in-place.
      params: This dictionary should be filled with the parameters
        specific to this entity, required fields are:
        name: the name of the entity (names should have sentence-style caps)
        name_short: the short form name of the name ('org' vs 'organization')
        name_plural: the plural form of the name
        edit_form: the class of the Django form to be used when editing
        create_form: the class of the Django form to be used when creating
        edit_template: the Django template to be used for editing
        public_template: the Django template to be used as public page
        list_template: the Django template to be used as list page
        lists_template: the Django templates to search for the list page
        delete_redirect: the Django template to redirect to on delete
        create_redirect: the Django template to redirect to after creation
        save_message: the message to display when the entity is saved
        edit_params: the params to use when editing
    """

    new_rights = {}
    new_rights['any_access'] = [access.checkIsUser]

    self._rights = dicts.merge(rights, new_rights)
    self._params = params

  def public(self, request, page=None, params=None, **kwargs):
    """Displays the public page for the entity specified by **kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: the Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('public', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    # create default template context for use with any templates
    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    if not all(kwargs.values()):
      #TODO: Change this into a proper redirect
      return http.HttpResponseRedirect('/')

    try:
      key_fields = self._logic.getKeyFieldsFromDict(kwargs)
      entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['public_template']
      return simple.errorResponse(request, page, error, template, context)

    self._public(request, entity, context)

    context['entity'] = entity
    context['entity_type'] = params['name']

    template = params['public_template']

    return helper.responses.respond(request, template, context)

  def create(self, request, page=None, params=None, **kwargs):
    """Displays the create page for this entity type

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: not used for create()
    """

    # Create page is an edit page with no key fields
    kwargs = {}
    fields = self._logic.getKeyFieldNames()
    for field in fields:
      kwargs[field] = None

    request.path = helper.requests.replaceSuffix(request.path,
                                                 old_suffix='create')
    request.path = helper.requests.replaceSuffix(request.path,
                                                 old_suffix='edit',
                                                 new_suffix='edit')

    return self.edit(request, page=page, **kwargs)

  def edit(self, request, page=None, params=None, **kwargs):
    """Displays the public page for the entity specified by **kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: The Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('edit', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    try:
      if all(kwargs.values()):
        key_fields = self._logic.getKeyFieldsFromDict(kwargs)
        entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['public_template']
      error.message = error.message + self.DEF_CREATE_NEW_ENTITY_MSG % {
          'entity_type_lower' : params['name'].lower(),
          'entity_type' : params['name'],
          'create' : params['create_redirect']
          }
      return simple.errorResponse(request, page, error, template, context)

    if request.method == 'POST':
      return self.editPost(request, entity, context, params=params)
    else:
      return self.editGet(request, entity, context, params=params)

  def editPost(self, request, entity, context, params=None):
    """Same as edit, but on POST
    """

    params = dicts.merge(params, self._params)

    if entity:
      form = params['edit_form'](request.POST)
    else:
      form = params['create_form'](request.POST)

    if not form.is_valid():
      return self._constructResponse(request, entity, context, form, params)

    fields = self.collectCleanedFields(form)

    # get the old_suffix before editing
    old_suffix = self._logic.getKeySuffix(entity)

    self._editPost(request, entity, fields)

    key_fields = self._logic.getKeyFieldsFromDict(fields)
    entity = self._logic.updateOrCreateFromFields(fields, key_fields)

    if not entity:
      return http.HttpResponseRedirect('/')

    page_params = params['edit_params']
    suffix = self._logic.getKeySuffix(entity)

    # redirect to (possibly new) location of the entity
    # (causes 'Profile saved' message to be displayed)
    return helper.responses.redirectToChangedSuffix(
        request, old_suffix, suffix,
        params=page_params)

  def editGet(self, request, entity, context, params=None):
    """Same as edit, but on GET
    """

    params = dicts.merge(params, self._params)
    suffix = self._logic.getKeySuffix(entity)

    # Remove the params from the request, this is relevant only if
    # someone bookmarked a POST page.
    is_self_referrer = helper.requests.isReferrerSelf(request, suffix=suffix)
    if request.GET.get(self.DEF_SUBMIT_MSG_PARAM_NAME):
      if (not entity) or (not is_self_referrer):
        return http.HttpResponseRedirect(request.path)

    if entity:
      # Note: no message will be displayed if parameter is not present
      context['notice'] = helper.requests.getSingleIndexedParamValue(
          request, self.DEF_SUBMIT_MSG_PARAM_NAME,
          values=params['save_message'])

      # populate form with the existing entity
      form = params['edit_form'](instance=entity)
      self._editGet(request, entity, form)
    else:
      form = params['create_form']()

    return self._constructResponse(request, entity, context, form, params)

  def list(self, request, page=None, params=None):
    """Displays the list page for the entity type

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('list', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    context = helper.responses.getUniversalContext(request)
    context['page'] = page

    offset, limit = helper.lists.cleanListParameters(
      offset=request.GET.get('offset'), limit=request.GET.get('limit'))

    # Fetch one more to see if there should be a 'next' link
    entities = self._logic.getForLimitAndOffset(limit + 1, offset=offset)

    context['pagination_form'] = helper.lists.makePaginationForm(request, limit)

    templates = params['lists_template']

    context = helper.lists.setList(request, context, entities,
                                 offset, limit, templates)

    context['entity_type'] = params['name']
    context['entity_type_plural'] = params['name_plural']

    template = params['list_template']

    return helper.responses.respond(request, template, context)

  def delete(self, request, page=None, params=None, **kwargs):
    """Shows the delete page for the entity specified by kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: The Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('delete', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    # create default template context for use with any templates
    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    try:
      key_fields = self._logic.getKeyFieldsFromDict(kwargs)
      entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['edit_template']
      error.message = error.message + self.DEF_CREATE_NEW_ENTITY_MSG % {
          'entity_type_lower' : params['name'].lower(),
          'entity_type' : params['name'],
          'create' : params['create_redirect']
          }
      return simple.errorResponse(request, page, error, template, context)

    if not entity:
      #TODO: Create a proper error page for this
      return http.HttpResponseRedirect('/')

    if not self._logic.isDeletable(entity):
      # TODO: Update the notice area telling the user that they
      # can't delete the entity
      pass

    self._logic.delete(entity)
    redirect = params['delete_redirect']

    return http.HttpResponseRedirect(redirect)

  def _editPost(self, request, entity, fields):
    """Performs any required processing on the entity to post its edit page

    Args:
      request: the django request object
      entity:  the entity to create or update from POST contents
      fields: the new field values
    """

    pass

  def _public(self, request, entity, context):
    """Performs any required processing to get an entities public page

    Args:
      request: the django request object
      entity: the entity to make public
      context: the context object
    """

    pass

  def _editGet(self, request, entity, form):
    """Performs any required processing on the form to get its edit page

    Args:
      request: the django request object
      entity: the entity to get
      form: the django form that will be used for the page
    """

    pass

  def checkUnspecified(self, access_type, request):
    """Checks whether an unspecified access_type should be allowed

    Args:
      access_type: the access type (such as 'list' or 'edit') that was
                   not present in the _rights dictionary when checking.
    """

    pass

  def _constructResponse(self, request, entity, context, form, params):
    """Updates the context and returns a response for the specified arguments

    Args:
      request: the django request object
      entity: the entity that is used
      context: the context to be used
      form: the form that will be used
    """

    suffix = self._logic.getKeySuffix(entity)

    context['form'] = form
    context['entity'] = entity
    context['entity_suffix'] = suffix
    context['entity_type'] = params['name']
    context['entity_type_plural'] = params['name_plural']
    context['entity_type_short'] = params['name_short']

    template = params['edit_template']

    return helper.responses.respond(request, template, context)

  def checkAccess(self, access_type, request):
    """Runs all the defined checks for the specified type

    Args:
      access_type: the type of request (such as 'list' or 'edit')
      request: the Django request object

    Returns:
      True: If all the required access checks have been made successfully
      False: If a check failed, in this case self._response will contain
             the response provided by the failed access check.
    """

    # Call each access checker
    for check in self._rights['any_access']:
      check(request)

    if access_type not in self._rights:
       # No checks defined, so do the 'generic check' and bail out
      self.checkUnspecified(access_type, request)
      return

    for check in self._rights[access_type]:
      check(request)

  def collectCleanedFields(self, form):
    """Collects all cleaned fields from form and returns them

    Args:
      form: The form from which the cleaned fields should be collected
    """

    fields = {}

    for field, value in form.cleaned_data.iteritems():
      fields[field] = value

    return fields
<commit_msg>Added support for providing a custom params dict<commit_after>#!/usr/bin/python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Helpers functions for displaying views.
"""

__authors__ = [
  '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  ]

from django import http
from django.utils.translation import ugettext_lazy

import soc.logic
import soc.logic.out_of_band
import soc.views.helper.lists
import soc.views.helper.responses
import soc.views.out_of_band

from soc.logic import dicts
from soc.logic import models
from soc.views import simple
from soc.views import helper
from soc.views.helper import access

class View:
  """Views for entity classes.

  The View class functions specific to Entity classes by relying
  on the the child-classes to define the following fields:

  self._logic: the logic singleton for this entity
  """

  DEF_SUBMIT_MSG_PARAM_NAME = 's'
  DEF_SUBMIT_MSG_PROFILE_SAVED = 0

  DEF_CREATE_NEW_ENTITY_MSG = ugettext_lazy(
      ' You can create a new %(entity_type_lower)s by visiting'
      ' Create '
      'a New %(entity_type)s page.')

  def __init__(self, params=None, rights=None, stop=False):
    """

    Args:
      rights: This dictionary should be filled with the access check
        functions that should be called, it will be modified in-place.
      params: This dictionary should be filled with the parameters
        specific to this entity, required fields are:
        name: the name of the entity (names should have sentence-style caps)
        name_short: the short form name of the name ('org' vs 'organization')
        name_plural: the plural form of the name
        edit_form: the class of the Django form to be used when editing
        create_form: the class of the Django form to be used when creating
        edit_template: the Django template to be used for editing
        public_template: the Django template to be used as public page
        list_template: the Django template to be used as list page
        lists_template: the Django templates to search for the list page
        delete_redirect: the Django template to redirect to on delete
        create_redirect: the Django template to redirect to after creation
        save_message: the message to display when the entity is saved
        edit_params: the params to use when editing
    """

    new_rights = {}
    new_rights['any_access'] = [access.checkIsUser]

    self._rights = dicts.merge(rights, new_rights)
    self._params = params

  def public(self, request, page=None, params=None, **kwargs):
    """Displays the public page for the entity specified by **kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: the Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('public', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    # create default template context for use with any templates
    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    if not all(kwargs.values()):
      #TODO: Change this into a proper redirect
      return http.HttpResponseRedirect('/')

    try:
      key_fields = self._logic.getKeyFieldsFromDict(kwargs)
      entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['public_template']
      return simple.errorResponse(request, page, error, template, context)

    self._public(request, entity, context)

    context['entity'] = entity
    context['entity_type'] = params['name']

    template = params['public_template']

    return helper.responses.respond(request, template, context)

  def create(self, request, page=None, params=None, **kwargs):
    """Displays the create page for this entity type

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: not used for create()
    """

    # Create page is an edit page with no key fields
    kwargs = {}
    fields = self._logic.getKeyFieldNames()
    for field in fields:
      kwargs[field] = None

    request.path = helper.requests.replaceSuffix(request.path,
                                                 old_suffix='create')
    request.path = helper.requests.replaceSuffix(request.path,
                                                 old_suffix='edit',
                                                 new_suffix='edit')

    return self.edit(request, page=page, params=params, **kwargs)

  def edit(self, request, page=None, params=None, **kwargs):
    """Displays the public page for the entity specified by **kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: The Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('edit', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    try:
      if all(kwargs.values()):
        key_fields = self._logic.getKeyFieldsFromDict(kwargs)
        entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['public_template']
      error.message = error.message + self.DEF_CREATE_NEW_ENTITY_MSG % {
          'entity_type_lower' : params['name'].lower(),
          'entity_type' : params['name'],
          'create' : params['create_redirect']
          }
      return simple.errorResponse(request, page, error, template, context)

    if request.method == 'POST':
      return self.editPost(request, entity, context, params=params)
    else:
      return self.editGet(request, entity, context, params=params)

  def editPost(self, request, entity, context, params=None):
    """Same as edit, but on POST
    """

    params = dicts.merge(params, self._params)

    if entity:
      form = params['edit_form'](request.POST)
    else:
      form = params['create_form'](request.POST)

    if not form.is_valid():
      return self._constructResponse(request, entity, context, form, params)

    fields = self.collectCleanedFields(form)

    # get the old_suffix before editing
    old_suffix = self._logic.getKeySuffix(entity)

    self._editPost(request, entity, fields)

    key_fields = self._logic.getKeyFieldsFromDict(fields)
    entity = self._logic.updateOrCreateFromFields(fields, key_fields)

    if not entity:
      return http.HttpResponseRedirect('/')

    page_params = params['edit_params']
    suffix = self._logic.getKeySuffix(entity)

    # redirect to (possibly new) location of the entity
    # (causes 'Profile saved' message to be displayed)
    return helper.responses.redirectToChangedSuffix(
        request, old_suffix, suffix,
        params=page_params)

  def editGet(self, request, entity, context, params=None):
    """Same as edit, but on GET
    """

    params = dicts.merge(params, self._params)
    suffix = self._logic.getKeySuffix(entity)

    # Remove the params from the request, this is relevant only if
    # someone bookmarked a POST page.
    is_self_referrer = helper.requests.isReferrerSelf(request, suffix=suffix)
    if request.GET.get(self.DEF_SUBMIT_MSG_PARAM_NAME):
      if (not entity) or (not is_self_referrer):
        return http.HttpResponseRedirect(request.path)

    if entity:
      # Note: no message will be displayed if parameter is not present
      context['notice'] = helper.requests.getSingleIndexedParamValue(
          request, self.DEF_SUBMIT_MSG_PARAM_NAME,
          values=params['save_message'])

      # populate form with the existing entity
      form = params['edit_form'](instance=entity)
      self._editGet(request, entity, form)
    else:
      form = params['create_form']()

    return self._constructResponse(request, entity, context, form, params)

  def list(self, request, page=None, params=None):
    """Displays the list page for the entity type

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('list', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    context = helper.responses.getUniversalContext(request)
    context['page'] = page

    offset, limit = helper.lists.cleanListParameters(
      offset=request.GET.get('offset'), limit=request.GET.get('limit'))

    # Fetch one more to see if there should be a 'next' link
    entities = self._logic.getForLimitAndOffset(limit + 1, offset=offset)

    context['pagination_form'] = helper.lists.makePaginationForm(request, limit)

    templates = params['lists_template']

    context = helper.lists.setList(request, context, entities,
                                 offset, limit, templates)

    context['entity_type'] = params['name']
    context['entity_type_plural'] = params['name_plural']

    template = params['list_template']

    return helper.responses.respond(request, template, context)

  def delete(self, request, page=None, params=None, **kwargs):
    """Shows the delete page for the entity specified by kwargs

    Args:
      request: the standard Django HTTP request object
      page: a soc.logic.site.page.Page object which is abstraction
        that combines a Django view with sidebar menu info
      kwargs: The Key Fields for the specified entity
    """

    params = dicts.merge(params, self._params)

    try:
      self.checkAccess('delete', request)
    except soc.views.out_of_band.AccessViolationResponse, alt_response:
      return alt_response.response()

    # create default template context for use with any templates
    context = helper.responses.getUniversalContext(request)
    context['page'] = page
    entity = None

    try:
      key_fields = self._logic.getKeyFieldsFromDict(kwargs)
      entity = self._logic.getIfFields(key_fields)
    except soc.logic.out_of_band.ErrorResponse, error:
      template = params['edit_template']
      error.message = error.message + self.DEF_CREATE_NEW_ENTITY_MSG % {
          'entity_type_lower' : params['name'].lower(),
          'entity_type' : params['name'],
          'create' : params['create_redirect']
          }
      return simple.errorResponse(request, page, error, template, context)

    if not entity:
      #TODO: Create a proper error page for this
      return http.HttpResponseRedirect('/')

    if not self._logic.isDeletable(entity):
      # TODO: Update the notice area telling the user that they
      # can't delete the entity
      pass

    self._logic.delete(entity)
    redirect = params['delete_redirect']

    return http.HttpResponseRedirect(redirect)

  def _editPost(self, request, entity, fields):
    """Performs any required processing on the entity to post its edit page

    Args:
      request: the django request object
      entity:  the entity to create or update from POST contents
      fields: the new field values
    """

    pass

  def _public(self, request, entity, context):
    """Performs any required processing to get an entities public page

    Args:
      request: the django request object
      entity: the entity to make public
      context: the context object
    """

    pass

  def _editGet(self, request, entity, form):
    """Performs any required processing on the form to get its edit page

    Args:
      request: the django request object
      entity: the entity to get
      form: the django form that will be used for the page
    """

    pass

  def checkUnspecified(self, access_type, request):
    """Checks whether an unspecified access_type should be allowed

    Args:
      access_type: the access type (such as 'list' or 'edit') that was
                   not present in the _rights dictionary when checking.
    """

    pass

  def _constructResponse(self, request, entity, context, form, params):
    """Updates the context and returns a response for the specified arguments

    Args:
      request: the django request object
      entity: the entity that is used
      context: the context to be used
      form: the form that will be used
    """

    suffix = self._logic.getKeySuffix(entity)

    context['form'] = form
    context['entity'] = entity
    context['entity_suffix'] = suffix
    context['entity_type'] = params['name']
    context['entity_type_plural'] = params['name_plural']
    context['entity_type_short'] = params['name_short']

    template = params['edit_template']

    return helper.responses.respond(request, template, context)

  def checkAccess(self, access_type, request):
    """Runs all the defined checks for the specified type

    Args:
      access_type: the type of request (such as 'list' or 'edit')
      request: the Django request object

    Returns:
      True: If all the required access checks have been made successfully
      False: If a check failed, in this case self._response will contain
             the response provided by the failed access check.
    """

    # Call each access checker
    for check in self._rights['any_access']:
      check(request)

    if access_type not in self._rights:
       # No checks defined, so do the 'generic check' and bail out
      self.checkUnspecified(access_type, request)
      return

    for check in self._rights[access_type]:
      check(request)

  def collectCleanedFields(self, form):
    """Collects all cleaned fields from form and returns them

    Args:
      form: The form from which the cleaned fields should be collected
    """

    fields = {}

    for field, value in form.cleaned_data.iteritems():
      fields[field] = value

    return fields
<|endoftext|>