<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
sys.path.append(os.path.dirname(__file__))
from scriptutil import *

import argparse
import re
from configparser import ConfigParser, ExtendedInterpolation
from textwrap import dedent

def update_changes(filename, new_version, init_changes = '(No Changes)\n\n'):
  print('  adding new section to %s...' % filename, end='', flush=True)
  matcher = re.compile(r'\d+\.\d+\.\d+\s+===')
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    match = new_version.previous_dot_matcher.search(line)
    if match is not None:
      buffer.append(line.replace(match.group(0), new_version.dot))
      buffer.append(init_changes)
    buffer.append(line)
    return match is not None

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def add_constant(new_version, deprecate):
  filename = 'lucene/core/src/java/org/apache/lucene/util/Version.java'
  print('  adding constant %s...' % new_version.constant, end='', flush=True)
  constant_prefix = 'public static final Version LUCENE_'
  matcher = re.compile(constant_prefix)
  prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep='_')

  def ensure_deprecated(buffer):
    last = buffer[-1]
    if last.strip() != '@Deprecated':
      spaces = ' ' * (len(last) - len(last.lstrip()) - 1)
      del buffer[-1] # Remove comment closer line
      if (len(buffer) >= 4 and re.search('for Lucene.\s*$', buffer[-1]) != None):
        del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
      buffer.append(( '{0} * @deprecated ({1}) Use latest\n'
                    + '{0} */\n'
                    + '{0}@Deprecated\n').format(spaces, new_version))

  def buffer_constant(buffer, line):
    spaces = ' ' * (len(line) - len(line.lstrip()))
    buffer.append(( '\n{0}/**\n'
                  + '{0} * Match settings and bugs in Lucene\'s {1} release.\n')
                  .format(spaces, new_version))
    if deprecate:
      buffer.append('%s * @deprecated Use latest\n' % spaces)
    else:
      buffer.append(( '{0} * <p>\n'
                    + '{0} * Use this to get the latest & greatest settings, bug\n'
                    + '{0} * fixes, etc, for Lucene.\n').format(spaces))
    buffer.append('%s */\n' % spaces)
    if deprecate:
      buffer.append('%s@Deprecated\n' % spaces)
    buffer.append('{0}public static final Version {1} = new Version({2}, {3}, {4});\n'.format
                  (spaces, new_version.constant, new_version.major, new_version.minor, new_version.bugfix))

  class Edit(object):
    found = -1
    def __call__(self, buffer, match, line):
      if new_version.constant in line:
        return None # constant already exists
      # outer match is just to find lines declaring version constants
      match = prev_matcher.search(line)
      if match is not None:
        ensure_deprecated(buffer) # old version should be deprecated
        self.found = len(buffer) + 1 # extra 1 for buffering current line below
      elif self.found != -1:
        # we didn't match, but we previously had a match, so insert new version here
        # first find where to insert (first empty line before current constant)
        c = []
        buffer_constant(c, line)
        tmp = buffer[self.found:]
        buffer[self.found:] = c
        buffer.extend(tmp)
        buffer.append(line)
        return True

      buffer.append(line)
      return False

  changed = update_file(filename, matcher, Edit())
  print('done' if changed else 'uptodate')

version_prop_re = re.compile('version\.base=(.*)')
def update_build_version(new_version):
  print('  changing version.base...', end='', flush=True)
  filename = 'lucene/version.properties'
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    buffer.append('version.base=' + new_version.dot + '\n')
    return True

  changed = update_file(filename, version_prop_re, edit)
  print('done' if changed else 'uptodate')

def update_latest_constant(new_version):
  print('  changing Version.LATEST to %s...' % new_version.constant, end='', flush=True)
  filename = 'lucene/core/src/java/org/apache/lucene/util/Version.java'
  matcher = re.compile('public static final Version LATEST')
  def edit(buffer, match, line):
    if new_version.constant in line:
      return None
    buffer.append(line.rpartition('=')[0] + ('= %s;\n' % new_version.constant))
    return True

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def onerror(x):
  raise x

def update_example_solrconfigs(new_version):
  print('  updating example solrconfig.xml files')
  matcher = re.compile('<luceneMatchVersion>')

  paths = ['solr/server/solr/configsets', 'solr/example', 'solr/core/src/test-files/solr/configsets/_default']
  for path in paths:
    if not os.path.isdir(path):
      raise RuntimeError("Can't locate configset dir (layout change?) : " + path)
    for root,dirs,files in os.walk(path, onerror=onerror):
      for f in files:
        if f == 'solrconfig.xml':
          update_solrconfig(os.path.join(root, f), matcher, new_version)

def update_solrconfig(filename, matcher, new_version):
  print('    %s...' % filename, end='', flush=True)
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    match = new_version.previous_dot_matcher.search(line)
    if match is None:
      return False
    buffer.append(line.replace(match.group(1), new_version.dot))
    return True

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def check_lucene_version_tests():
  print('  checking lucene version tests...', end='', flush=True)
  base_dir = os.getcwd()
  os.chdir('lucene/core')
  run('ant test -Dtestcase=TestVersion')
  os.chdir(base_dir)
  print('ok')

def check_solr_version_tests():
  print('  checking solr version tests...', end='', flush=True)
  base_dir = os.getcwd()
  os.chdir('solr/core')
  run('ant test -Dtestcase=TestLuceneMatchVersion')
  os.chdir(base_dir)
  print('ok')

def read_config(current_version):
  parser = argparse.ArgumentParser(description='Add a new version to CHANGES, to Version.java, lucene/version.properties and solrconfig.xml files')
  parser.add_argument('version', type=Version.parse)
  newconf = parser.parse_args()

  newconf.branch_type = find_branch_type()
  newconf.is_latest_version = newconf.version.on_or_after(current_version)

  print ("branch_type is %s " % newconf.branch_type)

  return newconf

# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files
def parse_properties_file(filename):
  contents = open(filename, encoding='ISO-8859-1').read().replace('%', '%%') # Escape interpolation metachar
  parser = ConfigParser(interpolation=ExtendedInterpolation())               # Handle ${property-name} interpolation
  parser.read_string("[DUMMY_SECTION]\n" + contents)                         # Add required section
  return dict(parser.items('DUMMY_SECTION'))

def get_solr_init_changes():
  return dedent('''
    Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

    Versions of Major Components
    ---------------------
    Apache Tika %(org.apache.tika.version)s
    Carrot2 %(/org.carrot2/carrot2-mini)s
    Velocity %(/org.apache.velocity/velocity-engine-core)s and Velocity Tools %(org.apache.velocity.tools.version)s
    Apache ZooKeeper %(/org.apache.zookeeper/zookeeper)s
    Jetty %(org.eclipse.jetty.version)s

    (No Changes)\n\n
    ''' % parse_properties_file('lucene/ivy-versions.properties'))

def main():
  if not os.path.exists('lucene/version.properties'):
    sys.exit("Tool must be run from the root of a source checkout.")
  current_version = Version.parse(find_current_version())
  newconf = read_config(current_version)

  print('\nAdding new version %s' % newconf.version)
  update_changes('lucene/CHANGES.txt', newconf.version)
  update_changes('solr/CHANGES.txt', newconf.version, get_solr_init_changes())

  latest_or_backcompat = newconf.is_latest_version or current_version.is_back_compat_with(newconf.version)
  if latest_or_backcompat:
    add_constant(newconf.version, not newconf.is_latest_version)
  else:
    print('\nNot adding constant for version %s because it is no longer supported' % newconf.version)

  if newconf.is_latest_version:
    print('\nUpdating latest version')
    update_build_version(newconf.version)
    update_latest_constant(newconf.version)
    update_example_solrconfigs(newconf.version)

  if newconf.version.is_major_release():
    print('\nTODO: ')
    print('  - Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility')
    print('  - Update IndexFormatTooOldException throw cases')
  elif latest_or_backcompat:
    print('\nTesting changes')
    check_lucene_version_tests()
    check_solr_version_tests()

  print()

if __name__ == '__main__':
  try:
    main()
  except KeyboardInterrupt:
    print('\nReceived Ctrl-C, exiting early')
<commit_msg>LUCENE-8883: addVersion.py now adds categories to CHANGES.txt<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import sys
sys.path.append(os.path.dirname(__file__))
from scriptutil import *

import argparse
import re
from configparser import ConfigParser, ExtendedInterpolation
from textwrap import dedent

def update_changes(filename, new_version, init_changes, headers):
  print('  adding new section to %s...' % filename, end='', flush=True)
  matcher = re.compile(r'\d+\.\d+\.\d+\s+===')
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    match = new_version.previous_dot_matcher.search(line)
    if match is not None:
      buffer.append(line.replace(match.group(0), new_version.dot))
      buffer.append(init_changes)
      for header in headers:
        buffer.append('%s\n---------------------\n(No changes)\n\n' % header)
    buffer.append(line)
    return match is not None

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def add_constant(new_version, deprecate):
  filename = 'lucene/core/src/java/org/apache/lucene/util/Version.java'
  print('  adding constant %s...' % new_version.constant, end='', flush=True)
  constant_prefix = 'public static final Version LUCENE_'
  matcher = re.compile(constant_prefix)
  prev_matcher = new_version.make_previous_matcher(prefix=constant_prefix, sep='_')

  def ensure_deprecated(buffer):
    last = buffer[-1]
    if last.strip() != '@Deprecated':
      spaces = ' ' * (len(last) - len(last.lstrip()) - 1)
      del buffer[-1] # Remove comment closer line
      if (len(buffer) >= 4 and re.search('for Lucene.\s*$', buffer[-1]) != None):
        del buffer[-3:] # drop the trailing lines '<p> / Use this to get the latest ... / ... for Lucene.'
      buffer.append(( '{0} * @deprecated ({1}) Use latest\n'
                    + '{0} */\n'
                    + '{0}@Deprecated\n').format(spaces, new_version))

  def buffer_constant(buffer, line):
    spaces = ' ' * (len(line) - len(line.lstrip()))
    buffer.append(( '\n{0}/**\n'
                  + '{0} * Match settings and bugs in Lucene\'s {1} release.\n')
                  .format(spaces, new_version))
    if deprecate:
      buffer.append('%s * @deprecated Use latest\n' % spaces)
    else:
      buffer.append(( '{0} * <p>\n'
                    + '{0} * Use this to get the latest & greatest settings, bug\n'
                    + '{0} * fixes, etc, for Lucene.\n').format(spaces))
    buffer.append('%s */\n' % spaces)
    if deprecate:
      buffer.append('%s@Deprecated\n' % spaces)
    buffer.append('{0}public static final Version {1} = new Version({2}, {3}, {4});\n'.format
                  (spaces, new_version.constant, new_version.major, new_version.minor, new_version.bugfix))

  class Edit(object):
    found = -1
    def __call__(self, buffer, match, line):
      if new_version.constant in line:
        return None # constant already exists
      # outer match is just to find lines declaring version constants
      match = prev_matcher.search(line)
      if match is not None:
        ensure_deprecated(buffer) # old version should be deprecated
        self.found = len(buffer) + 1 # extra 1 for buffering current line below
      elif self.found != -1:
        # we didn't match, but we previously had a match, so insert new version here
        # first find where to insert (first empty line before current constant)
        c = []
        buffer_constant(c, line)
        tmp = buffer[self.found:]
        buffer[self.found:] = c
        buffer.extend(tmp)
        buffer.append(line)
        return True

      buffer.append(line)
      return False

  changed = update_file(filename, matcher, Edit())
  print('done' if changed else 'uptodate')

version_prop_re = re.compile('version\.base=(.*)')
def update_build_version(new_version):
  print('  changing version.base...', end='', flush=True)
  filename = 'lucene/version.properties'
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    buffer.append('version.base=' + new_version.dot + '\n')
    return True

  changed = update_file(filename, version_prop_re, edit)
  print('done' if changed else 'uptodate')

def update_latest_constant(new_version):
  print('  changing Version.LATEST to %s...' % new_version.constant, end='', flush=True)
  filename = 'lucene/core/src/java/org/apache/lucene/util/Version.java'
  matcher = re.compile('public static final Version LATEST')
  def edit(buffer, match, line):
    if new_version.constant in line:
      return None
    buffer.append(line.rpartition('=')[0] + ('= %s;\n' % new_version.constant))
    return True

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def onerror(x):
  raise x

def update_example_solrconfigs(new_version):
  print('  updating example solrconfig.xml files')
  matcher = re.compile('<luceneMatchVersion>')

  paths = ['solr/server/solr/configsets', 'solr/example', 'solr/core/src/test-files/solr/configsets/_default']
  for path in paths:
    if not os.path.isdir(path):
      raise RuntimeError("Can't locate configset dir (layout change?) : " + path)
    for root,dirs,files in os.walk(path, onerror=onerror):
      for f in files:
        if f == 'solrconfig.xml':
          update_solrconfig(os.path.join(root, f), matcher, new_version)

def update_solrconfig(filename, matcher, new_version):
  print('    %s...' % filename, end='', flush=True)
  def edit(buffer, match, line):
    if new_version.dot in line:
      return None
    match = new_version.previous_dot_matcher.search(line)
    if match is None:
      return False
    buffer.append(line.replace(match.group(1), new_version.dot))
    return True

  changed = update_file(filename, matcher, edit)
  print('done' if changed else 'uptodate')

def check_lucene_version_tests():
  print('  checking lucene version tests...', end='', flush=True)
  base_dir = os.getcwd()
  os.chdir('lucene/core')
  run('ant test -Dtestcase=TestVersion')
  os.chdir(base_dir)
  print('ok')

def check_solr_version_tests():
  print('  checking solr version tests...', end='', flush=True)
  base_dir = os.getcwd()
  os.chdir('solr/core')
  run('ant test -Dtestcase=TestLuceneMatchVersion')
  os.chdir(base_dir)
  print('ok')

def read_config(current_version):
  parser = argparse.ArgumentParser(description='Add a new version to CHANGES, to Version.java, lucene/version.properties and solrconfig.xml files')
  parser.add_argument('version', type=Version.parse)
  newconf = parser.parse_args()

  newconf.branch_type = find_branch_type()
  newconf.is_latest_version = newconf.version.on_or_after(current_version)

  print ("branch_type is %s " % newconf.branch_type)

  return newconf

# Hack ConfigParser, designed to parse INI files, to parse & interpolate Java .properties files
def parse_properties_file(filename):
  contents = open(filename, encoding='ISO-8859-1').read().replace('%', '%%') # Escape interpolation metachar
  parser = ConfigParser(interpolation=ExtendedInterpolation())               # Handle ${property-name} interpolation
  parser.read_string("[DUMMY_SECTION]\n" + contents)                         # Add required section
  return dict(parser.items('DUMMY_SECTION'))

def get_solr_init_changes():
  return dedent('''
    Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

    Versions of Major Components
    ---------------------
    Apache Tika %(org.apache.tika.version)s
    Carrot2 %(/org.carrot2/carrot2-mini)s
    Velocity %(/org.apache.velocity/velocity-engine-core)s and Velocity Tools %(org.apache.velocity.tools.version)s
    Apache ZooKeeper %(/org.apache.zookeeper/zookeeper)s
    Jetty %(org.eclipse.jetty.version)s

    ''' % parse_properties_file('lucene/ivy-versions.properties'))

def main():
  if not os.path.exists('lucene/version.properties'):
    sys.exit("Tool must be run from the root of a source checkout.")
  current_version = Version.parse(find_current_version())
  newconf = read_config(current_version)
  is_bugfix = newconf.version.is_bugfix_release()

  print('\nAdding new version %s' % newconf.version)
  # See LUCENE-8883 for some thoughts on which categories to use
  update_changes('lucene/CHANGES.txt', newconf.version, '\n',
                 ['Bug Fixes'] if is_bugfix else ['API Changes', 'New Features', 'Improvements', 'Optimizations', 'Bug Fixes', 'Other'])
  update_changes('solr/CHANGES.txt', newconf.version, get_solr_init_changes(),
                 ['Bug Fixes'] if is_bugfix else ['Upgrade Notes', 'New Features', 'Improvements', 'Optimizations', 'Bug Fixes', 'Other Changes'])

  latest_or_backcompat = newconf.is_latest_version or current_version.is_back_compat_with(newconf.version)
  if latest_or_backcompat:
    add_constant(newconf.version, not newconf.is_latest_version)
  else:
    print('\nNot adding constant for version %s because it is no longer supported' % newconf.version)

  if newconf.is_latest_version:
    print('\nUpdating latest version')
    update_build_version(newconf.version)
    update_latest_constant(newconf.version)
    update_example_solrconfigs(newconf.version)

  if newconf.version.is_major_release():
    print('\nTODO: ')
    print('  - Move backcompat oldIndexes to unsupportedIndexes in TestBackwardsCompatibility')
    print('  - Update IndexFormatTooOldException throw cases')
  elif latest_or_backcompat:
    print('\nTesting changes')
    check_lucene_version_tests()
    check_solr_version_tests()

  print()

if __name__ == '__main__':
  try:
    main()
  except KeyboardInterrupt:
    print('\nReceived Ctrl-C, exiting early')
<|endoftext|>


<commit_before>from collections import defaultdict

from marshmallow import fields
from marshmallow.utils import missing

from .utils import resolve_obj_key

class SeparatedStr(fields.List):
    """
    Used for loading "value1,value2" strings. Works like marshmallow.fields.List,
    so you may pass another field in init (defaults to marshmallow.fields.String).
    """

    def __init__(self, cls_or_instance=None, separator=',', **kwargs):
        self.separator = separator
        super().__init__(cls_or_instance or fields.Str, **kwargs)

    def _serialize(self, value, attr, obj):
        value = self.separator.join(map(str, value))
        return super()._deserialize(value, attr, obj)

    def _deserialize(self, value, attr, data):
        value = tuple(filter(None, value.split(self.separator)))
        return super()._deserialize(value, attr, data)

class Nested(fields.Nested):
    # NOTE: Fix to fallback schema load to schema defaults
    # instead of overriding unknown='raise'.
    # While not merged https://github.com/marshmallow-code/marshmallow/pull/963

    def __init__(self, nested, **kwargs):
        kwargs.setdefault('unknown', None)
        super().__init__(nested, **kwargs)

class NestedLazy(Nested):
    """
    Nested schema with difference, that it may not be cached on field and
    initialized from callback.
    Usage: some_field = NestedLazy(lambda field: schema_cls_or_instance)
    """
    def __init__(self, nested_callable, *args, **kwargs):
        # if not callable(nested_callable):
        #     raise ValueError('nested_schema must be callable')
        self.nested_callable = nested_callable
        self.nested_cache = kwargs.pop('cache', False)
        super().__init__(None, *args, **kwargs)

    @property
    def schema(self):
        if hasattr(self, '_Nested__schema'):
            if self.nested_cache:
                return self._Nested__schema
            else:
                self._Nested__schema = None
        self.nested = self.nested_callable(self)
        return super().schema

class NestedFromValue(NestedLazy):
    """
    Nested schema that is based on value of other parent schema field.
    Usage:
    class MySchema(ma.Schema):
        TYPE_DATA_SCHEMA_MAP = {
            'type1': Type1DataSchema,
            'type2': Type2DataSchema,
        }
        type = ma.fields.Str()
        data = NestedFromValue('type', TYPE_DATA_SCHEMA_MAP)
    """
    def __init__(self, key_or_getter, schema_map, *args, **kwargs):
        if kwargs.get('many'):
            raise NotImplementedError()
        if callable(key_or_getter):
            self._getter = key_or_getter
        else:
            self._getter = lambda obj: resolve_obj_key(obj, key_or_getter)
        self._value = missing
        self.schema_map = schema_map
        super().__init__(self._get_schema_from_value, cache=False, **kwargs)

    def _get_schema_from_value(self, field):
        # NOTE: self and field is different instances, because self is not binded,
        # and field is binded (marshmallow do it using deepcopy)
        try:
            return self.schema_map[field._value]
        except KeyError:
            raise ValueError('Unknown schema for {}'.format(field._value))

    def _pre_load(self, obj):
        self._value = self._getter(obj)
        return obj

    _pre_load.__marshmallow_kwargs__ = _pre_load.__marshmallow_hook__ = defaultdict(dict)
    _pre_dump = _pre_load

    # Actually we don't need post processors, but just for sanity reasons
    def _post_load(self, obj):
        self._value = missing
        return obj

    _post_load.__marshmallow_kwargs__ = _post_load.__marshmallow_hook__ = defaultdict(dict)
    _post_dump = _post_load

    def _add_to_schema(self, field_name, parent):
        super()._add_to_schema(field_name, parent)

        # Binding tag processors to parent schema,
        # tricky shit because of marshmallow design...
        # We're setting processors for instance, not for class!
        for tag in ('pre_load', 'pre_dump', 'post_load', 'post_dump'):
            attr = '_{}_{}_schema'.format(tag, field_name)
            if hasattr(parent, attr):
                # Is fields binded on each load? wtf?
                continue
            setattr(parent, attr, getattr(self, '_{}'.format(tag)))
            if hasattr(parent, '__processors__'):
                # Older versions for 2.x and possible some beta versions
                hooks = parent.__processors__ = parent.__processors__.copy()
            else:
                # New version (at least since 3.0.0.b12)
                hooks = parent._hooks = parent._hooks.copy()
            # (tag, many?) - for now only for many=False just not to test other case,
            # there may be issues with many=True
            hooks[(tag, False)].append(attr)
<commit_msg>fixes for new marshmallow<commit_after>from collections import defaultdict

from marshmallow import fields
from marshmallow.utils import missing

from .utils import resolve_obj_key

class SeparatedStr(fields.List):
    """
    Used for loading "value1,value2" strings. Works like marshmallow.fields.List,
    so you may pass another field in init (defaults to marshmallow.fields.String).
    """

    def __init__(self, cls_or_instance=None, separator=',', **kwargs):
        self.separator = separator
        super().__init__(cls_or_instance or fields.Str, **kwargs)

    def _serialize(self, value, attr, obj):
        value = self.separator.join(map(str, value))
        return super()._deserialize(value, attr, obj)

    def _deserialize(self, value, attr, data):
        value = tuple(filter(None, value.split(self.separator)))
        return super()._deserialize(value, attr, data)

class Nested(fields.Nested):
    # NOTE: Fix to fallback schema load to schema defaults
    # instead of overriding unknown='raise'.
    # While not merged https://github.com/marshmallow-code/marshmallow/pull/963

    def __init__(self, nested, **kwargs):
        kwargs.setdefault('unknown', None)
        super().__init__(nested, **kwargs)

class NestedLazy(Nested):
    """
    Nested schema with difference, that it may not be cached on field and
    initialized from callback.
    Usage: some_field = NestedLazy(lambda field: schema_cls_or_instance)
    """
    def __init__(self, nested_callable, *args, **kwargs):
        # if not callable(nested_callable):
        #     raise ValueError('nested_schema must be callable')
        self.nested_callable = nested_callable
        self.nested_cache = kwargs.pop('cache', False)
        super().__init__(None, *args, **kwargs)

    @property
    def schema(self):
        if hasattr(self, '_Nested__schema'):
            if self.nested_cache:
                return self._Nested__schema
            else:
                self._Nested__schema = None
        self.nested = self.nested_callable(self)
        return super().schema

class NestedFromValue(NestedLazy):
    """
    Nested schema that is based on value of other parent schema field.
    Usage:
    class MySchema(ma.Schema):
        TYPE_DATA_SCHEMA_MAP = {
            'type1': Type1DataSchema,
            'type2': Type2DataSchema,
        }
        type = ma.fields.Str()
        data = NestedFromValue('type', TYPE_DATA_SCHEMA_MAP)
    """
    def __init__(self, key_or_getter, schema_map, *args, **kwargs):
        if kwargs.get('many'):
            raise NotImplementedError()
        if callable(key_or_getter):
            self._getter = key_or_getter
        else:
            self._getter = lambda obj: resolve_obj_key(obj, key_or_getter)
        self._value = missing
        self.schema_map = schema_map
        super().__init__(self._get_schema_from_value, cache=False, **kwargs)

    def _get_schema_from_value(self, field):
        # NOTE: self and field is different instances, because self is not binded,
        # and field is binded (marshmallow do it using deepcopy)
        try:
            return self.schema_map[field._value]
        except KeyError:
            raise ValueError('Unknown schema for {}'.format(field._value))

    def _pre_load(self, obj):
        self._value = self._getter(obj)
        return obj

    _pre_load.__marshmallow_kwargs__ = _pre_load.__marshmallow_hook__ = defaultdict(dict)
    _pre_dump = _pre_load

    # Actually we don't need post processors, but just for sanity reasons
    def _post_load(self, obj):
        self._value = missing
        return obj

    _post_load.__marshmallow_kwargs__ = _post_load.__marshmallow_hook__ = defaultdict(dict)
    _post_dump = _post_load

    def _bind_to_schema(self, field_name, parent):
        super()._bind_to_schema(field_name, parent)

        # Binding tag processors to parent schema,
        # tricky shit because of marshmallow design...
        # We're setting processors for instance, not for class!
        for tag in ('pre_load', 'pre_dump', 'post_load', 'post_dump'):
            attr = '_{}_{}_schema'.format(tag, field_name)
            if hasattr(parent, attr):
                # Is fields binded on each load? wtf?
                continue
            setattr(parent, attr, getattr(self, '_{}'.format(tag)))
            if hasattr(parent, '__processors__'):
                # Older versions for 2.x and possible some beta versions
                hooks = parent.__processors__ = parent.__processors__.copy()
            else:
                # New version (at least since 3.0.0.b12)
                hooks = parent._hooks = parent._hooks.copy()
            # (tag, many?) - for now only for many=False just not to test other case,
            # there may be issues with many=True
            hooks[(tag, False)].append(attr)
<|endoftext|>


<commit_before>import os
import datetime

from django.contrib.auth.models import AbstractUser
from django.core.mail import send_mail
from django.db import models
from django.template.loader import render_to_string
import mongoengine

from cub.settings import CONTACT_ADMINS, EMAIL_HOST_USER, BASE_DIR

class Account(AbstractUser):

    class Meta:
        app_label = 'backend'

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ('email', 'github_token')

    github_token = models.TextField(blank=False)
    name = models.CharField(max_length=128)
    github_url = models.URLField(blank=False)
    avatar_url = models.URLField()
    synced = models.IntegerField(null=True)

    @classmethod
    def verify(cls, data):
        if Account.objects.filter(username=data['username'], email=data['email']):
            account = Account.objects.get(
                username=data['username'], email=data['email'])

            if data['github_token'] != account.github_token:
                account.set_password(data['github_token'])
                account.github_token = data['github_token']
                account.save()
            return account
        else:
            return cls.create(data)

    @classmethod
    def create(cls, data):
        account = Account(
            username=data['username'], name=data['name'],
            email=data['email'], github_url=data['github_url'],
            avatar_url=data['avatar_url'], github_token=data['github_token'])

        account.set_password(data['github_token'])
        account.save()
        return account

class AbstractJSONDocument(mongoengine.Document):

    meta = {'abstract': True}

    @classmethod
    def save_document(cls, data):
        document = cls()
        for key, value in data.iteritems():
            setattr(document, key, value)
        document.save()
        return document

class Contribution(mongoengine.Document):
    meta = {
        'abstract': True,
        'indexes': [{'fields': ['updated_at']}],
        'ordering': ['-updated_at']
    }

    cub_account = mongoengine.StringField(required=True)
    html_url = mongoengine.URLField(required=True, unique_with='cub_account')
    url = mongoengine.URLField(required=True)
    html_repo_url = mongoengine.URLField(required=True)
    updated_at = mongoengine.DateTimeField(required=True)
    stats = mongoengine.DictField()

class CommitContribution(Contribution):

    files_stats = mongoengine.DictField()

class PRContribution(Contribution):

    title = mongoengine.StringField(required=True)
    state = mongoengine.StringField(required=True)

    @classmethod
    def create_or_update(cls, raw_data, account):
        if PRContribution.objects(
                cub_account=account, html_url=raw_data['html_url']):

            pr = PRContribution.objects.get(
                    cub_account=account, html_url=raw_data['html_url'])

            pr = pr.modify(
                url=raw_data['url'],
                html_repo_url=raw_data['html_url'].split('pull')[0],
                updated_at=raw_data['updated_at'],
                title=raw_data['title'],
                state=raw_data['state'])
        else:
            pr = PRContribution(
                cub_account=account,
                html_url=raw_data['html_url'],
                url=raw_data['url'],
                html_repo_url=raw_data['html_url'].split('pull')[0],
                updated_at=raw_data['updated_at'],
                title=raw_data['title'],
                state=raw_data['state']).save()
        return pr

class Repository(mongoengine.Document):

    meta = {
        'indexes': [{'fields': ['updated_at']}],
        'ordering': ['-updated_at']
    }

    cub_account = mongoengine.StringField(required=True)
    name = mongoengine.StringField(required=True)
    description = mongoengine.StringField(required=True)
    html_url = mongoengine.URLField(required=True, unique_with='cub_account')
    url = mongoengine.URLField(required=True)
    created_at = mongoengine.DateTimeField(required=True)
    updated_at = mongoengine.DateTimeField(required=True)
    fork = mongoengine.StringField(required=True)
    forks_count = mongoengine.IntField(required=True, default=0)
    stargazers_count = mongoengine.IntField(required=True, default=0)
    watchers_count = mongoengine.IntField(required=True, default=0)
    open_issues = mongoengine.IntField(required=True, default=0)
    closed_issues = mongoengine.IntField(required=True, default=0)
    affiliation = mongoengine.StringField(required=True, default='unknown')

    @classmethod
    def create_or_update(cls, raw_data, account):
        if Repository.objects(
                cub_account=account, html_url=raw_data['html_url']):

            repo = Repository.objects.get(
                    cub_account=account, html_url=raw_data['html_url'])

            repo = repo.modify(
                name=raw_data['name'],
                description=raw_data['description'],
                url=raw_data['url'],
                created_at=raw_data['created_at'],
                updated_at=raw_data['updated_at'],
                forks_count=raw_data['forks'],
                fork=str(raw_data['fork']),
                stargazers_count=raw_data['stargazers_count'],
                watchers_count=raw_data['watchers_count'],
                open_issues=raw_data['open_issues_count'],
                affiliation=raw_data['affiliation'])
        else:
            repo = Repository(
                cub_account=account,
                name=raw_data['name'],
                description=raw_data['description'],
                html_url=raw_data['html_url'],
                url=raw_data['url'],
                created_at=raw_data['created_at'],
                updated_at=raw_data['updated_at'],
                forks_count=raw_data['forks'],
                fork=str(raw_data['fork']),
                stargazers_count=raw_data['stargazers_count'],
                watchers_count=raw_data['watchers_count'],
                open_issues=raw_data['open_issues_count'],
                affiliation=raw_data['affiliation']).save()
        return repo

class Score(mongoengine.Document):

    cub_account = mongoengine.StringField(required=True, unique=True)
    score = mongoengine.IntField(required=True)
    watchers_count = mongoengine.IntField()
    forks_count = mongoengine.IntField()
    stargazers_count = mongoengine.IntField()

    @classmethod
    def compute(cls, account):
        watchers = Repository.objects.filter(cub_account=account).sum('watchers_count')
        stargazers = Repository.objects.filter(cub_account=account).sum('stargazers_count')
        forks = Repository.objects.filter(cub_account=account).sum('forks_count')

        live_score =  watchers + stargazers + forks

        if Score.objects(cub_account=account):
            score = Score.objects.get(cub_account=account).modify(
                score=live_score, watchers_count=watchers,
                forks_count=forks, stargazers_count=stargazers)
        else:
            score = Score(cub_account=account, score=live_score,
                watchers_count=watchers, forks_count=forks,
                stargazers_count=stargazers).save()
        return score

class Contact(AbstractJSONDocument):

    meta = {
        'indexes': [{'fields': ['email']}]
    }

    name = mongoengine.StringField(required=True)
    email = mongoengine.EmailField(required=True)
    content = mongoengine.StringField(required=True)
    created_at = mongoengine.DateTimeField(required=True, default=datetime.datetime.now)

    @classmethod
    def send_email(cls, sender, document, **kwargs):
        msg_html = render_to_string(
            '%s/backend/templates/email/contact/notify_team.html' % BASE_DIR,
            {'document': document})

        send_mail(
            subject='CUB - Feedback received',
            message='New Feedback was received.',
            html_message=msg_html,
            from_email='Connect Hub - CUB <%s>' % EMAIL_HOST_USER,
            recipient_list=CONTACT_ADMINS,
            fail_silently=True)

        msg_html = render_to_string(
            '%s/backend/templates/email/contact/confirm_user.html' % BASE_DIR,
            {'document': document})

        send_mail(
            subject='CUB - Feedback sent',
            message='We confirm we received your feedback message.',
            html_message=msg_html,
            from_email='Connect Hub - CUB <%s>' % EMAIL_HOST_USER,
            recipient_list=document['email'],
            fail_silently=True)

mongoengine.signals.post_save.connect(Contact.send_email, sender=Contact)
<commit_msg>Fix email sending<commit_after>import os
import datetime

from django.contrib.auth.models import AbstractUser
from django.core.mail import send_mail
from django.db import models
from django.template.loader import render_to_string
import mongoengine

from cub.settings import CONTACT_ADMINS, EMAIL_HOST_USER, BASE_DIR

class Account(AbstractUser):

    class Meta:
        app_label = 'backend'

    USERNAME_FIELD = 'username'
    REQUIRED_FIELDS = ('email', 'github_token')

    github_token = models.TextField(blank=False)
    name = models.CharField(max_length=128)
    github_url = models.URLField(blank=False)
    avatar_url = models.URLField()
    synced = models.IntegerField(null=True)

    @classmethod
    def verify(cls, data):
        if Account.objects.filter(username=data['username'], email=data['email']):
            account = Account.objects.get(
                username=data['username'], email=data['email'])

            if data['github_token'] != account.github_token:
                account.set_password(data['github_token'])
                account.github_token = data['github_token']
                account.save()
            return account
        else:
            return cls.create(data)

    @classmethod
    def create(cls, data):
        account = Account(
            username=data['username'], name=data['name'],
            email=data['email'], github_url=data['github_url'],
            avatar_url=data['avatar_url'], github_token=data['github_token'])

        account.set_password(data['github_token'])
        account.save()
        return account

class AbstractJSONDocument(mongoengine.Document):

    meta = {'abstract': True}

    @classmethod
    def save_document(cls, data):
        document = cls()
        for key, value in data.iteritems():
            setattr(document, key, value)
        document.save()
        return document

class Contribution(mongoengine.Document):
    meta = {
        'abstract': True,
        'indexes': [{'fields': ['updated_at']}],
        'ordering': ['-updated_at']
    }

    cub_account = mongoengine.StringField(required=True)
    html_url = mongoengine.URLField(required=True, unique_with='cub_account')
    url = mongoengine.URLField(required=True)
    html_repo_url = mongoengine.URLField(required=True)
    updated_at = mongoengine.DateTimeField(required=True)
    stats = mongoengine.DictField()

class CommitContribution(Contribution):

    files_stats = mongoengine.DictField()

class PRContribution(Contribution):

    title = mongoengine.StringField(required=True)
    state = mongoengine.StringField(required=True)

    @classmethod
    def create_or_update(cls, raw_data, account):
        if PRContribution.objects(
                cub_account=account, html_url=raw_data['html_url']):

            pr = PRContribution.objects.get(
                    cub_account=account, html_url=raw_data['html_url'])

            pr = pr.modify(
                url=raw_data['url'],
                html_repo_url=raw_data['html_url'].split('pull')[0],
                updated_at=raw_data['updated_at'],
                title=raw_data['title'],
                state=raw_data['state'])
        else:
            pr = PRContribution(
                cub_account=account,
                html_url=raw_data['html_url'],
                url=raw_data['url'],
                html_repo_url=raw_data['html_url'].split('pull')[0],
                updated_at=raw_data['updated_at'],
                title=raw_data['title'],
                state=raw_data['state']).save()
        return pr

class Repository(mongoengine.Document):

    meta = {
        'indexes': [{'fields': ['updated_at']}],
        'ordering': ['-updated_at']
    }

    cub_account = mongoengine.StringField(required=True)
    name = mongoengine.StringField(required=True)
    description = mongoengine.StringField(required=True)
    html_url = mongoengine.URLField(required=True, unique_with='cub_account')
    url = mongoengine.URLField(required=True)
    created_at = mongoengine.DateTimeField(required=True)
    updated_at = mongoengine.DateTimeField(required=True)
    fork = mongoengine.StringField(required=True)
    forks_count = mongoengine.IntField(required=True, default=0)
    stargazers_count = mongoengine.IntField(required=True, default=0)
    watchers_count = mongoengine.IntField(required=True, default=0)
    open_issues = mongoengine.IntField(required=True, default=0)
    closed_issues = mongoengine.IntField(required=True, default=0)
    affiliation = mongoengine.StringField(required=True, default='unknown')

    @classmethod
    def create_or_update(cls, raw_data, account):
        if Repository.objects(
                cub_account=account, html_url=raw_data['html_url']):

            repo = Repository.objects.get(
                    cub_account=account, html_url=raw_data['html_url'])

            repo = repo.modify(
                name=raw_data['name'],
                description=raw_data['description'],
                url=raw_data['url'],
                created_at=raw_data['created_at'],
                updated_at=raw_data['updated_at'],
                forks_count=raw_data['forks'],
                fork=str(raw_data['fork']),
                stargazers_count=raw_data['stargazers_count'],
                watchers_count=raw_data['watchers_count'],
                open_issues=raw_data['open_issues_count'],
                affiliation=raw_data['affiliation'])
        else:
            repo = Repository(
                cub_account=account,
                name=raw_data['name'],
                description=raw_data['description'],
                html_url=raw_data['html_url'],
                url=raw_data['url'],
                created_at=raw_data['created_at'],
                updated_at=raw_data['updated_at'],
                forks_count=raw_data['forks'],
                fork=str(raw_data['fork']),
                stargazers_count=raw_data['stargazers_count'],
                watchers_count=raw_data['watchers_count'],
                open_issues=raw_data['open_issues_count'],
                affiliation=raw_data['affiliation']).save()
        return repo

class Score(mongoengine.Document):

    cub_account = mongoengine.StringField(required=True, unique=True)
    score = mongoengine.IntField(required=True)
    watchers_count = mongoengine.IntField()
    forks_count = mongoengine.IntField()
    stargazers_count = mongoengine.IntField()

    @classmethod
    def compute(cls, account):
        watchers = Repository.objects.filter(cub_account=account).sum('watchers_count')
        stargazers = Repository.objects.filter(cub_account=account).sum('stargazers_count')
        forks = Repository.objects.filter(cub_account=account).sum('forks_count')

        live_score =  watchers + stargazers + forks

        if Score.objects(cub_account=account):
            score = Score.objects.get(cub_account=account).modify(
                score=live_score, watchers_count=watchers,
                forks_count=forks, stargazers_count=stargazers)
        else:
            score = Score(cub_account=account, score=live_score,
                watchers_count=watchers, forks_count=forks,
                stargazers_count=stargazers).save()
        return score

class Contact(AbstractJSONDocument):

    meta = {
        'indexes': [{'fields': ['email']}]
    }

    name = mongoengine.StringField(required=True)
    email = mongoengine.EmailField(required=True)
    content = mongoengine.StringField(required=True)
    created_at = mongoengine.DateTimeField(required=True, default=datetime.datetime.now)

    @classmethod
    def send_email(cls, sender, document, **kwargs):
        msg_html = render_to_string(
            '%s/backend/templates/email/contact/notify_team.html' % BASE_DIR,
            {'document': document})

        send_mail(
            subject='CUB - Feedback received',
            message='New Feedback was received.',
            html_message=msg_html,
            from_email='Connect Hub - CUB <%s>' % EMAIL_HOST_USER,
            recipient_list=CONTACT_ADMINS,
            fail_silently=True)

        msg_html = render_to_string(
            '%s/backend/templates/email/contact/confirm_user.html' % BASE_DIR,
            {'document': document})

        send_mail(
            subject='CUB - Feedback sent',
            message='We confirm we received your feedback message.',
            html_message=msg_html,
            from_email='Connect Hub - CUB <%s>' % EMAIL_HOST_USER,
            recipient_list=document.email,
            fail_silently=True)

mongoengine.signals.post_save.connect(Contact.send_email, sender=Contact)
<|endoftext|>


<commit_before># encoding: utf8
import logging

from pyramid.httpexceptions import HTTPSeeOther
from pyramid.security import effective_principals
from pyramid.view import view_config
import wtforms
from wtforms.ext.sqlalchemy.fields import QuerySelectMultipleField

from floof import model
from floof.forms import FloofForm, MultiCheckboxField
from floof.lib.openid_ import OpenIDError, openid_begin, openid_end
from floof.lib.stash import stash_request

log = logging.getLogger(__name__)

# XXX this all needs cleaning up, and some ajax, and whatever.
# XXX two forms on one page also raises questions about how best to handle invalid forms
class AddOpenIDForm(wtforms.form.Form):
    new_openid = wtforms.TextField(u'New OpenID', [wtforms.validators.Required()])

class RemoveOpenIDForm(FloofForm):
    #openids = MultiCheckboxField(u'', coerce=int)
    openids = QuerySelectMultipleField(u'Remove OpenIDs', get_label=lambda row: row.url)

    def validate_openids(form, field):
        if not field.data:
            raise wtforms.ValidationError('You must select at least one OpenID identity URL to delete.')

        if len(field.data) >= len(field._get_object_list()):
            raise wtforms.ValidationError('You must keep at least one OpenID identity URL.')

        # XXX less hackish way to do this without adding an attr to
        # Authenticizer for every freakin property of the session's auth?
        curr_openid_url = form.request.auth.state.get('openid_url')
        if curr_openid_url in [obj.url for key, obj in field._get_object_list()]:
            raise wtforms.ValidationError(
                    'You cannot remove the OpenID identity URL with which you '
                    'are currently logged in.')

class AuthenticationForm(FloofForm):
    cert_auth = wtforms.fields.SelectField(u'Certificate Certificates', choices=[
            (u'disabled', u'Disallow using client certificates for login (default)'),
            (u'allowed', u'Allow using client certificates for login'),
            (u'sensitive_required', u'Allow using client certificates for login; Require for Sensitive Operations'),
            (u'required', u'Require using client certificates for login'),
            ])

    def validate_cert_auth(form, field):
        request = form.request
        if field.data in ['required', 'sensitive_required']:
            if not request.user.valid_certificates:
                raise wtforms.ValidationError('You cannot make a selection '
                        'that requires an SSL certificate to log in or to '
                        'change this setting while you have no valid SSL '
                        'certificates registered against your account.')
            if not 'trusted:cert' in effective_principals(request):
                raise wtforms.ValidationError('To prevent locking yourself '
                        'out, you cannot make a selection that requires an '
                        'SSL certificate to log in or to change this '
                        'setting without first loading this page while the '
                        'certificate is installed in your browser and being '
                        'successfully sent to the site.')

@view_config(
    route_name='controls.openid',
    permission='auth.openid',
    request_method='GET',
    renderer='account/controls/openid.mako')
def openid(context, request):
    user = request.user
    add_openid_form = AddOpenIDForm()
    remove_openid_form = RemoveOpenIDForm(request)
    remove_openid_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    return dict(
        add_openid_form=add_openid_form,
        remove_openid_form=remove_openid_form,
    )

@view_config(
    route_name='controls.openid.add',
    permission='auth.openid',
    request_method='POST',
    renderer='account/controls/openid.mako')
def openid_add(context, request):
    user = request.user
    form = AddOpenIDForm(request.POST) # XXX fetch_post(session, request))
    remove_form = RemoveOpenIDForm(request)
    remove_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        add_openid_form=form,
        remove_openid_form=remove_form,
    )

    # Add an OpenID identity URL
    if not form.validate():
        return ret

    try:
        return HTTPSeeOther(location=openid_begin(
            identifier=form.new_openid.data,
            return_url=request.route_url('controls.openid.add_finish'),
            request=request,
            sreg=False,
        ))
    except OpenIDError as exc:
        form.new_openid.errors.append(exc.args[0])
        return ret

@view_config(
    route_name='controls.openid.add_finish',
    permission='auth.openid',
    request_method='GET',
    renderer='account/controls/openid.mako')
def openid_add_finish(context, request):
    user = request.user
    # XXX we should put the attempted openid in here
    form = AddOpenIDForm() # XXX fetch_post(session, request))
    remove_form = RemoveOpenIDForm(request)
    remove_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        add_openid_form=form,
        remove_openid_form=remove_form,
    )

    # Process a returning OpenID check
    form.validate()  # Ensure new_openid.errors is an appendable list
    try:
        identity_url, identity_webfinger, auth_time, sreg_res = openid_end(request.path_url, request)
    except OpenIDError as exc:
        form.new_openid.errors.append(exc.args[0])
        return ret

    # Allow an OpenID identity to be registered to only one user.
    existing_url = model.session.query(model.IdentityURL) \
        .filter_by(url=identity_url) \
        .first()
    if existing_url:
        if existing_url.user_id == user.id:
            form.new_openid.errors.append(u'You can already authenticate with that OpenID identity.')
        else:
            form.new_openid.errors.append(u'That OpenID identity is already in use by another account.')
        return ret

    openid = model.IdentityURL(url=identity_url)
    user.identity_urls.append(openid)
    request.session.flash(
        u"Added a new identity: {0}".format(identity_url),
        level=u'success', icon=u'user--plus')

    return HTTPSeeOther(location=request.route_url('controls.openid'))

@view_config(
    route_name='controls.openid.remove',
    permission='auth.openid',
    request_method='POST',
    renderer='account/controls/openid.mako')
def openid_remove(context, request):
    user = request.user
    form = RemoveOpenIDForm(request, request.POST)
    form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        remove_openid_form=form,
        add_openid_form=AddOpenIDForm(),
    )

    # Delete one or more OpenID identity URLs
    if not form.validate():
        return ret  # XXX uhh

    for target in form.openids.data:
        request.session.flash(
            u"Removed OpenID identifier: {0}".format(target.url),
            level=u'success')
        model.session.delete(target)
    return HTTPSeeOther(location=request.route_url('controls.openid'))

@view_config(
    route_name='controls.auth',
    permission='auth.method',
    request_method='GET',
    renderer='account/controls/authentication.mako')
def authentication(context, request):
    form = AuthenticationForm(request, obj=request.user)
    return dict(
        form=form,
    )

# XXX this one is full of things to fix
@view_config(
    route_name='controls.auth',
    permission='auth.method',
    request_method='POST')
def authentication_commit(context, request):
    form = AuthenticationForm(request, request.POST, request.user)

    if form.validate():
        form.populate_obj(request.user)
        request.session.flash(u'Authentication options updated.', level=u'success')
        return HTTPSeeOther(location=request.path_url)

    stash_request(request, _immediate=True)
    return HTTPSeeOther(location=request.path_url)
<commit_msg>Fix silly error in a7817e6<commit_after># encoding: utf8
import logging

from pyramid.httpexceptions import HTTPSeeOther
from pyramid.security import effective_principals
from pyramid.view import view_config
import wtforms
from wtforms.ext.sqlalchemy.fields import QuerySelectMultipleField

from floof import model
from floof.forms import FloofForm, MultiCheckboxField
from floof.lib.openid_ import OpenIDError, openid_begin, openid_end
from floof.lib.stash import stash_request

log = logging.getLogger(__name__)

# XXX this all needs cleaning up, and some ajax, and whatever.
# XXX two forms on one page also raises questions about how best to handle invalid forms
class AddOpenIDForm(wtforms.form.Form):
    new_openid = wtforms.TextField(u'New OpenID', [wtforms.validators.Required()])

class RemoveOpenIDForm(FloofForm):
    #openids = MultiCheckboxField(u'', coerce=int)
    openids = QuerySelectMultipleField(u'Remove OpenIDs', get_label=lambda row: row.url)

    def validate_openids(form, field):
        if not field.data:
            raise wtforms.ValidationError('You must select at least one OpenID identity URL to delete.')

        if len(field.data) >= len(field._get_object_list()):
            raise wtforms.ValidationError('You must keep at least one OpenID identity URL.')

        # XXX less hackish way to do this without adding an attr to
        # Authenticizer for every freakin property of the session's auth?
        curr_openid_url = form.request.auth.state.get('openid_url')
        if curr_openid_url in [obj.url for obj in field.data]:
            raise wtforms.ValidationError(
                    'You cannot remove the OpenID identity URL with which you '
                    'are currently logged in.')

class AuthenticationForm(FloofForm):
    cert_auth = wtforms.fields.SelectField(u'Certificate Certificates', choices=[
            (u'disabled', u'Disallow using client certificates for login (default)'),
            (u'allowed', u'Allow using client certificates for login'),
            (u'sensitive_required', u'Allow using client certificates for login; Require for Sensitive Operations'),
            (u'required', u'Require using client certificates for login'),
            ])

    def validate_cert_auth(form, field):
        request = form.request
        if field.data in ['required', 'sensitive_required']:
            if not request.user.valid_certificates:
                raise wtforms.ValidationError('You cannot make a selection '
                        'that requires an SSL certificate to log in or to '
                        'change this setting while you have no valid SSL '
                        'certificates registered against your account.')
            if not 'trusted:cert' in effective_principals(request):
                raise wtforms.ValidationError('To prevent locking yourself '
                        'out, you cannot make a selection that requires an '
                        'SSL certificate to log in or to change this '
                        'setting without first loading this page while the '
                        'certificate is installed in your browser and being '
                        'successfully sent to the site.')

@view_config(
    route_name='controls.openid',
    permission='auth.openid',
    request_method='GET',
    renderer='account/controls/openid.mako')
def openid(context, request):
    user = request.user
    add_openid_form = AddOpenIDForm()
    remove_openid_form = RemoveOpenIDForm(request)
    remove_openid_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    return dict(
        add_openid_form=add_openid_form,
        remove_openid_form=remove_openid_form,
    )

@view_config(
    route_name='controls.openid.add',
    permission='auth.openid',
    request_method='POST',
    renderer='account/controls/openid.mako')
def openid_add(context, request):
    user = request.user
    form = AddOpenIDForm(request.POST) # XXX fetch_post(session, request))
    remove_form = RemoveOpenIDForm(request)
    remove_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        add_openid_form=form,
        remove_openid_form=remove_form,
    )

    # Add an OpenID identity URL
    if not form.validate():
        return ret

    try:
        return HTTPSeeOther(location=openid_begin(
            identifier=form.new_openid.data,
            return_url=request.route_url('controls.openid.add_finish'),
            request=request,
            sreg=False,
        ))
    except OpenIDError as exc:
        form.new_openid.errors.append(exc.args[0])
        return ret

@view_config(
    route_name='controls.openid.add_finish',
    permission='auth.openid',
    request_method='GET',
    renderer='account/controls/openid.mako')
def openid_add_finish(context, request):
    user = request.user
    # XXX we should put the attempted openid in here
    form = AddOpenIDForm() # XXX fetch_post(session, request))
    remove_form = RemoveOpenIDForm(request)
    remove_form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        add_openid_form=form,
        remove_openid_form=remove_form,
    )

    # Process a returning OpenID check
    form.validate()  # Ensure new_openid.errors is an appendable list
    try:
        identity_url, identity_webfinger, auth_time, sreg_res = openid_end(request.path_url, request)
    except OpenIDError as exc:
        form.new_openid.errors.append(exc.args[0])
        return ret

    # Allow an OpenID identity to be registered to only one user.
    existing_url = model.session.query(model.IdentityURL) \
        .filter_by(url=identity_url) \
        .first()
    if existing_url:
        if existing_url.user_id == user.id:
            form.new_openid.errors.append(u'You can already authenticate with that OpenID identity.')
        else:
            form.new_openid.errors.append(u'That OpenID identity is already in use by another account.')
        return ret

    openid = model.IdentityURL(url=identity_url)
    user.identity_urls.append(openid)
    request.session.flash(
        u"Added a new identity: {0}".format(identity_url),
        level=u'success', icon=u'user--plus')

    return HTTPSeeOther(location=request.route_url('controls.openid'))

@view_config(
    route_name='controls.openid.remove',
    permission='auth.openid',
    request_method='POST',
    renderer='account/controls/openid.mako')
def openid_remove(context, request):
    user = request.user
    form = RemoveOpenIDForm(request, request.POST)
    form.openids.query = model.session.query(model.IdentityURL).with_parent(user)

    ret = dict(
        remove_openid_form=form,
        add_openid_form=AddOpenIDForm(),
    )

    # Delete one or more OpenID identity URLs
    if not form.validate():
        return ret  # XXX uhh

    for target in form.openids.data:
        request.session.flash(
            u"Removed OpenID identifier: {0}".format(target.url),
            level=u'success')
        model.session.delete(target)
    return HTTPSeeOther(location=request.route_url('controls.openid'))

@view_config(
    route_name='controls.auth',
    permission='auth.method',
    request_method='GET',
    renderer='account/controls/authentication.mako')
def authentication(context, request):
    form = AuthenticationForm(request, obj=request.user)
    return dict(
        form=form,
    )

# XXX this one is full of things to fix
@view_config(
    route_name='controls.auth',
    permission='auth.method',
    request_method='POST')
def authentication_commit(context, request):
    form = AuthenticationForm(request, request.POST, request.user)

    if form.validate():
        form.populate_obj(request.user)
        request.session.flash(u'Authentication options updated.', level=u'success')
        return HTTPSeeOther(location=request.path_url)

    stash_request(request, _immediate=True)
    return HTTPSeeOther(location=request.path_url)
<|endoftext|>


<commit_before>import logging

from flow.orchestrator.flow.job import Job

LOG = logging.getLogger(__name__)

class SuccessfulCompletionService(object):
    def __init__(self, redis=None, publish_function=None):
        self.redis = redis
        self.publish_function = publish_function

#        self.publish_function = functools.partial(exchange_manager.publish,
#                **basic_publish_properties)

    def message_handler(self, properties, input_data,
            ack_callback, reject_callback)

        job = Job.get(self.redis, key=job_key)
        jobs_to_start = job.complete()
        for jts in jobs_to_start:
            jts.start(publish_function=self.publish_function)

        ack_callback()

    def bad_data_handler(self, properties, body, ack_callback, reject_callback):
        LOG.debug('Got bad data, properties = %s: %s', properties, body)
        reject_callback()

class UnsuccessfulCompletionService(object):
    def __init__(self, redis=None):
        self.redis = redis

    def message_handler(self, properties, input_data,
            ack_callback, reject_callback)

        job = Job.get(self.redis, key=job_key)
        job.fail()

        ack_callback()

    def bad_data_handler(self, properties, body, ack_callback, reject_callback):
        LOG.debug('Got bad data, properties = %s: %s', properties, body)
        reject_callback()
<commit_msg>remove useless old orchestrator service module<commit_after><|endoftext|>


<commit_before># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4

# Flumotion - a video streaming server
# Copyright (C) 2004 Fluendo
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

import ConfigParser
import optparse
import os
import signal
import sys
import warnings
import string

try:
    warnings.filterwarnings('ignore', category=FutureWarning)
except:
    pass

sys_argv = sys.argv
sys.argv = sys_argv[:1]
import gst
sys.argv = sys_argv

from flumotion.twisted import gstreactor
gstreactor.install()

from twisted.internet import reactor
from twisted.web import server, resource

from flumotion import errors, twisted
from flumotion.server.controller import ControllerServerFactory
from flumotion.server.converter import Converter
from flumotion.server.producer import Producer
from flumotion.server import streamer
from flumotion.utils import log, gstutils

def set_proc_text(text):
    return

    i = 0
    for item in proc:
        n = len(item)
        if not text:
            value = '\0'
        elif len(text) > n:
            value = text[:n+1]
            text = text[n:]
        else:
            value = text + '\0'
            text = None

        print '%d = %r' % (i, value)
        proc[i] = value
        i += 1

    #print len(proc)
    #proc[0] = 'flumotion - laun'
    #proc[1] = 'cher\0'
    #proc[2] = '\0'
    #proc[3] = '\0'
    #proc[4] = '\0'

class Launcher:
    def __init__(self, controller_port):
        self.children = []
        self.controller_pid = None
        self.controller_port = controller_port

        signal.signal(signal.SIGCHLD, signal.SIG_IGN)
        signal.signal(signal.SIGSEGV, self.segv_handler)

        set_proc_text('flumotion [launcher]')

    def msg(self, *args):
        log.msg('launcher', *args)

    def start_controller(self, port):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [controller]')
            factory = ControllerServerFactory()
            self.controller = reactor.listenTCP(port, factory)
            try:
                reactor.run(False)
            except KeyboardInterrupt:
                pass

            raise SystemExit
        self.controller_pid = pid

    def signal_handler(self, *args):
        for pid in self.children:
            try:
                os.kill(0, pid)
                os.waitpid(pid, 0)
            except OSError:
                pass

    def segv_handler(self, *args):
        print args

    def spawn(self, component):
        pid = os.getpid()
        def exit_cb(*args):
            print 'EXIT'
            component.pipeline_stop()
            raise SystemExit

        signal.signal(signal.SIGCHLD, signal.SIG_DFL)
        signal.signal(signal.SIGINT, exit_cb)
        reactor.connectTCP('localhost', self.controller_port,
                           component.factory)

        try:
            reactor.run(False)
        except KeyboardInterrupt:
            pass
        component.pipeline_stop()

    def start(self, component):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [%s]' % component.component_name)
            self.spawn(component)
            raise SystemExit
        else:
            self.msg('Starting %s (%s) on pid %d' %
                    (component.component_name,
                     component.getKind(),
                     pid))
            self.children.append(pid)

    def start_streamer(self, component, factory, port):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [%s]' % component.component_name)
            reactor.listenTCP(port, factory)
            self.spawn(component)
            raise SystemExit
        else:
            self.msg('Starting %s (%s) on pid %d' %
                    (component.component_name,
                     component.getKind(),
                     pid))
            self.children.append(pid)

    def run(self):
        while self.children:
            for pid in self.children:
                try:
                    os.kill(pid, 0)
                except OSError:
                    self.children.remove(pid)

            if not self.children:
                continue

            try:
                pid = os.waitpid(self.children[0], 0)
                self.children.remove(pid)
                self.msg('%d is dead pid' % pid)
            except (KeyboardInterrupt, OSError):
                pass

        if self.controller_pid:
            try:
                os.kill(self.controller_pid, signal.SIGINT)
            except OSError:
                pass

        self.msg('Shutting down reactor')
        reactor.stop()

        raise SystemExit

    def load_config(self, filename):
        c = ConfigParser.ConfigParser()
        self.msg('Loading configuration file `%s\'' % filename)
        c.read(filename)

        components = {}
        for section in c.sections():
            name = section
            if not c.has_option(section, 'kind'):
                raise AssertionError
            kind = c.get(section, 'kind')
            if not kind in ('producer', 'converter', 'streamer'):
                raise AssertionError

            # XXX: Throw nice warnings
            if kind == 'producer' or kind == 'converter':
                assert c.has_option(section, 'pipeline')
                pipeline = c.get(section, 'pipeline')
                if c.has_option(section, 'feeds'):
                    feeds = c.get(section, 'feeds').split(',')
                else:
                    feeds = ['default']
            else:
                feeds = []
                pipeline = ''

            if kind == 'converter' or kind == 'streamer':
                assert (c.has_option(section, 'source') or
                        c.has_option(section, 'sources'))
                if c.has_option(section, 'source'):
                    sources = [c.get(section, 'source')]
                else:
                    sources = c.get(section, 'sources').split(',')
            else:
                sources = []

            if kind == 'producer':
                self.start(Producer(name, sources, feeds, pipeline))
            elif kind == 'converter':
                self.start(Converter(name, sources, feeds, pipeline))
            elif kind == 'streamer':
                assert c.has_option(section, 'protocol')
                protocol = c.get(section, 'protocol')

                if protocol == 'http':
                    assert c.has_option(section, 'port')
                    port = c.getint(section, 'port')
                    component = streamer.FakeSinkStreamer(name, sources)
                    factory = server.Site(resource=streamer.StreamingResource(component))
                    self.start_streamer(component, factory, port)
                elif protocol == 'file':
                    assert c.has_option(section, 'location')
                    location = c.get(section, 'location')
                    self.start(streamer.FileSinkStreamer(name, sources, location))
                else:
                    raise AssertionError, "unknown protocol: %s" % protocol
            else:
                raise AssertionError, "unknown component kind: %s" % kind

def get_options_for(kind, args):
    if kind == 'producer':
        need_sources = False
        need_pipeline = True
    elif kind == 'converter':
        need_sources = True
        need_pipeline = True
    elif kind == 'streamer':
        need_sources = True
        need_pipeline = False
    else:
        raise AssertionError

    usage = "usage: flumotion %s [options]" % kind
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")

    group = optparse.OptionGroup(parser, '%s%s options' % (kind[0].upper(),
                                                           kind[1:]))
    group.add_option('-n', '--name',
                     action="store", type="string", dest="name",
                     default=None,
                     help="Name of component")
    if need_pipeline:
        group.add_option('-p', '--pipeline',
                         action="store", type="string", dest="pipeline",
                         default=None,
                         help="Pipeline to run")
        group.add_option('-f', '--feeds',
                         action="store", type="string", dest="feeds",
                         default=[],
                         help="Feeds to provide")

    if need_sources:
        group.add_option('-s', '--sources',
                         action="store", type="string", dest="sources",
                         default="",
                         help="Host sources to get data from, separated by ,")

    if kind == 'streamer':
        group.add_option('-p', '--protocol',
                         action="store", type="string", dest="protocol",
                         default="http",
                         help="Protocol to use [default http]")
        group.add_option('-o', '--listen-port',
                         action="store", type="int", dest="listen_port",
                         default=8080,
                         help="Port to bind to [default 8080]")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser, "Controller options")
    group.add_option('-c', '--controller-host',
                     action="store", type="string", dest="host",
                     default="localhost",
                     help="Controller to connect to [default localhost]")
    group.add_option('', '--controller-port',
                     action="store", type="int", dest="port",
                     default=8890,
                     help="Controller port to connect to [default 8890]")
    parser.add_option_group(group)

    options, args = parser.parse_args(args)

    if options.name is None:
        raise errors.OptionError, 'Need a name'
    elif need_pipeline:
        if options.pipeline is None:
            raise errors.OptionError, 'Need a pipeline'
        if options.feeds is None:
            raise errors.OptionError, 'Need feeds'
        else:
            options.feeds = options.feeds.split(',')
    elif need_sources and options.sources is None:
        raise OptionError, 'Need a source'
    elif kind == 'streamer':
        if not options.protocol:
            raise errors.OptionError, 'Need a protocol'
        elif not options.listen_port:
            raise errors.OptionError, 'Need a listen_port'
            return 2

    if options.verbose:
        log.enableLogging()

    if need_sources:
        if ',' in  options.sources:
            options.sources = options.sources.split(',')
        else:
            options.sources = [options.sources]
    else:
        options.sources = []

    return options

def run_launcher(args):
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")
    parser.add_option('-c', '--controller-port',
                      action="store", type="int", dest="port",
                      help="Controller port", default=8890)

    options, args = parser.parse_args(args)

    if len(args) < 3:
        print 'Need a configuration file'
        return -1

    if options.verbose:
        log.enableLogging()

    launcher = Launcher(options.port)

    if not gstutils.is_port_free(options.port):
        launcher.msg('Controller is already started')
    else:
        launcher.start_controller(options.port)

    launcher.load_config(args[2])

    launcher.run()

    return 0

def run_controller(args):
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")
    group = optparse.OptionGroup(parser, "Controller options")
    group.add_option('-p', '--port',
                     action="store", type="int", dest="port",
                     default=8890,
                     help="Port to listen on [default 8890]")
    parser.add_option_group(group)

    options, args = parser.parse_args(args)

    if options.verbose:
        log.enableLogging()

    factory = ControllerServerFactory()

    log.msg('controller', 'Starting at port %d' % options.port)
    reactor.listenTCP(options.port, factory)
    reactor.run()

    return 0

def run_component(name, args):
    try:
        options = get_options_for(name, args)
    except errors.OptionError, e:
        print 'ERROR:', e
        raise SystemExit

    if name == 'producer':
        klass = Producer
        args = (options.name, options.sources, options.feeds, options.pipeline)
    elif name == 'converter':
        klass = Converter
        args = (options.name, options.sources, options.feeds, options.pipeline)
    elif name == 'streamer':
        if options.protocol == 'http':
            web_factory = server.Site(resource=StreamingResource(component))
            reactor.listenTCP(options.listen_port, web_factory)
            klass = streamer.FakeSinkStreamer
            args = (options.name, options.sources)
        elif options.protocol == 'file':
            klass = streamer.FileStreamer
            args = (options.name, options.sources, options.location)
        else:
            print 'Only http and file protcol supported right now'
            return
    else:
        raise AssertionError

    try:
        print klass, args
        component = klass(*args)
    except twisted.errors.PipelineParseError, e:
        print 'Bad pipeline: %s' % e
        raise SystemExit

    reactor.connectTCP(options.host, options.port, component.factory)

    reactor.run()

def usage():
    print 'Usage: flumotion command [command-options-and-arguments]'

def show_commands():
    print 'Flumotion commands are:'
    print '\tlauncher      Component launcher'
    print '\tcontroller    Component controller'
    print '\tproducer      Producer component'
    print '\tconverter     Converter component'
    print '\tstreamer      Streamer component'
    print
    print '(Specify the --help option for a list of other help options)'

def main(args):
    if len(args) < 2:
        usage()
        return -1

    name = args[1]
    if name == 'controller':
        return run_controller(args)
    if name in ['producer', 'converter', 'streamer']:
        return run_component(args[1], args[2:])
    elif name == 'launcher':
        return run_launcher(args)
    else:
        print "Unknown command: `%s'" % name
        print
        show_commands()
        return -1

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>nice baby nice<commit_after># -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4

# Flumotion - a video streaming server
# Copyright (C) 2004 Fluendo
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

import ConfigParser
import optparse
import os
import signal
import sys
import warnings
import string

try:
    warnings.filterwarnings('ignore', category=FutureWarning)
except:
    pass

sys_argv = sys.argv
sys.argv = sys_argv[:1]
import gst
sys.argv = sys_argv

from flumotion.twisted import gstreactor
gstreactor.install()

from twisted.internet import reactor
from twisted.web import server, resource

from flumotion import errors, twisted
from flumotion.server.controller import ControllerServerFactory
from flumotion.server.converter import Converter
from flumotion.server.producer import Producer
from flumotion.server import streamer
from flumotion.utils import log, gstutils

def set_proc_text(text):
    return

    i = 0
    for item in proc:
        n = len(item)
        if not text:
            value = '\0'
        elif len(text) > n:
            value = text[:n+1]
            text = text[n:]
        else:
            value = text + '\0'
            text = None

        print '%d = %r' % (i, value)
        proc[i] = value
        i += 1

    #print len(proc)
    #proc[0] = 'flumotion - laun'
    #proc[1] = 'cher\0'
    #proc[2] = '\0'
    #proc[3] = '\0'
    #proc[4] = '\0'

class Launcher:
    def __init__(self, controller_port):
        self.children = []
        self.controller_pid = None
        self.controller_port = controller_port

        signal.signal(signal.SIGCHLD, signal.SIG_IGN)
        signal.signal(signal.SIGSEGV, self.segv_handler)

        set_proc_text('flumotion [launcher]')

    def msg(self, *args):
        log.msg('launcher', *args)

    def start_controller(self, port):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [controller]')
            factory = ControllerServerFactory()
            self.controller = reactor.listenTCP(port, factory)
            try:
                reactor.run(False)
            except KeyboardInterrupt:
                pass

            raise SystemExit
        self.controller_pid = pid

    def signal_handler(self, *args):
        for pid in self.children:
            try:
                os.kill(0, pid)
                os.waitpid(pid, 0)
            except OSError:
                pass

    def segv_handler(self, *args):
        print args

    def spawn(self, component):
        pid = os.getpid()
        def exit_cb(*args):
            print 'EXIT'
            component.pipeline_stop()
            raise SystemExit

        signal.signal(signal.SIGCHLD, signal.SIG_DFL)
        signal.signal(signal.SIGINT, exit_cb)
        reactor.connectTCP('localhost', self.controller_port,
                           component.factory)

        try:
            reactor.run(False)
        except KeyboardInterrupt:
            pass
        component.pipeline_stop()

    def start(self, component, nice = 0):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [%s]' % component.component_name)
            self.spawn(component)
            raise SystemExit
        else:
            self.msg('Starting %s (%s) on pid %d' %
                    (component.component_name,
                     component.getKind(),
                     pid))
            self.children.append(pid)
            if nice:
                self.msg('Renicing with %d' % nice)
                os.nice (nice)

    def start_streamer(self, component, factory, port):
        pid = os.fork()
        if not pid:
            set_proc_text('flumotion [%s]' % component.component_name)
            reactor.listenTCP(port, factory)
            self.spawn(component)
            raise SystemExit
        else:
            self.msg('Starting %s (%s) on pid %d' %
                    (component.component_name,
                     component.getKind(),
                     pid))
            self.children.append(pid)

    def run(self):
        while self.children:
            for pid in self.children:
                try:
                    os.kill(pid, 0)
                except OSError:
                    self.children.remove(pid)

            if not self.children:
                continue

            try:
                pid = os.waitpid(self.children[0], 0)
                self.children.remove(pid)
                self.msg('%d is dead pid' % pid)
            except (KeyboardInterrupt, OSError):
                pass

        if self.controller_pid:
            try:
                os.kill(self.controller_pid, signal.SIGINT)
            except OSError:
                pass

        self.msg('Shutting down reactor')
        reactor.stop()

        raise SystemExit

    def load_config(self, filename):
        c = ConfigParser.ConfigParser()
        self.msg('Loading configuration file `%s\'' % filename)
        c.read(filename)

        components = {}
        for section in c.sections():
            name = section
            if not c.has_option(section, 'kind'):
                raise AssertionError
            kind = c.get(section, 'kind')
            if not kind in ('producer', 'converter', 'streamer'):
                raise AssertionError

            # XXX: Throw nice warnings
            if kind == 'producer' or kind == 'converter':
                assert c.has_option(section, 'pipeline')
                pipeline = c.get(section, 'pipeline')
                if c.has_option(section, 'feeds'):
                    feeds = c.get(section, 'feeds').split(',')
                else:
                    feeds = ['default']
            else:
                feeds = []
                pipeline = ''

            if kind == 'converter' or kind == 'streamer':
                assert (c.has_option(section, 'source') or
                        c.has_option(section, 'sources'))
                if c.has_option(section, 'source'):
                    sources = [c.get(section, 'source')]
                else:
                    sources = c.get(section, 'sources').split(',')
            else:
                sources = []

            if kind == 'producer':
                self.start(Producer(name, sources, feeds, pipeline))
            elif kind == 'converter':
                self.start(Converter(name, sources, feeds, pipeline), nice=5)
            elif kind == 'streamer':
                assert c.has_option(section, 'protocol')
                protocol = c.get(section, 'protocol')

                if protocol == 'http':
                    assert c.has_option(section, 'port')
                    port = c.getint(section, 'port')
                    component = streamer.FakeSinkStreamer(name, sources)
                    factory = server.Site(resource=streamer.StreamingResource(component))
                    self.start_streamer(component, factory, port)
                elif protocol == 'file':
                    assert c.has_option(section, 'location')
                    location = c.get(section, 'location')
                    self.start(streamer.FileSinkStreamer(name, sources, location))
                else:
                    raise AssertionError, "unknown protocol: %s" % protocol
            else:
                raise AssertionError, "unknown component kind: %s" % kind

def get_options_for(kind, args):
    if kind == 'producer':
        need_sources = False
        need_pipeline = True
    elif kind == 'converter':
        need_sources = True
        need_pipeline = True
    elif kind == 'streamer':
        need_sources = True
        need_pipeline = False
    else:
        raise AssertionError

    usage = "usage: flumotion %s [options]" % kind
    parser = optparse.OptionParser(usage=usage)
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")

    group = optparse.OptionGroup(parser, '%s%s options' % (kind[0].upper(),
                                                           kind[1:]))
    group.add_option('-n', '--name',
                     action="store", type="string", dest="name",
                     default=None,
                     help="Name of component")
    if need_pipeline:
        group.add_option('-p', '--pipeline',
                         action="store", type="string", dest="pipeline",
                         default=None,
                         help="Pipeline to run")
        group.add_option('-f', '--feeds',
                         action="store", type="string", dest="feeds",
                         default=[],
                         help="Feeds to provide")

    if need_sources:
        group.add_option('-s', '--sources',
                         action="store", type="string", dest="sources",
                         default="",
                         help="Host sources to get data from, separated by ,")

    if kind == 'streamer':
        group.add_option('-p', '--protocol',
                         action="store", type="string", dest="protocol",
                         default="http",
                         help="Protocol to use [default http]")
        group.add_option('-o', '--listen-port',
                         action="store", type="int", dest="listen_port",
                         default=8080,
                         help="Port to bind to [default 8080]")
    parser.add_option_group(group)

    group = optparse.OptionGroup(parser, "Controller options")
    group.add_option('-c', '--controller-host',
                     action="store", type="string", dest="host",
                     default="localhost",
                     help="Controller to connect to [default localhost]")
    group.add_option('', '--controller-port',
                     action="store", type="int", dest="port",
                     default=8890,
                     help="Controller port to connect to [default 8890]")
    parser.add_option_group(group)

    options, args = parser.parse_args(args)

    if options.name is None:
        raise errors.OptionError, 'Need a name'
    elif need_pipeline:
        if options.pipeline is None:
            raise errors.OptionError, 'Need a pipeline'
        if options.feeds is None:
            raise errors.OptionError, 'Need feeds'
        else:
            options.feeds = options.feeds.split(',')
    elif need_sources and options.sources is None:
        raise OptionError, 'Need a source'
    elif kind == 'streamer':
        if not options.protocol:
            raise errors.OptionError, 'Need a protocol'
        elif not options.listen_port:
            raise errors.OptionError, 'Need a listen_port'
            return 2

    if options.verbose:
        log.enableLogging()

    if need_sources:
        if ',' in  options.sources:
            options.sources = options.sources.split(',')
        else:
            options.sources = [options.sources]
    else:
        options.sources = []

    return options

def run_launcher(args):
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")
    parser.add_option('-c', '--controller-port',
                      action="store", type="int", dest="port",
                      help="Controller port", default=8890)

    options, args = parser.parse_args(args)

    if len(args) < 3:
        print 'Need a configuration file'
        return -1

    if options.verbose:
        log.enableLogging()

    launcher = Launcher(options.port)

    if not gstutils.is_port_free(options.port):
        launcher.msg('Controller is already started')
    else:
        launcher.start_controller(options.port)

    launcher.load_config(args[2])

    launcher.run()

    return 0

def run_controller(args):
    parser = optparse.OptionParser()
    parser.add_option('-v', '--verbose',
                      action="store_true", dest="verbose",
                      help="Be verbose")
    group = optparse.OptionGroup(parser, "Controller options")
    group.add_option('-p', '--port',
                     action="store", type="int", dest="port",
                     default=8890,
                     help="Port to listen on [default 8890]")
    parser.add_option_group(group)

    options, args = parser.parse_args(args)

    if options.verbose:
        log.enableLogging()

    factory = ControllerServerFactory()

    log.msg('controller', 'Starting at port %d' % options.port)
    reactor.listenTCP(options.port, factory)
    reactor.run()

    return 0

def run_component(name, args):
    try:
        options = get_options_for(name, args)
    except errors.OptionError, e:
        print 'ERROR:', e
        raise SystemExit

    if name == 'producer':
        klass = Producer
        args = (options.name, options.sources, options.feeds, options.pipeline)
    elif name == 'converter':
        klass = Converter
        args = (options.name, options.sources, options.feeds, options.pipeline)
    elif name == 'streamer':
        if options.protocol == 'http':
            web_factory = server.Site(resource=StreamingResource(component))
            reactor.listenTCP(options.listen_port, web_factory)
            klass = streamer.FakeSinkStreamer
            args = (options.name, options.sources)
        elif options.protocol == 'file':
            klass = streamer.FileStreamer
            args = (options.name, options.sources, options.location)
        else:
            print 'Only http and file protcol supported right now'
            return
    else:
        raise AssertionError

    try:
        print klass, args
        component = klass(*args)
    except twisted.errors.PipelineParseError, e:
        print 'Bad pipeline: %s' % e
        raise SystemExit

    reactor.connectTCP(options.host, options.port, component.factory)

    reactor.run()

def usage():
    print 'Usage: flumotion command [command-options-and-arguments]'

def show_commands():
    print 'Flumotion commands are:'
    print '\tlauncher      Component launcher'
    print '\tcontroller    Component controller'
    print '\tproducer      Producer component'
    print '\tconverter     Converter component'
    print '\tstreamer      Streamer component'
    print
    print '(Specify the --help option for a list of other help options)'

def main(args):
    if len(args) < 2:
        usage()
        return -1

    name = args[1]
    if name == 'controller':
        return run_controller(args)
    if name in ['producer', 'converter', 'streamer']:
        return run_component(args[1], args[2:])
    elif name == 'launcher':
        return run_launcher(args)
    else:
        print "Unknown command: `%s'" % name
        print
        show_commands()
        return -1

    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>