<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A utility class to generate JSON results from given test results and upload
them to the specified results server.

"""

from __future__ import with_statement

import codecs
import logging
import os
import time

import simplejson
from slave.gtest.test_result import TestResult
from slave.gtest.test_results_uploader import TestResultsUploader

# A JSON results generator for generic tests.

JSON_PREFIX = 'ADD_RESULTS('
JSON_SUFFIX = ');'

def test_did_pass(test_result):
  return not test_result.failed and test_result.modifier == TestResult.NONE

def add_path_to_trie(path, value, trie):
  """Inserts a single flat directory path and associated value into a directory
  trie structure."""
  if not '/' in path:
    trie[path] = value
    return

  # we don't use slash
  # pylint: disable=W0612
  directory, slash, rest = path.partition('/')
  if not directory in trie:
    trie[directory] = {}
  add_path_to_trie(rest, value, trie[directory])

def generate_test_timings_trie(individual_test_timings):
  """Breaks a test name into chunks by directory and puts the test time as a
  value in the lowest part, e.g.
  foo/bar/baz.html: 1ms
  foo/bar/baz1.html: 3ms

  becomes
  foo: {
    bar: {
      baz.html: 1,
      baz1.html: 3
    }
  }
  """
  trie = {}
  for test_result in individual_test_timings:
    test = test_result.test_name

    add_path_to_trie(test, int(1000 * test_result.test_run_time), trie)

  return trie

class JSONResultsGenerator(object):
  """A JSON results generator for generic tests."""

  FAIL_LABEL = 'FAIL'
  PASS_LABEL = 'PASS'
  FLAKY_LABEL = ' '.join([FAIL_LABEL, PASS_LABEL])
  SKIP_LABEL = 'SKIP'

  ACTUAL = 'actual'
  BLINK_REVISION = 'blink_revision'
  BUILD_NUMBER = 'build_number'
  BUILDER_NAME = 'builder_name'
  CHROMIUM_REVISION = 'chromium_revision'
  EXPECTED = 'expected'
  FAILURE_SUMMARY = 'num_failures_by_type'
  SECONDS_SINCE_EPOCH = 'seconds_since_epoch'
  TEST_TIME = 'time'
  TESTS = 'tests'
  VERSION = 'version'
  VERSION_NUMBER = 3

  RESULTS_FILENAME = 'results.json'
  TIMES_MS_FILENAME = 'times_ms.json'
  INCREMENTAL_RESULTS_FILENAME = 'incremental_results.json'

  def __init__(self, builder_name, build_name, build_number,
               results_file_base_path, builder_base_url,
               test_results_map, svn_revisions=None,
               test_results_server=None,
               test_type='',
               master_name='',
               file_writer=None):
    """Modifies the results.json file. Grabs it off the archive directory
    if it is not found locally.

    Args
      builder_name: the builder name (e.g. Webkit).
      build_name: the build name (e.g. webkit-rel).
      build_number: the build number.
      results_file_base_path: Absolute path to the directory containing the
        results json file.
      builder_base_url: the URL where we have the archived test results.
        If this is None no archived results will be retrieved.
      test_results_map: A dictionary that maps test_name to TestResult.
      svn_revisions: A (json_field_name, revision) pair for SVN
        repositories that tests rely on.  The SVN revision will be
        included in the JSON with the given json_field_name.
      test_results_server: server that hosts test results json.
      test_type: test type string (e.g. 'layout-tests').
      master_name: the name of the buildbot master.
      file_writer: if given the parameter is used to write JSON data to a file.
        The parameter must be the function that takes two arguments, 'file_path'
        and 'data' to be written into the file_path.
    """
    self._builder_name = builder_name
    self._build_name = build_name
    self._build_number = build_number
    self._builder_base_url = builder_base_url
    self._results_directory = results_file_base_path

    self._test_results_map = test_results_map
    self._test_results = test_results_map.values()

    self._svn_revisions = svn_revisions
    if not self._svn_revisions:
      self._svn_revisions = {}

    self._test_results_server = test_results_server
    self._test_type = test_type
    self._master_name = master_name
    self._file_writer = file_writer

  def generate_json_output(self):
    json = self.get_full_results_json()
    if json:
      file_path = os.path.join(self._results_directory,
                               self.INCREMENTAL_RESULTS_FILENAME)
      self._write_json(json, file_path)

  def generate_times_ms_file(self):
    times = generate_test_timings_trie(self._test_results_map.values())
    file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME)
    self._write_json(times, file_path)

  def get_full_results_json(self):
    results = {self.VERSION: self.VERSION_NUMBER}

    # Metadata generic to all results.
    results[self.BUILDER_NAME] = self._builder_name
    results[self.BUILD_NUMBER] = self._build_number
    results[self.SECONDS_SINCE_EPOCH] = int(time.time())
    for name, revision in self._svn_revisions:
      results[name + '_revision'] = revision

    tests = results.setdefault(self.TESTS, {})
    for test_name in self._test_results_map.iterkeys():
      tests[test_name] = self._make_test_data(test_name)

    self._insert_failure_map(results)

    return results

  def _insert_failure_map(self, results):
    # FAIL, PASS, NOTRUN
    summary = {self.PASS_LABEL: 0, self.FAIL_LABEL: 0, self.SKIP_LABEL: 0}
    for result in self._test_results_map.itervalues():
      if test_did_pass(result):
        summary[self.PASS_LABEL] += 1
      elif result.modifier == TestResult.DISABLED:
        summary[self.SKIP_LABEL] += 1
      elif result.failed:
        summary[self.FAIL_LABEL] += 1

    results[self.FAILURE_SUMMARY] = summary

  def _make_test_data(self, test_name):
    test_data = {}
    expected, actual = self._get_expected_and_actual_results(test_name)
    test_data[self.EXPECTED] = expected
    test_data[self.ACTUAL] = actual
    run_time = int(self._test_results_map[test_name].test_run_time)
    test_data[self.TEST_TIME] = run_time

    return test_data

  def _get_expected_and_actual_results(self, test_name):
    test_result = self._test_results_map[test_name]
    modifier = test_result.modifier

    if modifier == TestResult.DISABLED:
      return (self.SKIP_LABEL, self.SKIP_LABEL)

    actual = self.FAIL_LABEL if test_result.failed else self.PASS_LABEL
    if modifier == TestResult.NONE:
      return (self.PASS_LABEL, actual)
    if modifier == TestResult.FLAKY:
      return (self.FLAKY_LABEL, actual)
    if modifier == TestResult.FAILS:
      return (self.FAIL_LABEL, actual)

  def upload_json_files(self, json_files):
    """Uploads the given json_files to the test_results_server (if the
    test_results_server is given)."""
    if not self._test_results_server:
      return

    if not self._master_name:
      logging.error('--test-results-server was set, but --master-name was not. '
                    'Not uploading JSON files.')
      return

    print 'Uploading JSON files for builder: %s' % self._builder_name
    attrs = [('builder', self._builder_name),
             ('testtype', self._test_type),
             ('master', self._master_name)]

    files = [(f, os.path.join(self._results_directory, f)) for f in json_files]

    uploader = TestResultsUploader(self._test_results_server)
    # Set uploading timeout in case appengine server is having problem.
    # 120 seconds are more than enough to upload test results.
    uploader.upload(attrs, files, 120)

    print 'JSON files uploaded.'

  def _write_json(self, json_object, file_path):
    # Specify separators in order to get compact encoding.
    json_data = simplejson.dumps(json_object, separators=(',', ':'))
    json_string = json_data
    if self._file_writer:
      self._file_writer(file_path, json_string)
    else:
      with codecs.open(file_path, 'w', 'utf8') as f:
        f.write(json_string)
<commit_msg>Fix json file name.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""A utility class to generate JSON results from given test results and upload
them to the specified results server.

"""

from __future__ import with_statement

import codecs
import logging
import os
import time

import simplejson
from slave.gtest.test_result import TestResult
from slave.gtest.test_results_uploader import TestResultsUploader

# A JSON results generator for generic tests.

JSON_PREFIX = 'ADD_RESULTS('
JSON_SUFFIX = ');'

def test_did_pass(test_result):
  return not test_result.failed and test_result.modifier == TestResult.NONE

def add_path_to_trie(path, value, trie):
  """Inserts a single flat directory path and associated value into a directory
  trie structure."""
  if not '/' in path:
    trie[path] = value
    return

  # we don't use slash
  # pylint: disable=W0612
  directory, slash, rest = path.partition('/')
  if not directory in trie:
    trie[directory] = {}
  add_path_to_trie(rest, value, trie[directory])

def generate_test_timings_trie(individual_test_timings):
  """Breaks a test name into chunks by directory and puts the test time as a
  value in the lowest part, e.g.
  foo/bar/baz.html: 1ms
  foo/bar/baz1.html: 3ms

  becomes
  foo: {
    bar: {
      baz.html: 1,
      baz1.html: 3
    }
  }
  """
  trie = {}
  for test_result in individual_test_timings:
    test = test_result.test_name

    add_path_to_trie(test, int(1000 * test_result.test_run_time), trie)

  return trie

class JSONResultsGenerator(object):
  """A JSON results generator for generic tests."""

  FAIL_LABEL = 'FAIL'
  PASS_LABEL = 'PASS'
  FLAKY_LABEL = ' '.join([FAIL_LABEL, PASS_LABEL])
  SKIP_LABEL = 'SKIP'

  ACTUAL = 'actual'
  BLINK_REVISION = 'blink_revision'
  BUILD_NUMBER = 'build_number'
  BUILDER_NAME = 'builder_name'
  CHROMIUM_REVISION = 'chromium_revision'
  EXPECTED = 'expected'
  FAILURE_SUMMARY = 'num_failures_by_type'
  SECONDS_SINCE_EPOCH = 'seconds_since_epoch'
  TEST_TIME = 'time'
  TESTS = 'tests'
  VERSION = 'version'
  VERSION_NUMBER = 3

  RESULTS_FILENAME = 'results.json'
  TIMES_MS_FILENAME = 'times_ms.json'
  FULL_RESULTS_FILENAME = 'full_results.json'

  def __init__(self, builder_name, build_name, build_number,
               results_file_base_path, builder_base_url,
               test_results_map, svn_revisions=None,
               test_results_server=None,
               test_type='',
               master_name='',
               file_writer=None):
    """Modifies the results.json file. Grabs it off the archive directory
    if it is not found locally.

    Args
      builder_name: the builder name (e.g. Webkit).
      build_name: the build name (e.g. webkit-rel).
      build_number: the build number.
      results_file_base_path: Absolute path to the directory containing the
        results json file.
      builder_base_url: the URL where we have the archived test results.
        If this is None no archived results will be retrieved.
      test_results_map: A dictionary that maps test_name to TestResult.
      svn_revisions: A (json_field_name, revision) pair for SVN
        repositories that tests rely on.  The SVN revision will be
        included in the JSON with the given json_field_name.
      test_results_server: server that hosts test results json.
      test_type: test type string (e.g. 'layout-tests').
      master_name: the name of the buildbot master.
      file_writer: if given the parameter is used to write JSON data to a file.
        The parameter must be the function that takes two arguments, 'file_path'
        and 'data' to be written into the file_path.
    """
    self._builder_name = builder_name
    self._build_name = build_name
    self._build_number = build_number
    self._builder_base_url = builder_base_url
    self._results_directory = results_file_base_path

    self._test_results_map = test_results_map
    self._test_results = test_results_map.values()

    self._svn_revisions = svn_revisions
    if not self._svn_revisions:
      self._svn_revisions = {}

    self._test_results_server = test_results_server
    self._test_type = test_type
    self._master_name = master_name
    self._file_writer = file_writer

  def generate_json_output(self):
    json = self.get_full_results_json()
    if json:
      file_path = os.path.join(self._results_directory,
                               self.FULL_RESULTS_FILENAME)
      self._write_json(json, file_path)

  def generate_times_ms_file(self):
    times = generate_test_timings_trie(self._test_results_map.values())
    file_path = os.path.join(self._results_directory, self.TIMES_MS_FILENAME)
    self._write_json(times, file_path)

  def get_full_results_json(self):
    results = {self.VERSION: self.VERSION_NUMBER}

    # Metadata generic to all results.
    results[self.BUILDER_NAME] = self._builder_name
    results[self.BUILD_NUMBER] = self._build_number
    results[self.SECONDS_SINCE_EPOCH] = int(time.time())
    for name, revision in self._svn_revisions:
      results[name + '_revision'] = revision

    tests = results.setdefault(self.TESTS, {})
    for test_name in self._test_results_map.iterkeys():
      tests[test_name] = self._make_test_data(test_name)

    self._insert_failure_map(results)

    return results

  def _insert_failure_map(self, results):
    # FAIL, PASS, NOTRUN
    summary = {self.PASS_LABEL: 0, self.FAIL_LABEL: 0, self.SKIP_LABEL: 0}
    for result in self._test_results_map.itervalues():
      if test_did_pass(result):
        summary[self.PASS_LABEL] += 1
      elif result.modifier == TestResult.DISABLED:
        summary[self.SKIP_LABEL] += 1
      elif result.failed:
        summary[self.FAIL_LABEL] += 1

    results[self.FAILURE_SUMMARY] = summary

  def _make_test_data(self, test_name):
    test_data = {}
    expected, actual = self._get_expected_and_actual_results(test_name)
    test_data[self.EXPECTED] = expected
    test_data[self.ACTUAL] = actual
    run_time = int(self._test_results_map[test_name].test_run_time)
    test_data[self.TEST_TIME] = run_time

    return test_data

  def _get_expected_and_actual_results(self, test_name):
    test_result = self._test_results_map[test_name]
    modifier = test_result.modifier

    if modifier == TestResult.DISABLED:
      return (self.SKIP_LABEL, self.SKIP_LABEL)

    actual = self.FAIL_LABEL if test_result.failed else self.PASS_LABEL
    if modifier == TestResult.NONE:
      return (self.PASS_LABEL, actual)
    if modifier == TestResult.FLAKY:
      return (self.FLAKY_LABEL, actual)
    if modifier == TestResult.FAILS:
      return (self.FAIL_LABEL, actual)

  def upload_json_files(self, json_files):
    """Uploads the given json_files to the test_results_server (if the
    test_results_server is given)."""
    if not self._test_results_server:
      return

    if not self._master_name:
      logging.error('--test-results-server was set, but --master-name was not. '
                    'Not uploading JSON files.')
      return

    print 'Uploading JSON files for builder: %s' % self._builder_name
    attrs = [('builder', self._builder_name),
             ('testtype', self._test_type),
             ('master', self._master_name)]

    files = [(f, os.path.join(self._results_directory, f)) for f in json_files]

    uploader = TestResultsUploader(self._test_results_server)
    # Set uploading timeout in case appengine server is having problem.
    # 120 seconds are more than enough to upload test results.
    uploader.upload(attrs, files, 120)

    print 'JSON files uploaded.'

  def _write_json(self, json_object, file_path):
    # Specify separators in order to get compact encoding.
    json_data = simplejson.dumps(json_object, separators=(',', ':'))
    json_string = json_data
    if self._file_writer:
      self._file_writer(file_path, json_string)
    else:
      with codecs.open(file_path, 'w', 'utf8') as f:
        f.write(json_string)
<|endoftext|>


<commit_before>import io
import re

from django.test import TestCase
from django.core import management

EXPECTED_CHANGES = {
    'aldryn_faq': [
        # Py3K-izing default=b'{}', which is interpreted with encoding UTF-8:
        # migrations.AlterField(
        #     model_name='faqconfig',
        #     name='app_data',
        #     field=app_data.fields.AppDataField(default='{}', editable=False),
        #     preserve_default=True,
        # ),
        '- Alter field app_data on faqconfig',
    ],
    'aldryn_newsblog': [
        # newsblogconfig.app_data is same story as faqconfig.app_data above
        # migrations.AlterField(
        #     model_name='newsblogconfig',
        #     name='app_data',
        #     field=app_data.fields.AppDataField(default='{}', editable=False),
        #     preserve_default=True,
        # ),
        '- Alter field app_data on newsblogconfig',
        # Removing choices since choices has a different value since the last
        # time (now: [], before: [(b'dummy', b'dummy')])
        # migrations.AlterField(
        #     model_name='newsblogconfig',
        #     name='template_prefix',
        #     field=models.CharField(blank=True, verbose_name='Prefix for template dirs', max_length=20, null=True),
        #     preserve_default=True,
        # ),
        '- Alter field template_prefix on newsblogconfig',
    ],
}

class MigrationCommandOutput(object):
    """Parse output from makemigrations command, generating a series of
      (module, changed_fields)"""

    INITIAL = 'INITIAL'
    READING = 'READING'  # reading changes for package

    def __init__(self, input_file):
        self.input = input_file

    def read(self):
        actual_changes = []
        module = ''
        state = self.INITIAL
        while True:
            l = self.input.readline()
            if l in ('', 'No changes detected'):
                break
            if l.startswith('Migrations'):
                if state != self.INITIAL:
                    yield module, actual_changes
                state = self.READING
                module = re.match(r'^Migrations for \'(.+)\'', l).group(1)
                actual_changes = []
            elif re.search(r'^ +.*\.py:', l):
                continue
            else:
                if state != self.READING:
                    raise ValueError('State is <%s> instead of <%s>' % (
                        state, self.READING
                    ))
                actual_changes.append(l.strip())
        if state != self.INITIAL:
            yield module, actual_changes

class MissingMigrationTestCase(TestCase):

    def test(self):
        actual_modules = []
        with io.StringIO() as f:
            management.call_command('makemigrations', dry_run=True, stdout=f, no_color=True)
            f.seek(0)
            migrations = MigrationCommandOutput(f)
            for module, actual_changes in migrations.read():
                actual_modules.append(module)
                self.assertEqual(EXPECTED_CHANGES[module], actual_changes)
        self.assertEqual(sorted(EXPECTED_CHANGES.keys()), sorted(actual_modules))
<commit_msg>line length fix<commit_after>import io
import re

from django.test import TestCase
from django.core import management

EXPECTED_CHANGES = {
    'aldryn_faq': [
        # Py3K-izing default=b'{}', which is interpreted with encoding UTF-8:
        # migrations.AlterField(
        #     model_name='faqconfig',
        #     name='app_data',
        #     field=app_data.fields.AppDataField(default='{}', editable=False),
        #     preserve_default=True,
        # ),
        '- Alter field app_data on faqconfig',
    ],
    'aldryn_newsblog': [
        # newsblogconfig.app_data is same story as faqconfig.app_data above
        # migrations.AlterField(
        #     model_name='newsblogconfig',
        #     name='app_data',
        #     field=app_data.fields.AppDataField(default='{}', editable=False),
        #     preserve_default=True,
        # ),
        '- Alter field app_data on newsblogconfig',
        # Removing choices since choices has a different value since the last
        # time (now: [], before: [(b'dummy', b'dummy')])
        # migrations.AlterField(
        #     model_name='newsblogconfig',
        #     name='template_prefix',
        #     field=models.CharField(blank=True, verbose_name='Prefix for template dirs',
        #         max_length=20, null=True),
        #     preserve_default=True,
        # ),
        '- Alter field template_prefix on newsblogconfig',
    ],
}

class MigrationCommandOutput(object):
    """Parse output from makemigrations command, generating a series of
      (module, changed_fields)"""

    INITIAL = 'INITIAL'
    READING = 'READING'  # reading changes for package

    def __init__(self, input_file):
        self.input = input_file

    def read(self):
        actual_changes = []
        module = ''
        state = self.INITIAL
        while True:
            l = self.input.readline()
            if l in ('', 'No changes detected'):
                break
            if l.startswith('Migrations'):
                if state != self.INITIAL:
                    yield module, actual_changes
                state = self.READING
                module = re.match(r'^Migrations for \'(.+)\'', l).group(1)
                actual_changes = []
            elif re.search(r'^ +.*\.py:', l):
                continue
            else:
                if state != self.READING:
                    raise ValueError('State is <%s> instead of <%s>' % (
                        state, self.READING
                    ))
                actual_changes.append(l.strip())
        if state != self.INITIAL:
            yield module, actual_changes

class MissingMigrationTestCase(TestCase):

    def test(self):
        actual_modules = []
        with io.StringIO() as f:
            management.call_command('makemigrations', dry_run=True, stdout=f, no_color=True)
            f.seek(0)
            migrations = MigrationCommandOutput(f)
            for module, actual_changes in migrations.read():
                actual_modules.append(module)
                self.assertEqual(EXPECTED_CHANGES[module], actual_changes)
        self.assertEqual(sorted(EXPECTED_CHANGES.keys()), sorted(actual_modules))
<|endoftext|>


<commit_before>import django.views.static
from django.conf import settings
from django.conf.urls import include, url
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.views.generic import TemplateView

admin.autodiscover()

urlpatterns = [
    url(r'^$', TemplateView.as_view(template_name='base.html')),
    url(r'^admin/', admin.site.urls),
    {%- if cookiecutter.use_djangocms == 'y' %}
    url(r'^', include('cms.urls')),
    {%- endif %}
]

if settings.DEBUG:
    import debug_toolbar
    urlpatterns = [
        url(r'^media/(?P<path>.*)$', django.views.static.serve,
            {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
        url(r'^__debug__/', include(debug_toolbar.urls)),
    ] + staticfiles_urlpatterns() + urlpatterns
<commit_msg>Use `path` helper for URLConf<commit_after>import django.views.static
from django.conf import settings
from django.urls import include, path
from django.contrib import admin
from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.views.generic import TemplateView

admin.autodiscover()

urlpatterns = [
    path('', TemplateView.as_view(template_name='base.html')),
    path('admin/', admin.site.urls),
    {%- if cookiecutter.use_djangocms == 'y' %}
    path('', include('cms.urls')),
    {%- endif %}
]

if settings.DEBUG:
    import debug_toolbar
    urlpatterns = [
        path('media/<path:path>/', django.views.static.serve,
            {'document_root': settings.MEDIA_ROOT, 'show_indexes': True}),
        path('__debug__/', include(debug_toolbar.urls)),
    ] + staticfiles_urlpatterns() + urlpatterns
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, division
from armet.authorization import ManagedAuthorization
from armet.exceptions import ImproperlyConfigured

try:
    import shield
    from shield.contrib.sqlalchemy import evaluate

except ImportError:
    shield = None

class ShieldAuthorization(ManagedAuthorization):
    """Implements managed authorization using the shield permissions framework.
    """

    def __init__(self, permissions=None):
        # Ensure we have access to the shield framework.
        if shield is None:
            raise ImproperlyConfigured(
                "'shield' is required to use 'ShieldAuthorization'")

        #! Permissions to check for each operation.
        self.permissions = permissions
        if self.permissions is None:
            self.permissions = {
                'read': ('read',),
                'create': ('create',),
                'update': ('update',),
                'destroy': ('destroy',)
            }

    def is_authorized(self, user, operation, resource, item):
        expr = shield.has(
            *self.permissions[operation], bearer=user, target=item)
        return evaluate(expr, item)

    def filter(self, user, operation, resource, iterable):
        expr = shield.filter(
            *self.permissions[operation],
            bearer=user, target=resource.meta.model)

        clause = evaluate(expr, resource.meta.model)
        return resource.filter(clause, iterable)
<commit_msg>Preliminary shield fixes [ci skip]<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals, division
from armet.authorization import ManagedAuthorization
from armet.exceptions import ImproperlyConfigured

try:
    import shield

except ImportError:
    shield = None

class ShieldAuthorization(ManagedAuthorization):
    """Implements managed authorization using the shield permissions framework.
    """

    def __init__(self, permissions=None):
        # Ensure we have access to the shield framework.
        if shield is None:
            raise ImproperlyConfigured(
                "'shield' is required to use 'ShieldAuthorization'")

        #! Permissions to check for each operation.
        self.permissions = permissions
        if self.permissions is None:
            self.permissions = {
                'read': ('read',),
                'create': ('create',),
                'update': ('update',),
                'destroy': ('destroy',)
            }

    def is_authorized(self, user, operation, resource, item):
        return shield.has(
            *self.permissions[operation],
            bearer=user,
            target=item,
            session=self.session)

    def filter(self, user, operation, resource, iterable):
        query = shield.filter(
            *self.permissions[operation],
            bearer=user,
            target=resource.meta.model,
            session=self.session)

        return resource.filter(query, iterable)
<|endoftext|>


<commit_before>import logging

from cloudfoundry_client.calls import caller, OutputFormat
from cloudfoundry_client.credentials import CredentialsManager
from cloudfoundry_client.v2.organizations import OrganizationManager
from cloudfoundry_client.v2.spaces import SpaceManager
from cloudfoundry_client.v2.services import ServiceManager
from cloudfoundry_client.v2.service_plans import ServicePlanManager
from cloudfoundry_client.v2.service_instances import ServiceInstanceManager
from cloudfoundry_client.v2.service_bindings import ServiceBindingManager
from cloudfoundry_client.v2.service_brokers import ServiceBrokerManager
from cloudfoundry_client.v2.applications import ApplicationsManager
from cloudfoundry_client.loggregator.loggregator import LoggregatorManager

_logger = logging.getLogger(__name__)

class InvalidStatusCode(Exception):
    def __init__(self, status_code):
        self.status_code = status_code

class CloudFoundryClient(object):
    proxy = None

    def __init__(self, target_endpoint, client_id='cf', client_secret='', proxy=None, skip_verification=False):
        caller.proxy(proxy)
        caller.skip_verifications(skip_verification)
        info = caller.get('%s/info' % target_endpoint, output_format=OutputFormat.JSON)
        if info['version'] != 2:
            raise AssertionError('Only version 2 is supported for now. Found %d' % info['version'])
        # to get loggregator url
        info = caller.get('%s/v2/info' % target_endpoint, output_format=OutputFormat.JSON)
        self.credentials_manager = CredentialsManager(info['authorization_endpoint'], client_id, client_secret)
        self.organization = OrganizationManager(target_endpoint, self.credentials_manager)
        self.space = SpaceManager(target_endpoint, self.credentials_manager)
        self.service = ServiceManager(target_endpoint, self.credentials_manager)
        self.service_plan = ServicePlanManager(target_endpoint, self.credentials_manager)
        self.service_instance = ServiceInstanceManager(target_endpoint, self.credentials_manager)
        self.service_binding = ServiceBindingManager(target_endpoint, self.credentials_manager)
        self.service_broker = ServiceBrokerManager(target_endpoint, self.credentials_manager)
        self.application = ApplicationsManager(target_endpoint, self.credentials_manager)
        self.loggregator = LoggregatorManager(info['logging_endpoint'], self.credentials_manager)

    def init_with_credentials(self, login, password):
        self.credentials_manager.init_with_credentials(login, password)

    def init_with_refresh(self, refresh_token):
        self.credentials_manager.init_with_refresh(refresh_token)

<commit_msg>Look at /v2/info rather than /info to get the api version<commit_after>import logging

from cloudfoundry_client.calls import caller, OutputFormat
from cloudfoundry_client.credentials import CredentialsManager
from cloudfoundry_client.v2.organizations import OrganizationManager
from cloudfoundry_client.v2.spaces import SpaceManager
from cloudfoundry_client.v2.services import ServiceManager
from cloudfoundry_client.v2.service_plans import ServicePlanManager
from cloudfoundry_client.v2.service_instances import ServiceInstanceManager
from cloudfoundry_client.v2.service_bindings import ServiceBindingManager
from cloudfoundry_client.v2.service_brokers import ServiceBrokerManager
from cloudfoundry_client.v2.applications import ApplicationsManager
from cloudfoundry_client.loggregator.loggregator import LoggregatorManager

_logger = logging.getLogger(__name__)

class InvalidStatusCode(Exception):
    def __init__(self, status_code):
        self.status_code = status_code

class CloudFoundryClient(object):
    proxy = None

    def __init__(self, target_endpoint, client_id='cf', client_secret='', proxy=None, skip_verification=False):
        caller.proxy(proxy)
        caller.skip_verifications(skip_verification)
        # to get loggregator url
        info = caller.get('%s/v2/info' % target_endpoint, output_format=OutputFormat.JSON)
        if not info['api_version'].startswith('2.'):
            raise AssertionError('Only version 2 is supported for now. Found %s' % info['api_version'])

        self.credentials_manager = CredentialsManager(info['authorization_endpoint'], client_id, client_secret)
        self.organization = OrganizationManager(target_endpoint, self.credentials_manager)
        self.space = SpaceManager(target_endpoint, self.credentials_manager)
        self.service = ServiceManager(target_endpoint, self.credentials_manager)
        self.service_plan = ServicePlanManager(target_endpoint, self.credentials_manager)
        self.service_instance = ServiceInstanceManager(target_endpoint, self.credentials_manager)
        self.service_binding = ServiceBindingManager(target_endpoint, self.credentials_manager)
        self.service_broker = ServiceBrokerManager(target_endpoint, self.credentials_manager)
        self.application = ApplicationsManager(target_endpoint, self.credentials_manager)
        self.loggregator = LoggregatorManager(info['logging_endpoint'], self.credentials_manager)

    def init_with_credentials(self, login, password):
        self.credentials_manager.init_with_credentials(login, password)

    def init_with_refresh(self, refresh_token):
        self.credentials_manager.init_with_refresh(refresh_token)

<|endoftext|>


<commit_before>import os
import faiss
import argparse
from multiprocessing.dummy import Pool as ThreadPool

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('--inputs', nargs='*', required=True,
                        help='input indexes to merge')
    parser.add_argument('--l0', type=int, default=0)
    parser.add_argument('--l1', type=int, default=-1)

    parser.add_argument('--nt', default=-1,
                        help='nb threads')

    parser.add_argument('--output', required=True,
                        help='output index filename')
    parser.add_argument('--outputIL',
                        help='output invfile filename')

    args = parser.parse_args()

    if args.nt != -1:
        print('set nb of threads to', args.nt)

    ils = faiss.InvertedListsPtrVector()
    ils_dont_dealloc = []

    pool = ThreadPool(20)

    def load_index(fname):
        print("loading", fname)
        try:
            index = faiss.read_index(fname, faiss.IO_FLAG_MMAP | faiss.IO_FLAG_READ_ONLY)
        except RuntimeError as e:
            print('could not load %s: %s' % (fname, e))
            return fname, None

        print("  %d entries" % index.ntotal)
        return fname, index

    index0 = None

    for fname, index in pool.imap(load_index, args.inputs):
        if index is None:
            continue
        index_ivf = faiss.extract_index_ivf(index)
        il = faiss.downcast_InvertedLists(index_ivf.invlists)
        index_ivf.invlists = None
        il.this.own()
        ils_dont_dealloc.append(il)
        if (args.l0, args.l1) != (0, -1):
            print('restricting to lists %d:%d' % (args.l0, args.l1))
            # il = faiss.SliceInvertedLists(il, args.l0, args.l1)

            il.crop_invlists(args.l0, args.l1)
            ils_dont_dealloc.append(il)
        ils.push_back(il)

        if index0 is None:
            index0 = index

    print("loaded %d invlists" % ils.size())

    if not args.outputIL:
        args.outputIL = args.output + '_invlists'

    il0 = ils.at(0)

    il = faiss.OnDiskInvertedLists(
        il0.nlist, il0.code_size,
        args.outputIL)

    print("perform merge")

    ntotal = il.merge_from(ils.data(), ils.size(), True)

    print("swap into index0")

    index0_ivf = faiss.extract_index_ivf(index0)
    index0_ivf.nlist = il0.nlist
    index0_ivf.ntotal = index0.ntotal = ntotal
    index0_ivf.invlists = il
    index0_ivf.own_invlists = False

    print("write", args.output)

    faiss.write_index(index0, args.output)
<commit_msg>Update merge_to_ondisk.py<commit_after># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import os
import faiss
import argparse
from multiprocessing.dummy import Pool as ThreadPool

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('--inputs', nargs='*', required=True,
                        help='input indexes to merge')
    parser.add_argument('--l0', type=int, default=0)
    parser.add_argument('--l1', type=int, default=-1)

    parser.add_argument('--nt', default=-1,
                        help='nb threads')

    parser.add_argument('--output', required=True,
                        help='output index filename')
    parser.add_argument('--outputIL',
                        help='output invfile filename')

    args = parser.parse_args()

    if args.nt != -1:
        print('set nb of threads to', args.nt)

    ils = faiss.InvertedListsPtrVector()
    ils_dont_dealloc = []

    pool = ThreadPool(20)

    def load_index(fname):
        print("loading", fname)
        try:
            index = faiss.read_index(fname, faiss.IO_FLAG_MMAP | faiss.IO_FLAG_READ_ONLY)
        except RuntimeError as e:
            print('could not load %s: %s' % (fname, e))
            return fname, None

        print("  %d entries" % index.ntotal)
        return fname, index

    index0 = None

    for fname, index in pool.imap(load_index, args.inputs):
        if index is None:
            continue
        index_ivf = faiss.extract_index_ivf(index)
        il = faiss.downcast_InvertedLists(index_ivf.invlists)
        index_ivf.invlists = None
        il.this.own()
        ils_dont_dealloc.append(il)
        if (args.l0, args.l1) != (0, -1):
            print('restricting to lists %d:%d' % (args.l0, args.l1))
            # il = faiss.SliceInvertedLists(il, args.l0, args.l1)

            il.crop_invlists(args.l0, args.l1)
            ils_dont_dealloc.append(il)
        ils.push_back(il)

        if index0 is None:
            index0 = index

    print("loaded %d invlists" % ils.size())

    if not args.outputIL:
        args.outputIL = args.output + '_invlists'

    il0 = ils.at(0)

    il = faiss.OnDiskInvertedLists(
        il0.nlist, il0.code_size,
        args.outputIL)

    print("perform merge")

    ntotal = il.merge_from(ils.data(), ils.size(), True)

    print("swap into index0")

    index0_ivf = faiss.extract_index_ivf(index0)
    index0_ivf.nlist = il0.nlist
    index0_ivf.ntotal = index0.ntotal = ntotal
    index0_ivf.invlists = il
    index0_ivf.own_invlists = False

    print("write", args.output)

    faiss.write_index(index0, args.output)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Compare current specter extraction to a reference run, reporting differences
"""

from __future__ import absolute_import, division, print_function
import sys, os
import time
import numpy as np

from specter.extract import ex2d
import specter.psf

def get_current_results():
    thisdir = os.path.split(os.path.abspath(__file__))[0]
    psffile = thisdir + '/../etc/psfnight-r0.fits'
    psf = specter.psf.load_psf(psffile)

    np.random.seed(0)
    ny, nx = psf.npix_y, psf.npix_x
    image = np.random.normal(loc=0, scale=1, size=(ny,nx))
    imageivar = np.ones_like(image)

    #- Spectra and wavelengths to extract
    nspec = 10
    wave = np.arange(psf.wmin_all, psf.wmin_all+200, 1)

    #- Wake up code
    flux, ivar, R = ex2d(image, imageivar, psf, 0, nspec, wave)

    #- Now do it for real
    t0 = time.time()
    flux, ivar, R = ex2d(image, imageivar, psf, 0, nspec, wave, bundlesize=nspec)
    runtime = time.time() - t0

    results = dict(flux=flux, ivar=ivar, R=R, runtime=runtime, nspec=nspec, wave=wave)
    return results

#-------------------------------------------------------------------------
import optparse

'''
Write a reference file with current extraction results:

    compare-extract.py --output ref1.npz

Compare saved results to current results (without saving current results):

    compare-extract.py ref1.npz

Compare saved results to current results and save current results:

    compare-extract.py ref1.npz --output ref2.npz

Compare two previously saved reference results:

    compare-extract.py ref1.npz ref2.npz
'''

parser = optparse.OptionParser(usage = "%prog [options]")
# parser.add_option("-i", "--input", type=str,  help="input data")
parser.add_option("-o", "--output", type=str,  help="output results for current extractions")
# parser.add_option("-v", "--verbose", action="store_true", help="some flag")

opts, reffiles = parser.parse_args()

if len(reffiles) == 0:
    assert opts.output is not None
    results = get_current_results()
    print('Saving results to {}'.format(opts.output))
    np.savez(opts.output, **results)
    sys.exit(0)
elif len(reffiles) == 1:
    ref1 = np.load(reffiles[0])
    ref2 = get_current_results()
    if opts.output is not None:
        print('Saving results to {}'.format(opts.output))
        np.savez(opts.output, **ref2)
else:
    ref1 = np.load(reffiles[0])
    ref2 = np.load(reffiles[1])

if ref1['nspec'] != ref2['nspec'] or \
    not np.all(ref1['wave'] == ref2['wave']) \
    or ref1['flux'].shape != ref2['flux'].shape:
        raise RuntimeError('Different extraction parameters')

dt = 100*(ref1['runtime'] - ref2['runtime']) / ref1['runtime']
if np.abs(dt) < 1:
    print('Same runtime')
elif dt < 0:
    print('Runtime {:.0f}% slower'.format(-dt))
else:
    print('Runtime {:.0f}% faster'.format(dt))

print('Comparing results:')
print('key   exact  close')
for key in ['flux', 'ivar', 'R']:
    print('{:4s}  {:5s}  {:5s}'.format(
        key, str(np.all(ref1[key] == ref2[key])),
        str(np.allclose(ref1[key], ref2[key]))
        ))

<commit_msg>wake up but avoid caching<commit_after>#!/usr/bin/env python

"""
Compare current specter extraction to a reference run, reporting differences
"""

from __future__ import absolute_import, division, print_function
import sys, os
import time
import numpy as np

from specter.extract import ex2d
import specter.psf

def get_current_results():
    thisdir = os.path.split(os.path.abspath(__file__))[0]
    psffile = thisdir + '/../etc/psfnight-r0.fits'
    psf = specter.psf.load_psf(psffile)

    np.random.seed(0)
    ny, nx = psf.npix_y, psf.npix_x
    image = np.random.normal(loc=0, scale=1, size=(ny,nx))
    imageivar = np.ones_like(image)

    #- Spectra and wavelengths to extract
    nspec = 10
    nwave = 1000
    wave = np.arange(psf.wmin_all, psf.wmin_all+nwave, 1)

    #- Wake up code, but using a different spectral range
    flux, ivar, R = ex2d(image, imageivar, psf, nspec, nspec, wave[0:10])

    #- Now do it for real
    t0 = time.time()
    flux, ivar, R = ex2d(image, imageivar, psf, 0, nspec, wave, bundlesize=nspec)
    runtime = time.time() - t0

    results = dict(flux=flux, ivar=ivar, R=R, runtime=runtime, nspec=nspec, wave=wave)
    return results

#-------------------------------------------------------------------------
import optparse

'''
Write a reference file with current extraction results:

    compare-extract.py --output ref1.npz

Compare saved results to current results (without saving current results):

    compare-extract.py ref1.npz

Compare saved results to current results and save current results:

    compare-extract.py ref1.npz --output ref2.npz

Compare two previously saved reference results:

    compare-extract.py ref1.npz ref2.npz
'''

parser = optparse.OptionParser(usage = "%prog [options]")
# parser.add_option("-i", "--input", type=str,  help="input data")
parser.add_option("-o", "--output", type=str,  help="output results for current extractions")
# parser.add_option("-v", "--verbose", action="store_true", help="some flag")

opts, reffiles = parser.parse_args()

if len(reffiles) == 0:
    assert opts.output is not None
    results = get_current_results()
    print('Saving results to {}'.format(opts.output))
    np.savez(opts.output, **results)
    sys.exit(0)
elif len(reffiles) == 1:
    ref1 = np.load(reffiles[0])
    ref2 = get_current_results()
    if opts.output is not None:
        print('Saving results to {}'.format(opts.output))
        np.savez(opts.output, **ref2)
else:
    ref1 = np.load(reffiles[0])
    ref2 = np.load(reffiles[1])

if ref1['nspec'] != ref2['nspec'] or \
    not np.all(ref1['wave'] == ref2['wave']) \
    or ref1['flux'].shape != ref2['flux'].shape:
        raise RuntimeError('Different extraction parameters')

dt = 100*(ref1['runtime'] - ref2['runtime']) / ref1['runtime']
if np.abs(dt) < 1:
    print('Same runtime')
elif dt < 0:
    print('Runtime {:.0f}% slower'.format(-dt))
else:
    print('Runtime {:.0f}% faster'.format(dt))

print('Comparing results:')
print('key   exact  close')
for key in ['flux', 'ivar', 'R']:
    print('{:4s}  {:5s}  {:5s}'.format(
        key, str(np.all(ref1[key] == ref2[key])),
        str(np.allclose(ref1[key], ref2[key]))
        ))

<|endoftext|>


<commit_before>
# Import Gaussian filter from scipy module

import numpy as np
import itertools
from scipy.ndimage.filters import gaussian_filter

def smooth(input, sigma, slice):
	'''
	Return array of same shape as input.
	The multidimensional filter is implemented as a sequence of one-dimensional
	filters.
	The intermediate arrays are sorted in the same data type as the output.

	Parameters
	----------
	input: A numpy array of an image data in one task run

	sigma: The standard deviations for Gaussian kernel.

	slice: The index time of the fourth dimension.

	Returns
	-------
	Returned array of same shape as input.

	'''
	input_slice = input[...,slice]
	data_array = gaussian_filter(input, sigma)
	return data_array

<commit_msg>Remove smoothing.py<commit_after><|endoftext|>


<commit_before>from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect, HttpResponseForbidden
from django.core.urlresolvers import reverse

from cog.models import *
from cog.forms import *
from constants import PERMISSION_DENIED_MESSAGE

def news_list(request, project_short_name):

    project = get_object_or_404(Project, short_name__iexact=project_short_name)

    return render_to_response('cog/news/news_list.html',
                              {'project': project,
                               'title': '%s News' % project.short_name,
                               'project_news': news(project) },
                               context_instance=RequestContext(request))

def news_detail(request, news_id):

    news = get_object_or_404(News, pk=news_id)

    return render_to_response('cog/news/news_detail.html',
                              {'project': news.project,
                               'news': news,
                               'title': news.title },
                               context_instance=RequestContext(request))

@login_required
def news_update(request, news_id):

    news = get_object_or_404(News, pk=news_id)

    # check permission
    if not userHasUserPermission(request.user, news.project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # GET method pre-populates the form with the news properties
    if (request.method=='GET'):

        # create form from instance
        form = NewsForm(news.project, request.user, instance=news)
        return render_news_form(request, request.GET, form, news.project)

    # POST method saves the modified instance
    elif (request.method=='POST'):

        # update existing database model with form data
        form = NewsForm(news.project, request.user, request.POST, instance=news)
        if (form.is_valid()):

            # save data from web
            news = form.save()

            # assign related projects
            news.other_projects = []
            for proj in (  list(form.cleaned_data['parent_projects'])
                         + list(form.cleaned_data['peer_projects'])
                         + list(form.cleaned_data['child_projects']) ):
                if proj not in news.other_projects.all():
                    news.other_projects.add(proj)

            # save new m2m relations
            news.save()

            # redirect to project home (GET-POST-REDIRECT)
            return HttpResponseRedirect(reverse('project_home', args=[news.project.short_name.lower()]))

        # invalid data
        else:
            print "Form is invalid: %s" % form.errors
            news = form.instance
            return render_news_form(request, request.POST, form, news.project)

@login_required
def news_add(request, project_short_name):

    project = get_object_or_404(Project, short_name__iexact=project_short_name)

    # check permission
    if not userHasUserPermission(request.user, project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # GET method pre-populates the form from the request parameters
    if (request.method=='GET'):

        # create empty News object
        news = News()

        # set main project
        news.project = project

        # create form from (unsaved) instance
        form = NewsForm(project, request.user, instance=news)

        return render_news_form(request, request.GET, form, news.project)

    # POST method validates the form data and saves instance to database
    else:

        # create form object from form data
        form = NewsForm(project, request.user, request.POST)

        if form.is_valid():

            # save object to the database
            news = form.save()

            # assign related projects
            for proj in form.cleaned_data['parent_projects']:
                news.other_projects.add(proj)
            for proj in form.cleaned_data['peer_projects']:
                news.other_projects.add(proj)
            for proj in form.cleaned_data['child_projects']:
                news.other_projects.add(proj)

            # save m2m relationships
            news.save()

            # redirect to project home (GET-POST-REDIRECT)
            return HttpResponseRedirect(reverse('project_home', args=[news.project.short_name.lower()]))

        # invalid data
        else:
            print "Form is invalid: %s" % form.errors
            news = form.instance
            return render_news_form(request, request.POST, form, news.project)

@login_required
def news_delete(request, news_id):

    news = get_object_or_404(News, pk=news_id)
    project = news.project

    # check permission
    if not userHasUserPermission(request.user, project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # show confirmation form
    if (request.method=='GET'):
        return render_to_response('cog/news/news_delete.html',
                                  {'news': news, 'project':project, 'title': 'Delete News' },
                                  context_instance=RequestContext(request))

    # execute, and redirect to project's home page
    else:
        news.delete()
        # redirect to project home (GET-POST-REDIRECT)
        return HttpResponseRedirect(reverse('project_home', args=[project.short_name.lower()]))

# function to extract the other projects from the GET/POST request parameters
def get_other_projects(qdict, method):
    other_projects_refs =  qdict.getlist('other_projects')
    other_projects = []
    for ref in other_projects_refs:
        if (method=='GET'):
            other_project = get_object_or_404(Project, short_name=ref)
        else:
            other_project = get_object_or_404(Project, id=ref)
        other_projects.append(other_project)
    return other_projects

def render_news_form(request, qdict, form, project):

    return render_to_response('cog/news/news_form.html',
                              {'form': form, 'title':
                               'Publish %s News' % project.short_name,
                               'project' : project,
                               'other_projects' : get_other_projects(qdict, request.method) },
                               context_instance=RequestContext(request))
    <commit_msg>PEP update<commit_after>from django.contrib.auth.decorators import login_required
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.http import HttpResponseRedirect, HttpResponseForbidden
from django.core.urlresolvers import reverse

from cog.models import *
from cog.forms import *
from constants import PERMISSION_DENIED_MESSAGE

def news_list(request, project_short_name):

    project = get_object_or_404(Project, short_name__iexact=project_short_name)

    return render_to_response('cog/news/news_list.html',
                              {'project': project,
                               'title': '%s News' % project.short_name,
                               'project_news': news(project)},
                              context_instance=RequestContext(request))

def news_detail(request, news_id):

    news = get_object_or_404(News, pk=news_id)

    return render_to_response('cog/news/news_detail.html',
                              {'project': news.project,
                               'news': news,
                               'title': news.title },
                              context_instance=RequestContext(request))

@login_required
def news_update(request, news_id):

    news = get_object_or_404(News, pk=news_id)

    # check permission
    if not userHasUserPermission(request.user, news.project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # GET method pre-populates the form with the news properties
    if request.method == 'GET':

        # create form from instance
        form = NewsForm(news.project, request.user, instance=news)
        return render_news_form(request, request.GET, form, news.project)

    # POST method saves the modified instance
    elif request.method == 'POST':

        # update existing database model with form data
        form = NewsForm(news.project, request.user, request.POST, instance=news)
        if form.is_valid():

            # save data from web
            news = form.save()

            # assign related projects
            news.other_projects = []
            for proj in (list(form.cleaned_data['parent_projects'])
                         + list(form.cleaned_data['peer_projects'])
                         + list(form.cleaned_data['child_projects'])):
                if proj not in news.other_projects.all():
                    news.other_projects.add(proj)

            # save new m2m relations
            news.save()

            # redirect to project home (GET-POST-REDIRECT)
            return HttpResponseRedirect(reverse('project_home', args=[news.project.short_name.lower()]))

        # invalid data
        else:
            print "Form is invalid: %s" % form.errors
            news = form.instance
            return render_news_form(request, request.POST, form, news.project)

@login_required
def news_add(request, project_short_name):

    project = get_object_or_404(Project, short_name__iexact=project_short_name)

    # check permission
    if not userHasUserPermission(request.user, project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # GET method pre-populates the form from the request parameters
    if request.method == 'GET':

        # create empty News object
        news = News()

        # set main project
        news.project = project

        # create form from (unsaved) instance
        form = NewsForm(project, request.user, instance=news)

        return render_news_form(request, request.GET, form, news.project)

    # POST method validates the form data and saves instance to database
    else:

        # create form object from form data
        form = NewsForm(project, request.user, request.POST)

        if form.is_valid():

            # save object to the database
            news = form.save()

            # assign related projects
            for proj in form.cleaned_data['parent_projects']:
                news.other_projects.add(proj)
            for proj in form.cleaned_data['peer_projects']:
                news.other_projects.add(proj)
            for proj in form.cleaned_data['child_projects']:
                news.other_projects.add(proj)

            # save m2m relationships
            news.save()

            # redirect to project home (GET-POST-REDIRECT)
            return HttpResponseRedirect(reverse('project_home', args=[news.project.short_name.lower()]))

        # invalid data
        else:
            print "Form is invalid: %s" % form.errors
            news = form.instance
            return render_news_form(request, request.POST, form, news.project)

@login_required
def news_delete(request, news_id):

    news = get_object_or_404(News, pk=news_id)
    project = news.project

    # check permission
    if not userHasUserPermission(request.user, project):
        return HttpResponseForbidden(PERMISSION_DENIED_MESSAGE)

    # show confirmation form
    if request.method == 'GET':
        return render_to_response('cog/news/news_delete.html',
                                  {'news': news,
                                   'project': project,
                                   'title': 'Delete News'},
                                  context_instance=RequestContext(request))

    # execute, and redirect to project's home page
    else:
        news.delete()
        # redirect to project home (GET-POST-REDIRECT)
        return HttpResponseRedirect(reverse('project_home', args=[project.short_name.lower()]))

# function to extract the other projects from the GET/POST request parameters
def get_other_projects(qdict, method):
    other_projects_refs = qdict.getlist('other_projects')
    other_projects = []
    for ref in other_projects_refs:
        if method == 'GET':
            other_project = get_object_or_404(Project, short_name=ref)
        else:
            other_project = get_object_or_404(Project, id=ref)
        other_projects.append(other_project)
    return other_projects

def render_news_form(request, qdict, form, project):

    return render_to_response('cog/news/news_form.html',
                              {'form': form, 'title':
                               'Publish %s News' % project.short_name,
                               'project': project,
                               'other_projects' : get_other_projects(qdict, request.method) },
                              context_instance=RequestContext(request))<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2008, Frank Scholz <coherence@beebits.net>

import pygst
pygst.require('0.10')
import gst
import gobject
gobject.threads_init ()

class DataSink(gst.Element):

    _sinkpadtemplate = gst.PadTemplate ("sinkpadtemplate",
                                        gst.PAD_SINK,
                                        gst.PAD_ALWAYS,
                                        gst.caps_new_any())

    def __init__(self,destination=None,request=None):
        gst.Element.__init__(self)
        self.sinkpad = gst.Pad(self._sinkpadtemplate, "sink")
        self.add_pad(self.sinkpad)

        self.sinkpad.set_chain_function(self.chainfunc)
        self.sinkpad.set_event_function(self.eventfunc)
        self.destination = destination
        self.request = request

        if self.destination != None:
            self.destination = open(self.destination, 'wb')
        self.buffer = ''
        self.data_size = 0
        self.got_new_segment = False
        self.closed = False

    def chainfunc(self, pad, buffer):
        if self.closed == True:
            return gst.FLOW_OK
        if self.destination != None:
            self.destination.write(buffer.data)
        elif self.request != None:
            self.buffer += buffer.data
            if len(self.buffer) > 200000:
                self.request.write(self.buffer)
                self.buffer = ''
        else:
            self.buffer += buffer.data

        self.data_size += buffer.size
        return gst.FLOW_OK

    def eventfunc(self, pad, event):
        #print "eventfunc", event
        if event.type == gst.EVENT_NEWSEGMENT:
            if self.got_new_segment == False:
                self.got_new_segment = True
            else:
                self.closed = True
        elif event.type == gst.EVENT_EOS:
            if self.destination != None:
                self.destination.close()
            elif self.request != None:
                if len(self.buffer) > 0:
                    self.request.write(self.buffer)
                self.request.finish()

            print "finished", self.data_size
        return True

gobject.type_register(DataSink)

from twisted.web import resource, server

from coherence import log

class PCMTranscoder(resource.Resource, log.Loggable):
    logCategory = 'transcoder'
    addSlash = True

    def __init__(self,source,destination=None):
        self.source = source
        self.destination = destination
        resource.Resource.__init__(self)

    def getChild(self, name, request):
        self.info('getChild %s, %s' % (name, request))
        return self

    def render_GET(self,request):
        self.info('render GET %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'audio/L16;rate=44100;channels=2')
        request.write('')

        headers = request.getAllHeaders()
        if('connection' in headers and
           headers['connection'] == 'close'):
            return

        self.start(request)
        return server.NOT_DONE_YET

    def render_HEAD(self,request):
        self.info('render HEAD %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'audio/L16;rate=44100;channels=2')
        request.write('')

    def start(self,request=None):
        print "start", request
        src = gst.element_factory_make('filesrc')
        src.set_property('location', self.source)
        sink = DataSink(destination=self.destination,request=request)

        decodebin = gst.element_factory_make('decodebin')
        decodebin.connect('new-decoded-pad', self.__on_new_decoded_pad)
        audioconvert = gst.element_factory_make('audioconvert')

        self.__audioconvert_pad = audioconvert.get_pad('sink')

        caps = gst.Caps("audio/x-raw-int,rate=44100,endianness=4321,channels=2,width=16,depth=16,signed=true")
        filter = gst.element_factory_make("capsfilter", "filter")
        filter.set_property("caps", caps)

        self.pipeline = gst.Pipeline()
        self.pipeline.add(src, decodebin, audioconvert, filter, sink)

        src.link(decodebin)
        audioconvert.link(filter)
        filter.link(sink)
        self.pipeline.set_state(gst.STATE_PLAYING)

        def requestFinished(result):
            print "requestFinished", result
            """ we need to find a way to destroy the pipeline here
            """
            #from twisted.internet import reactor
            #reactor.callLater(0, self.pipeline.set_state, gst.STATE_NULL)
            #reactor.callLater(0, self.pipeline.get_state)
            gobject.idle_add(self.cleanup)
            return

        d = request.notifyFinish()
        d.addBoth(requestFinished)

    def __on_new_decoded_pad(self, element, pad, last):
        caps = pad.get_caps()
        name = caps[0].get_name()
        if 'audio' in name:
            if not self.__audioconvert_pad.is_linked(): # Only link once
                pad.link(self.__audioconvert_pad)

    def cleanup(self):
        self.pipeline.set_state(gst.STATE_NULL)

class MP4Transcoder(resource.Resource, log.Loggable):
    # Only works if H264 inside Quicktime/MP4 container is input
    logCategory = 'transcoder'
    addSlash = True

    def __init__(self,source,destination=None):
        self.source = source
        self.destination = destination
        resource.Resource.__init__(self)

    def getChild(self, name, request):
        self.info('getChild %s, %s' % (name, request))
        return self

    def render_GET(self,request):
        self.info('render GET %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'video/mp4')
        request.write('')

        headers = request.getAllHeaders()
        if('connection' in headers and
           headers['connection'] == 'close'):
            return

        self.start(request)
        return server.NOT_DONE_YET

    def render_HEAD(self,request):
        self.info('render HEAD %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'video/mp4')
        request.write('')

    def start(self,request=None):
        print "start", request
        self.pipeline = gst.parse_launch(
            "filesrc location=%s ! qtdemux name=d ! queue ! h264parse ! mp4mux name=mux d. ! queue ! mux." % self.source)
        mux = self.pipeline.get_by_name('mux')
        sink = DataSink(destination=self.destination,request=request)
        self.pipeline.add(sink)
        mux.link(sink)
        self.pipeline.set_state(gst.STATE_PLAYING)

        def requestFinished(result):
            print "requestFinished", result
            """ we need to find a way to destroy the pipeline here
            """
            #from twisted.internet import reactor
            #reactor.callLater(0, self.pipeline.set_state, gst.STATE_NULL)
            #reactor.callLater(0, self.pipeline.get_state)
            gobject.idle_add(self.cleanup)
            return

        d = request.notifyFinish()
        d.addBoth(requestFinished)

    def cleanup(self):
        self.pipeline.set_state(gst.STATE_NULL)
<commit_msg>Replace filesrc with just a uri<commit_after># -*- coding: utf-8 -*-

# Licensed under the MIT license
# http://opensource.org/licenses/mit-license.php

# Copyright 2008, Frank Scholz <coherence@beebits.net>

import pygst
pygst.require('0.10')
import gst
import gobject
gobject.threads_init ()

class DataSink(gst.Element):

    _sinkpadtemplate = gst.PadTemplate ("sinkpadtemplate",
                                        gst.PAD_SINK,
                                        gst.PAD_ALWAYS,
                                        gst.caps_new_any())

    def __init__(self,destination=None,request=None):
        gst.Element.__init__(self)
        self.sinkpad = gst.Pad(self._sinkpadtemplate, "sink")
        self.add_pad(self.sinkpad)

        self.sinkpad.set_chain_function(self.chainfunc)
        self.sinkpad.set_event_function(self.eventfunc)
        self.destination = destination
        self.request = request

        if self.destination != None:
            self.destination = open(self.destination, 'wb')
        self.buffer = ''
        self.data_size = 0
        self.got_new_segment = False
        self.closed = False

    def chainfunc(self, pad, buffer):
        if self.closed == True:
            return gst.FLOW_OK
        if self.destination != None:
            self.destination.write(buffer.data)
        elif self.request != None:
            self.buffer += buffer.data
            if len(self.buffer) > 200000:
                self.request.write(self.buffer)
                self.buffer = ''
        else:
            self.buffer += buffer.data

        self.data_size += buffer.size
        return gst.FLOW_OK

    def eventfunc(self, pad, event):
        #print "eventfunc", event
        if event.type == gst.EVENT_NEWSEGMENT:
            if self.got_new_segment == False:
                self.got_new_segment = True
            else:
                self.closed = True
        elif event.type == gst.EVENT_EOS:
            if self.destination != None:
                self.destination.close()
            elif self.request != None:
                if len(self.buffer) > 0:
                    self.request.write(self.buffer)
                self.request.finish()

            print "finished", self.data_size
        return True

gobject.type_register(DataSink)

from twisted.web import resource, server

from coherence import log

class PCMTranscoder(resource.Resource, log.Loggable):
    logCategory = 'transcoder'
    addSlash = True

    def __init__(self,source,destination=None):
        self.source = source
        self.destination = destination
        resource.Resource.__init__(self)

    def getChild(self, name, request):
        self.info('getChild %s, %s' % (name, request))
        return self

    def render_GET(self,request):
        self.info('render GET %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'audio/L16;rate=44100;channels=2')
        request.write('')

        headers = request.getAllHeaders()
        if('connection' in headers and
           headers['connection'] == 'close'):
            return

        self.start(request)
        return server.NOT_DONE_YET

    def render_HEAD(self,request):
        self.info('render HEAD %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'audio/L16;rate=44100;channels=2')
        request.write('')

    def start(self,request=None):
        print "start", request
        src = gst.element_factory_make('filesrc')
        src.set_property('location', self.source)
        sink = DataSink(destination=self.destination,request=request)

        decodebin = gst.element_factory_make('decodebin')
        decodebin.connect('new-decoded-pad', self.__on_new_decoded_pad)
        audioconvert = gst.element_factory_make('audioconvert')

        self.__audioconvert_pad = audioconvert.get_pad('sink')

        caps = gst.Caps("audio/x-raw-int,rate=44100,endianness=4321,channels=2,width=16,depth=16,signed=true")
        filter = gst.element_factory_make("capsfilter", "filter")
        filter.set_property("caps", caps)

        self.pipeline = gst.Pipeline()
        self.pipeline.add(src, decodebin, audioconvert, filter, sink)

        src.link(decodebin)
        audioconvert.link(filter)
        filter.link(sink)
        self.pipeline.set_state(gst.STATE_PLAYING)

        def requestFinished(result):
            print "requestFinished", result
            """ we need to find a way to destroy the pipeline here
            """
            #from twisted.internet import reactor
            #reactor.callLater(0, self.pipeline.set_state, gst.STATE_NULL)
            #reactor.callLater(0, self.pipeline.get_state)
            gobject.idle_add(self.cleanup)
            return

        d = request.notifyFinish()
        d.addBoth(requestFinished)

    def __on_new_decoded_pad(self, element, pad, last):
        caps = pad.get_caps()
        name = caps[0].get_name()
        if 'audio' in name:
            if not self.__audioconvert_pad.is_linked(): # Only link once
                pad.link(self.__audioconvert_pad)

    def cleanup(self):
        self.pipeline.set_state(gst.STATE_NULL)

class MP4Transcoder(resource.Resource, log.Loggable):
    # Only works if H264 inside Quicktime/MP4 container is input
    # Source has to be a valid uri
    logCategory = 'transcoder'
    addSlash = True

    def __init__(self,source,destination=None):
        self.source = source
        self.destination = destination
        resource.Resource.__init__(self)

    def getChild(self, name, request):
        self.info('getChild %s, %s' % (name, request))
        return self

    def render_GET(self,request):
        self.info('render GET %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'video/mp4')
        request.write('')

        headers = request.getAllHeaders()
        if('connection' in headers and
           headers['connection'] == 'close'):
            return

        self.start(request)
        return server.NOT_DONE_YET

    def render_HEAD(self,request):
        self.info('render HEAD %r' % (request))
        request.setResponseCode(200)
        request.setHeader('Content-Type', 'video/mp4')
        request.write('')

    def start(self,request=None):
        print "start", request
        self.pipeline = gst.parse_launch(
            "%s ! qtdemux name=d ! queue ! h264parse ! mp4mux name=mux d. ! queue ! mux." % self.source)
        mux = self.pipeline.get_by_name('mux')
        sink = DataSink(destination=self.destination,request=request)
        self.pipeline.add(sink)
        mux.link(sink)
        self.pipeline.set_state(gst.STATE_PLAYING)

        def requestFinished(result):
            print "requestFinished", result
            """ we need to find a way to destroy the pipeline here
            """
            #from twisted.internet import reactor
            #reactor.callLater(0, self.pipeline.set_state, gst.STATE_NULL)
            #reactor.callLater(0, self.pipeline.get_state)
            gobject.idle_add(self.cleanup)
            return

        d = request.notifyFinish()
        d.addBoth(requestFinished)

    def cleanup(self):
        self.pipeline.set_state(gst.STATE_NULL)
<|endoftext|>


<commit_before>from __future__ import print_function
import os
import json
import unittest
import abc

import riotwatcher
import data_path

default_report_freq = 100
default_summoner_id = 30890339
default_n_ids_min = 10

default_summoner_ids_directory = data_path.summoner_ids_dir
default_champions_usage_directory = data_path.champions_usage_dir

class DataCollector:
    __metaclass__ = abc.ABCMeta

    def __init__(self, riot=None, report_callback=None, report_freq=default_report_freq):
        if riot is None:
            riot = riotwatcher.RiotWatcher()
        self.riot = riot
        if report_callback is None:
            report_callback = print
        self.report_callback = report_callback
        self.report_freq = report_freq

class SummonerIdsCollector(DataCollector):
    def __init__(self, riot=None, report_callback=None, report_freq=default_report_freq):
        DataCollector.__init__(self, riot=riot, report_callback=report_callback, report_freq=report_freq)

    def match_id_to_summoner_ids(self, match_id):
        self.riot.wait()
        try:
            match_detail = self.riot.get_match(match_id)
            return [participantIdentity['player']['summonerId'] for participantIdentity in
                    match_detail['participantIdentities']]
        except riotwatcher.LoLException as e:
            print('LoLException: ' + str(e) + ', in match_id_to_summoner_ids with match_id = ' + str(match_id))
            return []

    def summoner_id_to_match_ids(self, summoner_id):
        self.riot.wait()
        try:
            match_history = self.riot.get_match_list(summoner_id, ranked_queues=['RANKED_SOLO_5x5'], seasons=['SEASON2015'])
            # return [match_summary['matchId'] for match_summary in match_history['matches'] if match_summary['season']=='SEASON2015']
            return [match_summary['matchId'] for match_summary in match_history['matches']]
        except riotwatcher.LoLException as e:
            print('LoLException: ' + str(e) + ', in summoner_id_to_match_ids with summoner_id = ' + str(summoner_id))
            return []

    def collect_summoner_ids(self, n_ids_min, initial_summoner_id):
        summoner_ids = set()
        match_ids = set()
        pending_summoner_ids = [initial_summoner_id]
        pending_match_ids = []
        while len(summoner_ids) < n_ids_min and len(pending_summoner_ids) > 0:
            for s_id in pending_summoner_ids:
                for m_id in self.summoner_id_to_match_ids(s_id):
                    if m_id not in match_ids:
                        match_ids.add(m_id)
                        pending_match_ids.append(m_id)
                if 5 * len(match_ids) > n_ids_min:
                    break
            del pending_summoner_ids[:]
            for m_id in pending_match_ids:
                for s_id in self.match_id_to_summoner_ids(m_id):
                    if s_id not in summoner_ids:
                        summoner_ids.add(s_id)
                        if len(summoner_ids) % self.report_freq == 0:
                            self.report_callback('Collecting summoner ids: {}/{} done'.format(len(summoner_ids), n_ids_min))
                        pending_summoner_ids.append(s_id)
                if len(summoner_ids) > n_ids_min:
                    break
            del pending_match_ids[:]
        return summoner_ids

    def make_summoner_ids(self, initial_id=default_summoner_id, n_min=default_n_ids_min, name=None):
        if name is None:
            name = 'init{}min{}'.format(initial_id, n_min)
        data = self.collect_summoner_ids(n_min, initial_id)
        data = list(data)
        summoner_ids = SummonerIds.Cons(initial_id, n_min, name, data)
        return summoner_ids

def collect_master_summoner_ids():
    riot = riotwatcher.RiotWatcher()
    riot.wait()
    resp = riot.get_master()
    ids = [int(entry['playerOrTeamId']) for entry in resp['entries']]
    print(len(ids))
    print(ids)
    with open(data_path.summoner_ids_master, 'w') as f:
        json.dump(ids, f)

class DataWrapper:
    __metaclass__ = abc.ABCMeta

    def __init__(self, infos, data, name):
        self.infos = infos
        self.data = data
        self.name = name

    @staticmethod
    @abc.abstractmethod
    def default_directory():
        raise NotImplementedError

    def get_info(self, key):
        return self.infos.get(key)

class SummonerIds(DataWrapper):
    def __init__(self, dic, data):
        DataWrapper.__init__(self, dic, data, dic['name'])

    @staticmethod
    def default_directory():
        return data_path.summoner_ids_dir

    @classmethod
    def Cons(cls, initial_summoner_id, n_summoners_required, name, data):
        d = {'class': 'SummonerIds',
             'n_summoners': len(data),
             'name': name,
             'initial_summoner_id': initial_summoner_id,
             'n_summoners_required': n_summoners_required}
        si = SummonerIds(d, data)
        return si

class DataFilesHandler:
    def __init__(self, directory=None):
        self.directory = directory

    class UndefinedFilenameError(Exception):
        pass

    def decide_filename(self, default_name, default_directory,
                        enforce_name, enforce_directory, enforce_fullname):
        if enforce_fullname is not None:
            return enforce_fullname
        name = default_name
        directory = default_directory
        if enforce_name is not None:
            name = enforce_name
        if self.directory is not None:
            directory = self.directory
        if enforce_directory is not None:
            directory = enforce_directory
        if name is None or directory is None:
            raise self.UndefinedFilenameError
        return os.path.join(directory, name)

    def dump(self, datawrapper, enforce_name=None, enforce_directory=None, enforce_fullname=None):
        filename = self.decide_filename(datawrapper.name, datawrapper.default_directory,
                                        enforce_name, enforce_directory, enforce_fullname)
        s1 = json.dumps(datawrapper.infos)
        s2 = json.dumps(datawrapper.data)
        with open(filename, 'w') as f:
            f.write(s1+'\n'+s2)
        return filename

    def readlines(self, name=None, enforce_directory=None, fullname=None):
        filename = self.decide_filename(None, None, name, enforce_directory, fullname)
        with open(filename, 'r') as f:
            s1 = f.readline()
            s2 = f.readline()
        return s1, s2

    def load_cls(self, cls, name=None, enforce_directory=None, fullname=None):
        s1, s2 = self.readlines(name=name, enforce_directory=enforce_directory, fullname=fullname)
        dic = json.loads(s1)
        data = json.loads(s2)
        return cls(dic, data)

    def load_infos(self, name=None, enforce_directory=None, fullname=None):
        s1, s2 = self.readlines(name=name, enforce_directory=enforce_directory, fullname=fullname)
        return json.loads(s1)

def make_ClsFilesHandler(cls):
    class ClsFilesHandler(DataFilesHandler):
        def __init__(self, directory=None):
            DataFilesHandler.__init__(self, directory=directory)
            if self.directory is None:
                self.directory = cls.default_directory()

        def load(self, name=None, enforce_directory=None, fullname=None):
            return self.load_cls(cls, name=name, enforce_directory=enforce_directory, fullname=fullname)

    return ClsFilesHandler

SummonerIdsFilesHandler = make_ClsFilesHandler(SummonerIds)

class TestCollectSummonerIds(unittest.TestCase):

    name = 'TestCollectSummonerIds'

    reports = []

    def send_report(self, s):
        self.reports.append(s)

    def test(self):
        n_min = 30
        self.reports = []
        collector = SummonerIdsCollector(report_freq=10, report_callback=self.send_report)
        summoner_ids = collector.make_summoner_ids(n_min=n_min, name=self.name)
        self.assertGreaterEqual(summoner_ids.get_info('n_summoners'), n_min)
        for i in [1,2,3]:
            j = 10*i
            self.assertEqual(self.reports.pop(0), 'Collecting summoner ids: {}/{} done'.format(j, n_min))

class TestSummonerIdsFilesHandler(unittest.TestCase):

   name = 'test'

   def test(self):
       collector = SummonerIdsCollector()
       summoner_ids = collector.make_summoner_ids(name=self.name)
       h = SummonerIdsFilesHandler()
       h.dump(summoner_ids)
       summoner_ids1 = h.load(self.name)
       self.assertEqual(summoner_ids.infos, summoner_ids1.infos)
       self.assertEqual(summoner_ids.data, summoner_ids1.data)

if __name__ == "__main__":

    unittest.main()

<commit_msg>Add global constants in collect_summoner_ids.py<commit_after>from __future__ import print_function
import os
import json
import unittest
import abc

import riotwatcher
import data_path

default_report_freq = 100
default_report_callback = print

default_summoner_id = 30890339
default_n_ids_min = 10

default_summoner_ids_directory = data_path.summoner_ids_dir
default_champions_usage_directory = data_path.champions_usage_dir

n_summoners_key = 'n_summoners'
name_key = 'name'
class_key = 'class'
initial_summoner_id_key = 'initial_summoner_id'
n_summoners_required_key = 'n_summoners_required'
tier_key = 'tier'

class DataCollector:
    __metaclass__ = abc.ABCMeta

    def __init__(self, riot=None, report_callback=default_report_callback, report_freq=default_report_freq):
        if riot is None:
            riot = riotwatcher.RiotWatcher()
        self.riot = riot
        if report_callback is None:
            report_callback = default_report_callback
        self.report_callback = report_callback
        self.report_freq = report_freq

class SummonerIdsCollector(DataCollector):
    def __init__(self, riot=None, report_callback=default_report_callback, report_freq=default_report_freq):
        DataCollector.__init__(self, riot=riot, report_callback=report_callback, report_freq=report_freq)

    def match_id_to_summoner_ids(self, match_id):
        self.riot.wait()
        try:
            match_detail = self.riot.get_match(match_id)
            return [participantIdentity['player']['summonerId'] for participantIdentity in
                    match_detail['participantIdentities']]
        except riotwatcher.LoLException as e:
            print('LoLException: ' + str(e) + ', in match_id_to_summoner_ids with match_id = ' + str(match_id))
            return []

    def summoner_id_to_match_ids(self, summoner_id):
        self.riot.wait()
        try:
            match_history = self.riot.get_match_list(summoner_id, ranked_queues=['RANKED_SOLO_5x5'], seasons=['SEASON2015'])
            # return [match_summary['matchId'] for match_summary in match_history['matches'] if match_summary['season']=='SEASON2015']
            return [match_summary['matchId'] for match_summary in match_history['matches']]
        except riotwatcher.LoLException as e:
            print('LoLException: ' + str(e) + ', in summoner_id_to_match_ids with summoner_id = ' + str(summoner_id))
            return []

    def collect_summoner_ids(self, n_ids_min, initial_summoner_id):
        summoner_ids = set()
        match_ids = set()
        pending_summoner_ids = [initial_summoner_id]
        pending_match_ids = []
        while len(summoner_ids) < n_ids_min and len(pending_summoner_ids) > 0:
            for s_id in pending_summoner_ids:
                for m_id in self.summoner_id_to_match_ids(s_id):
                    if m_id not in match_ids:
                        match_ids.add(m_id)
                        pending_match_ids.append(m_id)
                if 5 * len(match_ids) > n_ids_min:
                    break
            del pending_summoner_ids[:]
            for m_id in pending_match_ids:
                for s_id in self.match_id_to_summoner_ids(m_id):
                    if s_id not in summoner_ids:
                        summoner_ids.add(s_id)
                        if len(summoner_ids) % self.report_freq == 0:
                            self.report_callback('Collecting summoner ids: {}/{} done'.format(len(summoner_ids), n_ids_min))
                        pending_summoner_ids.append(s_id)
                if len(summoner_ids) > n_ids_min:
                    break
            del pending_match_ids[:]
        return summoner_ids

    def make_summoner_ids(self, initial_id=default_summoner_id, n_min=default_n_ids_min, name=None):
        if name is None:
            name = 'init{}min{}'.format(initial_id, n_min)
        data = self.collect_summoner_ids(n_min, initial_id)
        data = list(data)
        summoner_ids = SummonerIds.Cons(initial_id, n_min, name, data)
        return summoner_ids

    def make_summoner_ids_master(self, name=None):
        if name is None:
            name = 'master'
        self.riot.wait()
        resp = self.riot.get_master()
        data = [int(entry['playerOrTeamId']) for entry in resp['entries']]
        summoner_ids = SummonerIds.Cons_tier('master', name, data)
        return summoner_ids
#endclass SummonerIdsCollector

class DataWrapper:
    __metaclass__ = abc.ABCMeta

    def __init__(self, infos, data, name):
        self.infos = infos
        self.data = data
        self.name = name

    @staticmethod
    @abc.abstractmethod
    def default_directory():
        raise NotImplementedError

    def get_info(self, key):
        return self.infos.get(key)

    @property
    def get_n_summoners(self):
        return self.get_info(n_summoners_key)

class SummonerIds(DataWrapper):
    def __init__(self, dic, data):
        DataWrapper.__init__(self, dic, data, dic['name'])

    @staticmethod
    def default_directory():
        return data_path.summoner_ids_dir

    @classmethod
    def Cons(cls, initial_summoner_id, n_summoners_required, name, data):
        d = {class_key: cls.__name__,
             n_summoners_key: len(data),
             name_key: name,
             initial_summoner_id_key: initial_summoner_id,
             n_summoners_required_key: n_summoners_required}
        si = SummonerIds(d, data)
        return si

    @classmethod
    def Cons_tier(cls, tier, name, data, n_summoners_required=None):
        d = {class_key: cls.__name__,
             n_summoners_key: len(data),
             name_key: name,
             tier_key: tier}
        if n_summoners_required is not None:
            d[n_summoners_required_key] = n_summoners_required
        si = SummonerIds(d, data)
        return si

class DataFilesHandler:
    def __init__(self, directory=None):
        self.directory = directory

    class UndefinedFilenameError(Exception):
        pass

    def decide_filename(self, default_name, default_directory,
                        enforce_name, enforce_directory, enforce_fullname):
        if enforce_fullname is not None:
            return enforce_fullname
        name = default_name
        directory = default_directory
        if enforce_name is not None:
            name = enforce_name
        if self.directory is not None:
            directory = self.directory
        if enforce_directory is not None:
            directory = enforce_directory
        if name is None or directory is None:
            raise self.UndefinedFilenameError
        return os.path.join(directory, name)

    def dump(self, datawrapper, enforce_name=None, enforce_directory=None, enforce_fullname=None):
        filename = self.decide_filename(datawrapper.name, datawrapper.default_directory,
                                        enforce_name, enforce_directory, enforce_fullname)
        s1 = json.dumps(datawrapper.infos)
        s2 = json.dumps(datawrapper.data)
        with open(filename, 'w') as f:
            f.write(s1+'\n'+s2)
        return filename

    def readlines(self, name=None, enforce_directory=None, fullname=None):
        filename = self.decide_filename(None, None, name, enforce_directory, fullname)
        with open(filename, 'r') as f:
            s1 = f.readline()
            s2 = f.readline()
        return s1, s2

    def load_cls(self, cls, name=None, enforce_directory=None, fullname=None):
        s1, s2 = self.readlines(name=name, enforce_directory=enforce_directory, fullname=fullname)
        dic = json.loads(s1)
        data = json.loads(s2)
        return cls(dic, data)

    def load_infos(self, name=None, enforce_directory=None, fullname=None):
        s1, s2 = self.readlines(name=name, enforce_directory=enforce_directory, fullname=fullname)
        return json.loads(s1)

def make_ClsFilesHandler(cls):
    class ClsFilesHandler(DataFilesHandler):
        def __init__(self, directory=None):
            DataFilesHandler.__init__(self, directory=directory)
            if self.directory is None:
                self.directory = cls.default_directory()

        def load(self, name=None, enforce_directory=None, fullname=None):
            return self.load_cls(cls, name=name, enforce_directory=enforce_directory, fullname=fullname)

    return ClsFilesHandler

SummonerIdsFilesHandler = make_ClsFilesHandler(SummonerIds)

def collect_master_summoner_ids():
    # riot = riotwatcher.RiotWatcher()
    # riot.wait()
    # resp = riot.get_master()
    # ids = [int(entry['playerOrTeamId']) for entry in resp['entries']]
    # print(len(ids))
    # print(ids)
    # with open(data_path.summoner_ids_master, 'w') as f:
    #     json.dump(ids, f)
    collector = SummonerIdsCollector()
    summoner_ids = collector.make_summoner_ids_master()
    h = SummonerIdsFilesHandler()
    h.dump(summoner_ids)

class TestCollectSummonerIds(unittest.TestCase):

    name = 'TestCollectSummonerIds'

    reports = []

    def send_report(self, s):
        self.reports.append(s)

    def test(self):
        n_min = 30
        self.reports = []
        collector = SummonerIdsCollector(report_freq=10, report_callback=self.send_report)
        summoner_ids = collector.make_summoner_ids(n_min=n_min, name=self.name)
        self.assertGreaterEqual(summoner_ids.get_info('n_summoners'), n_min)
        for i in [1,2,3]:
            j = 10*i
            self.assertEqual(self.reports.pop(0), 'Collecting summoner ids: {}/{} done'.format(j, n_min))

class TestSummonerIdsFilesHandler(unittest.TestCase):

    name = 'test'

    def test(self):
        collector = SummonerIdsCollector()
        summoner_ids = collector.make_summoner_ids(name=self.name)
        h = SummonerIdsFilesHandler()
        h.dump(summoner_ids)
        summoner_ids1 = h.load(self.name)
        self.assertEqual(summoner_ids.infos, summoner_ids1.infos)
        self.assertEqual(summoner_ids.data, summoner_ids1.data)

if __name__ == "__main__":

    unittest.main()<|endoftext|>