<commit_before>from __future__ import unicode_literals

import json
import re
import os

import jinja2
from nipype.utils.filemanip import loadcrash
from pkg_resources import resource_filename as pkgrf

class Element(object):

    def __init__(self, name, file_pattern, title, description):
        self.name = name
        self.file_pattern = re.compile(file_pattern)
        self.title = title
        self.description = description
        self.files_contents = []

class SubReport(object):

    def __init__(self, name, elements, title=''):
        self.name = name
        self.title = title
        self.elements = []
        self.run_reports = []
        for e in elements:
            element = Element(**e)
            self.elements.append(element)

    def order_by_run(self):
        run_reps = {}
        for elem_index in range(len(self.elements) - 1, -1, -1):
            element = self.elements[elem_index]
            for index in range(len(element.files_contents) - 1, -1, -1):
                filename = element.files_contents[index][0]
                file_contents = element.files_contents[index][1]
                name, title = self.generate_name_title(filename)
                if not name:
                    continue
                new_elem = {'name': element.name, 'file_pattern': element.file_pattern,
                            'title': element.title, 'description': element.description}
                try:
                    new_element = Element(**new_elem)
                    run_reps[name].elements.append(new_element)
                    run_reps[name].elements[-1].files_contents.append((filename, file_contents))
                except KeyError:
                    run_reps[name] = SubReport(name, [new_elem], title=title)
                    run_reps[name].elements[0].files_contents.append((filename, file_contents))
        keys = list(run_reps.keys())
        keys.sort()
        for key in keys:
            self.run_reports.append(run_reps[key])

    def generate_name_title(self, filename):
        fname = os.path.basename(filename)
        expr = re.compile('^sub-(?P<subject_id>[a-zA-Z0-9]+)(_ses-(?P<session_id>[a-zA-Z0-9]+))?'
                          '(_task-(?P<task_id>[a-zA-Z0-9]+))?(_acq-(?P<acq_id>[a-zA-Z0-9]+))?'
                          '(_rec-(?P<rec_id>[a-zA-Z0-9]+))?(_run-(?P<run_id>[a-zA-Z0-9]+))?')
        outputs = expr.search(fname)
        if outputs:
            outputs = outputs.groupdict()
        else:
            return None, None

        name = '{session}{task}{acq}{rec}{run}'.format(
            session="_ses-" + outputs['session_id'] if outputs['session_id'] else '',
            task="_task-" + outputs['task_id'] if outputs['task_id'] else '',
            acq="_acq-" + outputs['acq_id'] if outputs['acq_id'] else '',
            rec="_rec-" + outputs['rec_id'] if outputs['rec_id'] else '',
            run="_run-" + outputs['run_id'] if outputs['run_id'] else ''
        )
        title = '{session}{task}{acq}{rec}{run}'.format(
            session=" Session: " + outputs['session_id'] if outputs['session_id'] else '',
            task=" Task: " + outputs['task_id'] if outputs['task_id'] else '',
            acq=" Acquisition: " + outputs['acq_id'] if outputs['acq_id'] else '',
            rec=" Reconstruction: " + outputs['rec_id'] if outputs['rec_id'] else '',
            run=" Run: " + outputs['run_id'] if outputs['run_id'] else ''
        )
        return name, title

class Report(object):

    def __init__(self, path, config, out_dir, out_filename='report.html'):
        self.root = path
        self.sub_reports = []
        self.errors = []
        self._load_config(config)
        self.out_dir = out_dir
        self.out_filename = out_filename

    def _load_config(self, config):
        try:
            config = json.load(open(config, 'r'))
        except Exception as e:
            print(e)
            return

        for e in config['sub_reports']:
            sub_report = SubReport(**e)
            self.sub_reports.append(sub_report)

        self.index()

    def index(self):
        for root, directories, filenames in os.walk(self.root):
            for f in filenames:
                f = os.path.join(root, f)
                for sub_report in self.sub_reports:
                    for element in sub_report.elements:
                        ext = f.split('.')[-1]
                        if element.file_pattern.search(f) and (ext == 'svg' or ext == 'html'):
                            with open(f) as fp:
                                content = fp.read()
                                content = '\n'.join(content.split('\n')[1:])
                                element.files_contents.append((f, content))
        for sub_report in self.sub_reports:
            sub_report.order_by_run()

        subject_dir = self.root.split('/')[-1]
        subject = re.search('^(?P<subject_id>sub-[a-zA-Z0-9]+)$', subject_dir).group()
        error_dir = os.path.join(self.root, '../../log', subject[4:])
        if os.path.isdir(error_dir):
            self.index_error_dir(error_dir)

    def index_error_dir(self, error_dir):
        ''' Crawl subjects most recent crash directory and return text for
            .pklz crash file found. '''
        # Crash directories for subject are named by a timestamp. Sort
        # listdir output to order it alphabetically, which for our timestamped
        # directories is also a chronological listing. Assumes no other
        # directories will be created in subject crash dir.
        try:
            newest_dir = [x for x in os.listdir(error_dir)
                          if os.path.isdir(os.path.join(error_dir, x))]
            newest_dir.sort()
            newest_dir = newest_dir[-1]
            newest_dir = os.path.join(error_dir, newest_dir)
        except IndexError:
            newest_dir = error_dir
        for root, directories, filenames in os.walk(newest_dir):
            for f in filenames:
                # Only deal with files that start with crash and end in pklz
                if not (f[:5] == 'crash' and f[:-4] == 'pklz'):
                    continue
                crash_data = loadcrash(os.path.join(root, f))
                error = {}
                node = None
                if 'node' in crash_data:
                    node = crash_data['node']
                error['traceback'] = []
                for elem in crash_data['traceback']:
                    error['traceback'].append("
".join(elem.split("\n")))
                error['file'] = f

                if node:
                    error['node'] = node
                    if node.base_dir:
                        error['node_dir'] = node.output_dir()
                    else:
                        error['node_dir'] = "Node crashed before execution"
                    error['inputs'] = sorted(node.inputs.traits().items())
                self.errors.append(error)

    def generate_report(self):
        searchpath = pkgrf('fmriprep', '/')
        env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(searchpath=searchpath),
            trim_blocks=True, lstrip_blocks=True
        )
        report_tpl = env.get_template('viz/report.tpl')
        report_render = report_tpl.render(sub_reports=self.sub_reports, errors=self.errors)
        with open(os.path.join(self.out_dir, self.out_filename), 'w') as fp:
            fp.write(report_render)
        return report_render

def run_reports(out_dir):
    reportlet_path = os.path.join(out_dir, 'reports/')
    config = pkgrf('fmriprep', 'viz/config.json')

    for root, _, _ in os.walk(reportlet_path):
        #  relies on the fact that os.walk does not return a trailing /
        dir = root.split('/')[-1]
        try:
            subject = re.search('^(?P<subject_id>sub-[a-zA-Z0-9]+)$', dir).group()
            out_filename = '{}{}'.format(subject, '.html')
            report = Report(root, config, out_dir, out_filename)
            report.generate_report()
        except AttributeError:
            continue
<commit_msg>input trait values now display in crash reports.<commit_after>from __future__ import unicode_literals

import json
import re
import os

import jinja2
from nipype.utils.filemanip import loadcrash
from pkg_resources import resource_filename as pkgrf

class Element(object):

    def __init__(self, name, file_pattern, title, description):
        self.name = name
        self.file_pattern = re.compile(file_pattern)
        self.title = title
        self.description = description
        self.files_contents = []

class SubReport(object):

    def __init__(self, name, elements, title=''):
        self.name = name
        self.title = title
        self.elements = []
        self.run_reports = []
        for e in elements:
            element = Element(**e)
            self.elements.append(element)

    def order_by_run(self):
        run_reps = {}
        for elem_index in range(len(self.elements) - 1, -1, -1):
            element = self.elements[elem_index]
            for index in range(len(element.files_contents) - 1, -1, -1):
                filename = element.files_contents[index][0]
                file_contents = element.files_contents[index][1]
                name, title = self.generate_name_title(filename)
                if not name:
                    continue
                new_elem = {'name': element.name, 'file_pattern': element.file_pattern,
                            'title': element.title, 'description': element.description}
                try:
                    new_element = Element(**new_elem)
                    run_reps[name].elements.append(new_element)
                    run_reps[name].elements[-1].files_contents.append((filename, file_contents))
                except KeyError:
                    run_reps[name] = SubReport(name, [new_elem], title=title)
                    run_reps[name].elements[0].files_contents.append((filename, file_contents))
        keys = list(run_reps.keys())
        keys.sort()
        for key in keys:
            self.run_reports.append(run_reps[key])

    def generate_name_title(self, filename):
        fname = os.path.basename(filename)
        expr = re.compile('^sub-(?P<subject_id>[a-zA-Z0-9]+)(_ses-(?P<session_id>[a-zA-Z0-9]+))?'
                          '(_task-(?P<task_id>[a-zA-Z0-9]+))?(_acq-(?P<acq_id>[a-zA-Z0-9]+))?'
                          '(_rec-(?P<rec_id>[a-zA-Z0-9]+))?(_run-(?P<run_id>[a-zA-Z0-9]+))?')
        outputs = expr.search(fname)
        if outputs:
            outputs = outputs.groupdict()
        else:
            return None, None

        name = '{session}{task}{acq}{rec}{run}'.format(
            session="_ses-" + outputs['session_id'] if outputs['session_id'] else '',
            task="_task-" + outputs['task_id'] if outputs['task_id'] else '',
            acq="_acq-" + outputs['acq_id'] if outputs['acq_id'] else '',
            rec="_rec-" + outputs['rec_id'] if outputs['rec_id'] else '',
            run="_run-" + outputs['run_id'] if outputs['run_id'] else ''
        )
        title = '{session}{task}{acq}{rec}{run}'.format(
            session=" Session: " + outputs['session_id'] if outputs['session_id'] else '',
            task=" Task: " + outputs['task_id'] if outputs['task_id'] else '',
            acq=" Acquisition: " + outputs['acq_id'] if outputs['acq_id'] else '',
            rec=" Reconstruction: " + outputs['rec_id'] if outputs['rec_id'] else '',
            run=" Run: " + outputs['run_id'] if outputs['run_id'] else ''
        )
        return name, title

class Report(object):

    def __init__(self, path, config, out_dir, out_filename='report.html'):
        self.root = path
        self.sub_reports = []
        self.errors = []
        self._load_config(config)
        self.out_dir = out_dir
        self.out_filename = out_filename

    def _load_config(self, config):
        try:
            config = json.load(open(config, 'r'))
        except Exception as e:
            print(e)
            return

        for e in config['sub_reports']:
            sub_report = SubReport(**e)
            self.sub_reports.append(sub_report)

        self.index()

    def index(self):
        for root, directories, filenames in os.walk(self.root):
            for f in filenames:
                f = os.path.join(root, f)
                for sub_report in self.sub_reports:
                    for element in sub_report.elements:
                        ext = f.split('.')[-1]
                        if element.file_pattern.search(f) and (ext == 'svg' or ext == 'html'):
                            with open(f) as fp:
                                content = fp.read()
                                content = '\n'.join(content.split('\n')[1:])
                                element.files_contents.append((f, content))
        for sub_report in self.sub_reports:
            sub_report.order_by_run()

        subject_dir = self.root.split('/')[-1]
        subject = re.search('^(?P<subject_id>sub-[a-zA-Z0-9]+)$', subject_dir).group()
        error_dir = os.path.join(self.root, '../../log', subject[4:])
        if os.path.isdir(error_dir):
            self.index_error_dir(error_dir)

    def index_error_dir(self, error_dir):
        ''' Crawl subjects most recent crash directory and return text for
            .pklz crash file found. '''
        # Crash directories for subject are named by a timestamp. Sort
        # listdir output to order it alphabetically, which for our timestamped
        # directories is also a chronological listing. Assumes no other
        # directories will be created in subject crash dir.
        try:
            newest_dir = [x for x in os.listdir(error_dir)
                          if os.path.isdir(os.path.join(error_dir, x))]
            newest_dir.sort()
            newest_dir = newest_dir[-1]
            newest_dir = os.path.join(error_dir, newest_dir)
        except IndexError:
            newest_dir = error_dir
        for root, directories, filenames in os.walk(newest_dir):
            for f in filenames:
                # Only deal with files that start with crash and end in pklz
                if not (f[:5] == 'crash' and f[-4:] == 'pklz'):
                    continue
                crash_data = loadcrash(os.path.join(root, f))
                error = {}
                node = None
                if 'node' in crash_data:
                    node = crash_data['node']
                error['traceback'] = []
                for elem in crash_data['traceback']:
                    error['traceback'].append("
".join(elem.split("\n")))
                error['file'] = f

                if node:
                    error['node'] = node
                    if node.base_dir:
                        error['node_dir'] = node.output_dir()
                    else:
                        error['node_dir'] = "Node crashed before execution"
                    error['inputs'] = sorted(node.inputs.trait_get().items())
                self.errors.append(error)

    def generate_report(self):
        searchpath = pkgrf('fmriprep', '/')
        env = jinja2.Environment(
            loader=jinja2.FileSystemLoader(searchpath=searchpath),
            trim_blocks=True, lstrip_blocks=True
        )
        report_tpl = env.get_template('viz/report.tpl')
        report_render = report_tpl.render(sub_reports=self.sub_reports, errors=self.errors)
        with open(os.path.join(self.out_dir, self.out_filename), 'w') as fp:
            fp.write(report_render)
        return report_render

def run_reports(out_dir):
    reportlet_path = os.path.join(out_dir, 'reports/')
    config = pkgrf('fmriprep', 'viz/config.json')

    for root, _, _ in os.walk(reportlet_path):
        #  relies on the fact that os.walk does not return a trailing /
        dir = root.split('/')[-1]
        try:
            subject = re.search('^(?P<subject_id>sub-[a-zA-Z0-9]+)$', dir).group()
            out_filename = '{}{}'.format(subject, '.html')
            report = Report(root, config, out_dir, out_filename)
            report.generate_report()
        except AttributeError:
            continue
<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__authors__ = [
  '"Augie Fackler" <durin42@gmail.com>',
  ]

import sys
import os

HERE = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                     '..'))
appengine_location = os.path.join(HERE, 'thirdparty', 'google_appengine')
extra_paths = [HERE,
               os.path.join(appengine_location, 'lib', 'django'),
               os.path.join(appengine_location, 'lib', 'webob'),
               os.path.join(appengine_location, 'lib', 'yaml', 'lib'),
               os.path.join(appengine_location, 'lib', 'antlr3'),
               appengine_location,
               os.path.join(HERE, 'app'),
               os.path.join(HERE, 'thirdparty', 'coverage'),
              ]

import nose
from nose import plugins

import logging
log =  logging.getLogger('nose.plugins.cover')

def begin(self):
  """Used to stub out nose.plugins.cover.Coverage.begin. The difference is that it loads Melange after coverage starts so the loading of models, logic and views can be tracked by coverage.
  """
  log.debug("Coverage begin")
  import coverage
  self.skipModules = sys.modules.keys()[:]
  if self.coverErase:
    log.debug("Clearing previously collected coverage statistics")
    coverage.erase()
  coverage.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]')
  coverage.start()
  load_melange()

def load_melange():
  """
  Registers a core, and SoC, GSoC and GHOP modules callbacks.
  """
  # Register a core for the test modules to use
  from soc.modules import callback
  from soc.modules.core import Core
  callback.registerCore(Core())
  current_core = callback.getCore()
  # Register SoC and GSoC modules callbacks
  current_core.registerModuleCallbacks()
  # Register GHOP modules callbacks
  from soc.modules.ghop.callback import Callback
  ghop_callback = Callback(current_core)
  ghop_callback.registerWithSitemap()

class AppEngineDatastoreClearPlugin(plugins.Plugin):
  """Nose plugin to clear the AppEngine datastore between tests.
  """
  name = 'AppEngineDatastoreClearPlugin'
  enabled = True
  def options(self, parser, env):
    return plugins.Plugin.options(self, parser, env)

  def configure(self, parser, env):
    plugins.Plugin.configure(self, parser, env)
    self.enabled = True

  def afterTest(self, test):
    from google.appengine.api import apiproxy_stub_map
    datastore = apiproxy_stub_map.apiproxy.GetStub('datastore')
    # clear datastore iff one is available
    if datastore is not None:
      datastore.Clear()

def main():
  sys.path = extra_paths + sys.path
  os.environ['SERVER_SOFTWARE'] = 'Development via nose'
  os.environ['SERVER_NAME'] = 'Foo'
  os.environ['SERVER_PORT'] = '8080'
  os.environ['APPLICATION_ID'] = 'test-app-run'
  os.environ['USER_EMAIL'] = 'test@example.com'
  os.environ['CURRENT_VERSION_ID'] = 'testing-version'
  os.environ['HTTP_HOST'] = 'some.testing.host.tld'
  import main as app_main
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_file_stub
  from google.appengine.api import mail_stub
  from google.appengine.api import user_service_stub
  from google.appengine.api import urlfetch_stub
  from google.appengine.api.memcache import memcache_stub
  from google.appengine.api.labs.taskqueue import taskqueue_stub
  apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
  apiproxy_stub_map.apiproxy.RegisterStub('urlfetch',
                                          urlfetch_stub.URLFetchServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('user',
                                          user_service_stub.UserServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('datastore',
    datastore_file_stub.DatastoreFileStub('test-app-run', None, None))
  apiproxy_stub_map.apiproxy.RegisterStub('memcache',
    memcache_stub.MemcacheServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('mail', mail_stub.MailServiceStub())
  yaml_location = os.path.join(HERE, 'app')
  apiproxy_stub_map.apiproxy.RegisterStub('taskqueue', taskqueue_stub.TaskQueueServiceStub(root_path=yaml_location))
  import django.test.utils
  django.test.utils.setup_test_environment()

  plugins = [AppEngineDatastoreClearPlugin()]

  if '--coverage' in sys.argv:
    from nose.plugins import cover
    plugin = cover.Coverage()
    from tests.pymox import stubout
    stubout_obj = stubout.StubOutForTesting()
    stubout_obj.SmartSet(plugin, 'begin', begin)
    plugins.append(plugin)

    args = ['--with-coverage',
            '--cover-package=soc.',
            '--cover-erase',
            '--cover-html',
            '--cover-html-dir=coverageResults']

    sys.argv.remove('--coverage')
    sys.argv += args
  else:
    load_melange()

  nose.main(addplugins=plugins)

if __name__ == '__main__':
  main()
<commit_msg>Register SoC, GSoC, GHOP modules sitemap, sidebar, rights for tests.<commit_after>#!/usr/bin/env python2.5
#
# Copyright 2009 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__authors__ = [
  '"Augie Fackler" <durin42@gmail.com>',
  ]

import sys
import os

HERE = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                     '..'))
appengine_location = os.path.join(HERE, 'thirdparty', 'google_appengine')
extra_paths = [HERE,
               os.path.join(appengine_location, 'lib', 'django'),
               os.path.join(appengine_location, 'lib', 'webob'),
               os.path.join(appengine_location, 'lib', 'yaml', 'lib'),
               os.path.join(appengine_location, 'lib', 'antlr3'),
               appengine_location,
               os.path.join(HERE, 'app'),
               os.path.join(HERE, 'thirdparty', 'coverage'),
              ]

import nose
from nose import plugins

import logging
log =  logging.getLogger('nose.plugins.cover')

def begin(self):
  """Used to stub out nose.plugins.cover.Coverage.begin. The difference is that
  it loads Melange after coverage starts so the loading of models, logic and
  views can be tracked by coverage.
  """
  log.debug("Coverage begin")
  import coverage
  self.skipModules = sys.modules.keys()[:]
  if self.coverErase:
    log.debug("Clearing previously collected coverage statistics")
    coverage.erase()
  coverage.exclude('#pragma[: ]+[nN][oO] [cC][oO][vV][eE][rR]')
  coverage.start()
  load_melange()

def load_melange():
  """
  Registers a core, and SoC, GSoC and GHOP modules callbacks, sitemap, sidebar
  and rights.
  """
  # Register a core for the test modules to use
  from soc.modules import callback
  from soc.modules.core import Core
  callback.registerCore(Core())
  current_core = callback.getCore()
  # Register SoC and GSoC modules callbacks
  current_core.registerModuleCallbacks()
  # Register GHOP modules callbacks
  from soc.modules.ghop.callback import Callback
  ghop_callback = Callback(current_core)
  ghop_callback.registerWithSitemap()
  ghop_callback.registerWithSidebar()
  ghop_callback.registerRights()
  from soc.modules.soc_core.callback import Callback as soc_core_Callback
  soc_core_callback = soc_core_Callback(current_core)
  soc_core_callback.registerWithSitemap()
  soc_core_callback.registerWithSidebar()
  soc_core_callback.registerRights()
  from soc.modules.gsoc.callback import Callback as gsoc_Callback
  gsoc_callback = gsoc_Callback(current_core)
  gsoc_callback.registerWithSitemap()
  gsoc_callback.registerWithSidebar()
  gsoc_callback.registerRights()

class AppEngineDatastoreClearPlugin(plugins.Plugin):
  """Nose plugin to clear the AppEngine datastore between tests.
  """
  name = 'AppEngineDatastoreClearPlugin'
  enabled = True
  def options(self, parser, env):
    return plugins.Plugin.options(self, parser, env)

  def configure(self, parser, env):
    plugins.Plugin.configure(self, parser, env)
    self.enabled = True

  def afterTest(self, test):
    from google.appengine.api import apiproxy_stub_map
    datastore = apiproxy_stub_map.apiproxy.GetStub('datastore')
    # clear datastore iff one is available
    if datastore is not None:
      datastore.Clear()

def main():
  sys.path = extra_paths + sys.path
  os.environ['SERVER_SOFTWARE'] = 'Development via nose'
  os.environ['SERVER_NAME'] = 'Foo'
  os.environ['SERVER_PORT'] = '8080'
  os.environ['APPLICATION_ID'] = 'test-app-run'
  os.environ['USER_EMAIL'] = 'test@example.com'
  os.environ['CURRENT_VERSION_ID'] = 'testing-version'
  os.environ['HTTP_HOST'] = 'some.testing.host.tld'
  import main as app_main
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_file_stub
  from google.appengine.api import mail_stub
  from google.appengine.api import user_service_stub
  from google.appengine.api import urlfetch_stub
  from google.appengine.api.memcache import memcache_stub
  from google.appengine.api.labs.taskqueue import taskqueue_stub
  apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
  apiproxy_stub_map.apiproxy.RegisterStub('urlfetch',
                                          urlfetch_stub.URLFetchServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('user',
                                          user_service_stub.UserServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('datastore',
    datastore_file_stub.DatastoreFileStub('test-app-run', None, None))
  apiproxy_stub_map.apiproxy.RegisterStub('memcache',
    memcache_stub.MemcacheServiceStub())
  apiproxy_stub_map.apiproxy.RegisterStub('mail', mail_stub.MailServiceStub())
  yaml_location = os.path.join(HERE, 'app')
  apiproxy_stub_map.apiproxy.RegisterStub('taskqueue',
                 taskqueue_stub.TaskQueueServiceStub(root_path=yaml_location))
  import django.test.utils
  django.test.utils.setup_test_environment()

  plugins = [AppEngineDatastoreClearPlugin()]

  if '--coverage' in sys.argv:
    from nose.plugins import cover
    plugin = cover.Coverage()
    from tests.pymox import stubout
    stubout_obj = stubout.StubOutForTesting()
    stubout_obj.SmartSet(plugin, 'begin', begin)
    plugins.append(plugin)

    args = ['--with-coverage',
            '--cover-package=soc.',
            '--cover-erase',
            '--cover-html',
            '--cover-html-dir=coverageResults']

    sys.argv.remove('--coverage')
    sys.argv += args
  else:
    load_melange()

  nose.main(addplugins=plugins)

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>from django.shortcuts import render, redirect
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login, logout
from django.contrib import messages
from datetime import timedelta
from coursereg import models
from django.core.mail import send_mail
from django.db.models import Q
from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.views.decorators.http import require_POST
from django.contrib.sites.shortcuts import get_current_site
from django.conf import settings

@require_POST
@login_required
def create(request):
    course_id = request.POST['course_id']
    user_id = request.POST['user_id']
    reg_type = request.POST['reg_type']

    user = models.User.objects.get(id=user_id)
    if request.POST['origin'] == 'adviser':
        if not request.user == user.adviser: raise PermissionDenied
        if not settings.CAN_ADVISER_ADD_COURSES_FOR_STUDENTS: raise PermissionDenied
    else:
        if not request.user == user: raise PermissionDenied

    if not course_id:
        messages.error(request, 'Select a course.')
    else:
        course = models.Course.objects.get(id=course_id)
        if not course.is_start_reg_date_passed(): raise PermissionDenied
        if request.POST['origin'] == 'adviser':
            if course.is_last_adviser_approval_date_passed(): raise PermissionDenied
        else:
            if course.is_last_reg_date_passed(): raise PermissionDenied
        if models.Participant.objects.filter(user__id=user_id, course__id=course_id):
            messages.error(request, 'Already registered for %s.' % course)
        else:
            participant = models.Participant.objects.create(
                user=user,
                course=course,
                participant_type=models.Participant.PARTICIPANT_STUDENT,
                registration_type=models.RegistrationType.objects.get(pk=reg_type),
                should_count_towards_cgpa=True,
            )
            participant.save()
            if request.POST['origin'] == 'adviser':
                participant.is_adviser_reviewed = True
                participant.save()
                msg = 'Enrolled in %s.' % participant.course
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_ADVISER,
                                                   message=msg)
                try:
                    send_mail('New course application', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            else:
                messages.success(request, 'Applied for %s.' % course)

    return redirect(request.POST.get('next', reverse('coursereg:index')))

@require_POST
@login_required
def update(request, participant_id):
    participant = models.Participant.objects.get(id=participant_id)
    if request.POST['origin'] == 'instructor':
        if not models.Participant.objects.filter(course=participant.course,
                user=request.user, participant_type=models.Participant.PARTICIPANT_INSTRUCTOR):
            raise PermissionDenied
        if request.POST['action'] == 'review':
            participant.is_instructor_reviewed = True
            participant.save()
        if request.POST['action'] == 'grade':
            if participant.course.is_last_grade_date_passed(): raise PermissionDenied
            if request.POST['grade'] == 'null':
                participant.grade = None
            else:
                participant.grade = models.Grade.objects.get(id=request.POST['grade'])
            participant.save()
    elif request.POST['origin'] == 'adviser':
        if not participant.user.adviser == request.user: raise PermissionDenied
        if request.POST['action'] == 'review':
            participant.is_adviser_reviewed = True
            participant.save()
        if request.POST['action'] == 'reg_type_change':
            reg_type = models.RegistrationType.objects.get(pk=request.POST['reg_type'])
            if participant.course.is_last_conversion_date_passed(): raise PermissionDenied
            participant.registration_type = reg_type
            participant.save()
            msg = 'Registration for %s has changed to %s.' % (participant.course, reg_type)
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'drop':
            if participant.course.is_last_drop_date_passed(): raise PermissionDenied
            participant.is_drop = True
            participant.save()
            msg = 'Course %s dropped.' % participant.course
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'undrop':
            if participant.course.is_last_drop_date_passed(): raise PermissionDenied
            participant.is_drop = False
            participant.save()
            msg = 'Drop request for %s is cancelled.' % participant.course
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'disable_student_edits':
            participant.lock_from_student = True
            participant.save()
        if request.POST['action'] == 'enable_student_edits':
            participant.lock_from_student = False
            participant.save()
    elif request.POST['origin'] == 'student':
        student = request.user
        if not request.user == participant.user: raise PermissionDenied
        if participant.lock_from_student:
            messages.error(request, "Permission denied.")
        else:
            if request.POST['action'] == 'reg_type_change':
                reg_type = models.RegistrationType.objects.get(pk=request.POST['reg_type'])
                if participant.course.is_last_conversion_date_passed(): raise PermissionDenied
                participant.registration_type = reg_type
                participant.is_adviser_reviewed = False
                participant.save()
                msg = 'Registration for %s by %s has changed to %s.' % (participant.course, participant.user, reg_type)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            if request.POST['action'] == 'drop':
                if participant.course.is_last_drop_date_passed(): raise PermissionDenied
                participant.is_drop = True
                participant.is_adviser_reviewed = False
                participant.save()
                msg = '%s has dropped %s.' % (participant.user, participant.course)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            if request.POST['action'] == 'undrop':
                if participant.course.is_last_drop_date_passed(): raise PermissionDenied
                participant.is_drop = False
                participant.is_adviser_reviewed = False
                participant.save()
                msg = 'Drop request for %s by %s cancelled.' % (participant.course, participant.user)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')

    return redirect(request.POST.get('next', reverse('coursereg:index')))

@require_POST
@login_required
def delete(request, participant_id):
    participant = models.Participant.objects.get(id=participant_id)
    student = participant.user
    if request.POST['origin'] == 'student':
        if request.user != participant.user: raise PermissionDenied
        if participant.course.is_last_cancellation_date_passed(): raise PermissionDenied
        if participant.lock_from_student:
            messages.error(request, "Permission denied.")
            raise PermissionDenied
        participant.delete()
        if participant.course.is_last_reg_date_passed():
            msg = 'Registration for %s was cancelled by %s.' % (participant.course, student)
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_STUDENT,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    elif request.POST['origin'] == 'adviser':
        if request.user != participant.user.adviser: raise PermissionDenied
        if participant.course.is_last_adviser_approval_date_passed(): raise PermissionDenied
        participant.delete()
        msg = 'Registration for %s was cancelled.' % participant.course
        models.Notification.objects.create(user=participant.user,
                                           origin=models.Notification.ORIGIN_ADVISER,
                                           message=msg)
        try:
            send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
        except:
            messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    elif request.POST['origin'] == 'instructor':
        if not models.Participant.objects.filter(course=participant.course, user=request.user, participant_type=models.Participant.PARTICIPANT_INSTRUCTOR).first(): raise PermissionDenied
        if participant.course.is_last_instructor_approval_date_passed(): raise PermissionDenied
        participant.delete()
        msg = 'Course registration application for %s by %s was cancelled by the course instructor.' % (participant.course, participant.user)
        models.Notification.objects.create(user=participant.user,
                                           origin=models.Notification.ORIGIN_INSTRUCTOR,
                                           message=msg)
        try:
            send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
        except:
            messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    return redirect(request.POST.get('next', reverse('coursereg:index')))
<commit_msg>Fix student course cancel bug<commit_after>from django.shortcuts import render, redirect
from django.core.urlresolvers import reverse
from django.http import HttpResponse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login, logout
from django.contrib import messages
from datetime import timedelta
from coursereg import models
from django.core.mail import send_mail
from django.db.models import Q
from django.conf import settings
from django.core.exceptions import PermissionDenied
from django.views.decorators.http import require_POST
from django.contrib.sites.shortcuts import get_current_site
from django.conf import settings

@require_POST
@login_required
def create(request):
    course_id = request.POST['course_id']
    user_id = request.POST['user_id']
    reg_type = request.POST['reg_type']

    user = models.User.objects.get(id=user_id)
    if request.POST['origin'] == 'adviser':
        if not request.user == user.adviser: raise PermissionDenied
        if not settings.CAN_ADVISER_ADD_COURSES_FOR_STUDENTS: raise PermissionDenied
    else:
        if not request.user == user: raise PermissionDenied

    if not course_id:
        messages.error(request, 'Select a course.')
    else:
        course = models.Course.objects.get(id=course_id)
        if not course.is_start_reg_date_passed(): raise PermissionDenied
        if request.POST['origin'] == 'adviser':
            if course.is_last_adviser_approval_date_passed(): raise PermissionDenied
        else:
            if course.is_last_reg_date_passed(): raise PermissionDenied
        if models.Participant.objects.filter(user__id=user_id, course__id=course_id):
            messages.error(request, 'Already registered for %s.' % course)
        else:
            participant = models.Participant.objects.create(
                user=user,
                course=course,
                participant_type=models.Participant.PARTICIPANT_STUDENT,
                registration_type=models.RegistrationType.objects.get(pk=reg_type),
                should_count_towards_cgpa=True,
            )
            participant.save()
            if request.POST['origin'] == 'adviser':
                participant.is_adviser_reviewed = True
                participant.save()
                msg = 'Enrolled in %s.' % participant.course
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_ADVISER,
                                                   message=msg)
                try:
                    send_mail('New course application', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            else:
                messages.success(request, 'Applied for %s.' % course)

    return redirect(request.POST.get('next', reverse('coursereg:index')))

@require_POST
@login_required
def update(request, participant_id):
    participant = models.Participant.objects.get(id=participant_id)
    if request.POST['origin'] == 'instructor':
        if not models.Participant.objects.filter(course=participant.course,
                user=request.user, participant_type=models.Participant.PARTICIPANT_INSTRUCTOR):
            raise PermissionDenied
        if request.POST['action'] == 'review':
            participant.is_instructor_reviewed = True
            participant.save()
        if request.POST['action'] == 'grade':
            if participant.course.is_last_grade_date_passed(): raise PermissionDenied
            if request.POST['grade'] == 'null':
                participant.grade = None
            else:
                participant.grade = models.Grade.objects.get(id=request.POST['grade'])
            participant.save()
    elif request.POST['origin'] == 'adviser':
        if not participant.user.adviser == request.user: raise PermissionDenied
        if request.POST['action'] == 'review':
            participant.is_adviser_reviewed = True
            participant.save()
        if request.POST['action'] == 'reg_type_change':
            reg_type = models.RegistrationType.objects.get(pk=request.POST['reg_type'])
            if participant.course.is_last_conversion_date_passed(): raise PermissionDenied
            participant.registration_type = reg_type
            participant.save()
            msg = 'Registration for %s has changed to %s.' % (participant.course, reg_type)
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'drop':
            if participant.course.is_last_drop_date_passed(): raise PermissionDenied
            participant.is_drop = True
            participant.save()
            msg = 'Course %s dropped.' % participant.course
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'undrop':
            if participant.course.is_last_drop_date_passed(): raise PermissionDenied
            participant.is_drop = False
            participant.save()
            msg = 'Drop request for %s is cancelled.' % participant.course
            models.Notification.objects.create(user=participant.user,
                                               origin=models.Notification.ORIGIN_ADVISER,
                                               message=msg)
            try:
                send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
            except:
                messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
        if request.POST['action'] == 'disable_student_edits':
            participant.lock_from_student = True
            participant.save()
        if request.POST['action'] == 'enable_student_edits':
            participant.lock_from_student = False
            participant.save()
    elif request.POST['origin'] == 'student':
        student = request.user
        if not request.user == participant.user: raise PermissionDenied
        if participant.lock_from_student:
            messages.error(request, "Permission denied.")
        else:
            if request.POST['action'] == 'reg_type_change':
                reg_type = models.RegistrationType.objects.get(pk=request.POST['reg_type'])
                if participant.course.is_last_conversion_date_passed(): raise PermissionDenied
                participant.registration_type = reg_type
                participant.is_adviser_reviewed = False
                participant.save()
                msg = 'Registration for %s by %s has changed to %s.' % (participant.course, participant.user, reg_type)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            if request.POST['action'] == 'drop':
                if participant.course.is_last_drop_date_passed(): raise PermissionDenied
                participant.is_drop = True
                participant.is_adviser_reviewed = False
                participant.save()
                msg = '%s has dropped %s.' % (participant.user, participant.course)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
            if request.POST['action'] == 'undrop':
                if participant.course.is_last_drop_date_passed(): raise PermissionDenied
                participant.is_drop = False
                participant.is_adviser_reviewed = False
                participant.save()
                msg = 'Drop request for %s by %s cancelled.' % (participant.course, participant.user)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')

    return redirect(request.POST.get('next', reverse('coursereg:index')))

@require_POST
@login_required
def delete(request, participant_id):
    participant = models.Participant.objects.get(id=participant_id)
    student = participant.user
    if request.POST['origin'] == 'student':
        if request.user != participant.user: raise PermissionDenied
        if participant.course.is_last_cancellation_date_passed(): raise PermissionDenied
        if participant.lock_from_student:
            messages.error(request, "Permission denied.")
        else:
            participant.delete()
            if participant.course.is_last_reg_date_passed():
                msg = 'Registration for %s was cancelled by %s.' % (participant.course, student)
                models.Notification.objects.create(user=participant.user,
                                                   origin=models.Notification.ORIGIN_STUDENT,
                                                   message=msg)
                try:
                    send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.adviser.email])
                except:
                    messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    elif request.POST['origin'] == 'adviser':
        if request.user != participant.user.adviser: raise PermissionDenied
        if participant.course.is_last_adviser_approval_date_passed(): raise PermissionDenied
        participant.delete()
        msg = 'Registration for %s was cancelled.' % participant.course
        models.Notification.objects.create(user=participant.user,
                                           origin=models.Notification.ORIGIN_ADVISER,
                                           message=msg)
        try:
            send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
        except:
            messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    elif request.POST['origin'] == 'instructor':
        if not models.Participant.objects.filter(course=participant.course, user=request.user, participant_type=models.Participant.PARTICIPANT_INSTRUCTOR).first(): raise PermissionDenied
        if participant.course.is_last_instructor_approval_date_passed(): raise PermissionDenied
        participant.delete()
        msg = 'Course registration application for %s by %s was cancelled by the course instructor.' % (participant.course, participant.user)
        models.Notification.objects.create(user=participant.user,
                                           origin=models.Notification.ORIGIN_INSTRUCTOR,
                                           message=msg)
        try:
            send_mail('Course registration changed', msg, settings.DEFAULT_FROM_EMAIL, [participant.user.email])
        except:
            messages.warning(request, 'Error sending e-mail. But a notification has been created on this website.')
    return redirect(request.POST.get('next', reverse('coursereg:index')))
<|endoftext|>


<commit_before>#!/usr/bin/python3
"""
apiage.py - a quick and dirty tool for tracking when apis become stable
and deprecated apis are to be removed.

PDX-License-Identifier: MIT
"""

import argparse
import copy
import json
import sys

def read_json(path):
    try:
        with open(path, "r") as fh:
            data = json.load(fh)
    except FileNotFoundError:
        return {}
    return data

def write_json(path, data):
    if path is None:
        raise ValueError("a valid path is required")
    with open(path, "w") as fh:
        json.dump(data, fh, indent=2)

def write_markdown(path, data):
    if path is None:
        return
    with open(path, "w") as fh:
        format_markdown(data, fh)

def copy_api(tracked, keys, src, defaults=None):
    dst = tracked
    for key in keys[:-1]:
        dst = dst.setdefault(key, {})
    dst = dst.setdefault(keys[-1], [])
    added = []
    for gfunc in src:
        name = gfunc["name"]
        if name in [d["name"] for d in dst]:
            continue
        gfunc.update(defaults or {})
        dst.append(gfunc)
        added.append(gfunc)
    return added

def compare_and_update(tracked, pkg, pkg_api, defaults=None):
    if defaults is None:
        defaults = {}
    new_deprecated = new_preview = new_stable = []
    if "deprecated_api" in pkg_api:
        new_deprecated = copy_api(
            tracked=tracked,
            keys=[pkg, "deprecated_api"],
            src=pkg_api["deprecated_api"],
            defaults={
                "deprecated_in_version": defaults.get(
                    "deprecated_in_version", ""
                ),
                "expected_remove_version": defaults.get(
                    "expected_remove_version", ""
                ),
            },
        )
    if "preview_api" in pkg_api:
        new_preview = copy_api(
            tracked=tracked,
            keys=[pkg, "preview_api"],
            src=pkg_api["preview_api"],
            defaults={
                "added_in_version": defaults.get("added_in_version", ""),
                "expected_stable_version": defaults.get(
                    "expected_stable_version", ""
                ),
            },
        )
    if "stable_api" in pkg_api:
        new_stable = copy_api(
            tracked=tracked,
            keys=[pkg, "stable_api"],
            src=pkg_api["stable_api"],
        )
    return new_deprecated, new_preview, new_stable

def api_update(tracked, src, copy_stable=False, defaults=None):
    for pkg, pkg_api in src.items():
        _, _, new_stable = compare_and_update(
            tracked, pkg, pkg_api, defaults=defaults
        )
        if new_stable and not copy_stable:
            return len(new_stable)
    return 0

def api_compare(tracked, src):
    problems = 0
    tmp = copy.deepcopy(tracked)
    for pkg, pkg_api in src.items():
        new_deprecated, new_preview, new_stable = compare_and_update(
            tmp, pkg, pkg_api
        )
        for dapi in new_deprecated:
            print("not tracked (deprecated):", pkg, dapi["name"])
            problems += 1
        for papi in new_preview:
            print("not tracked (preview):", pkg, papi["name"])
            problems += 1
        for sapi in new_stable:
            print("not tracked (stable):", pkg, sapi["name"])
            problems += 1
    for pkg, pkg_api in tmp.items():
        for api in pkg_api.get("deprecated_api", []):
            if not api.get("deprecated_in_version"):
                print("no deprecated_in_version set:", pkg, api["name"])
                problems += 1
        for api in pkg_api.get("preview_api", []):
            if not api.get("added_in_version"):
                print("no added_in_version set:", pkg, api["name"])
                problems += 1
            if not api.get("expected_stable_version"):
                print("no expected_stable_version set:", pkg, api["name"])
                problems += 1
    return problems

def format_markdown(tracked, outfh):
    print("<!-- GENERATED FILE: DO NOT EDIT DIRECTLY -->", file=outfh)
    print("", file=outfh)
    print("# go-ceph API Stability", file=outfh)
    print("", file=outfh)
    for pkg, pkg_api in tracked.items():
        print(f"## Package: {pkg}", file=outfh)
        print("", file=outfh)
        if "preview_api" in pkg_api:
            print("### Preview APIs", file=outfh)
            print("", file=outfh)
            _table(
                pkg_api["preview_api"],
                columns=[
                    ("Name", "name"),
                    ("Added in Version", "added_in_version"),
                    ("Expected Stable Version", "expected_stable_version"),
                ],
                outfh=outfh,
            )
            print("", file=outfh)
        if "deprecated_api" in pkg_api:
            print("### Deprecated APIs", file=outfh)
            print("", file=outfh)
            _table(
                pkg_api["deprecated_api"],
                columns=[
                    ("Name", "name"),
                    ("Deprecated in Version", "deprecated_in_version"),
                    ("Expected Removal Version", "expected_remove_version"),
                ],
                outfh=outfh,
            )
            print("", file=outfh)

def _table(data, columns, outfh):
    for key, _ in columns:
        outfh.write(key)
        outfh.write(" | ")
    outfh.write("\n")
    for key, _ in columns:
        outfh.write("-" * len(key))
        outfh.write(" | ")
    outfh.write("\n")
    for entry in data:
        for _, dname in columns:
            outfh.write(entry[dname])
            outfh.write(" | ")
        outfh.write("\n")

def _setif(dct, key, value):
    if value:
        dct[key] = value

def _vfmt(x, y, z):
    return f"v{x}.{y}.{z}"

def tag_to_versions(cli, version_tag):
    # first: parse the tag
    if not version_tag.startswith("v"):
        raise ValueError(f"unexpected tag: {version_tag}")
    try:
        x, y, z = [int(val) for val in version_tag[1:].split(".")]
    except ValueError:
        raise ValueError(f"unexpected tag: {version_tag}")
    # set values according to the simple policy:
    # where version is X.Y.Z
    #  * added in: X+1
    #  * expected stable in: X+1+2
    #  * deprecated in: X+1
    # if they weren't manually specified
    if not cli.added_in_version:
        cli.added_in_version = _vfmt(x, y + 1, z)
    if not cli.stable_in_version:
        cli.stable_in_version = _vfmt(x, y + 3, z)
    if not cli.deprecated_in_version:
        cli.deprecated_in_version = _vfmt(x, y + 1, z)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--source",
        "-s",
        default="./_results/implements.json",
        help="json describing state of code",
    )
    parser.add_argument(
        "--current",
        "-c",
        default="./docs/api-status.json",
        help="json tracking current apis",
    )
    parser.add_argument(
        "--document",
        "-d",
        default="./docs/api-status.md",
        help="markdown file describing current apis",
    )
    parser.add_argument(
        "--mode",
        choices=("compare", "update", "write-doc"),
        default="compare",
        help="either update current state or compare current state to source",
    )
    parser.add_argument(
        "--copy-stable-apis",
        action="store_true",
        help="allow copying of pre-existing stable APIs",
    )
    parser.add_argument(
        "--added-in-version",
        "-A",
        help="specify an added-in version for all new preview apis",
    )
    parser.add_argument(
        "--stable-in-version",
        "-S",
        help="specify a stable-in version for all new preview apis",
    )
    parser.add_argument(
        "--deprecated-in-version",
        "-D",
        help="specify a deprecated-in version for all newly deprecated apis",
    )
    parser.add_argument(
        "--remove-in-version",
        "-R",
        help="specify a version that this deprecated api is expected to be removed",
    )
    parser.add_argument(
        "--current-tag",
        "-t",
        help=(
            "Specify the current VCS tag. This will be used to automatically"
            " set version values if not otherwise specified."
        ),
    )
    cli = parser.parse_args()

    api_src = read_json(cli.source) if cli.source else {}
    api_tracked = read_json(cli.current) if cli.current else {}

    if not api_src:
        print(
            f"error: no source data found (path: {cli.source})", file=sys.stderr
        )
        sys.exit(1)

    if cli.current_tag:
        tag_to_versions(cli, cli.current_tag)

    if cli.mode == "compare":
        # just compare the json files. useful for CI
        pcount = api_compare(api_tracked, api_src)
        if pcount:
            print(f"error: {pcount} problems detected", file=sys.stderr)
            sys.exit(1)
    elif cli.mode == "update":
        # update the current/tracked apis with those from the source
        defaults = {}
        _setif(defaults, "added_in_version", cli.added_in_version)
        _setif(defaults, "expected_stable_version", cli.stable_in_version)
        _setif(defaults, "deprecated_in_version", cli.deprecated_in_version)
        _setif(defaults, "expected_remove_version", cli.remove_in_version)
        pcount = api_update(
            api_tracked,
            api_src,
            copy_stable=cli.copy_stable_apis,
            defaults=defaults,
        )
        if pcount:
            print(f"error: {pcount} problems detected", file=sys.stderr)
            sys.exit(1)
        write_json(cli.current, api_tracked)
        write_markdown(cli.document, api_tracked)
    elif cli.mode == "write-doc":
        write_markdown(cli.document, api_tracked)

if __name__ == "__main__":
    main()
<commit_msg>contrib: add new fix-versions mode to apiage.py<commit_after>#!/usr/bin/python3
"""
apiage.py - a quick and dirty tool for tracking when apis become stable
and deprecated apis are to be removed.

PDX-License-Identifier: MIT
"""

import argparse
import copy
import json
import re
import sys

def read_json(path):
    try:
        with open(path, "r") as fh:
            data = json.load(fh)
    except FileNotFoundError:
        return {}
    return data

def write_json(path, data):
    if path is None:
        raise ValueError("a valid path is required")
    with open(path, "w") as fh:
        json.dump(data, fh, indent=2)

def write_markdown(path, data):
    if path is None:
        return
    with open(path, "w") as fh:
        format_markdown(data, fh)

def copy_api(tracked, keys, src, defaults=None):
    dst = tracked
    for key in keys[:-1]:
        dst = dst.setdefault(key, {})
    dst = dst.setdefault(keys[-1], [])
    added = []
    for gfunc in src:
        name = gfunc["name"]
        if name in [d["name"] for d in dst]:
            continue
        gfunc.update(defaults or {})
        dst.append(gfunc)
        added.append(gfunc)
    return added

def compare_and_update(tracked, pkg, pkg_api, defaults=None):
    if defaults is None:
        defaults = {}
    new_deprecated = new_preview = new_stable = []
    if "deprecated_api" in pkg_api:
        new_deprecated = copy_api(
            tracked=tracked,
            keys=[pkg, "deprecated_api"],
            src=pkg_api["deprecated_api"],
            defaults={
                "deprecated_in_version": defaults.get(
                    "deprecated_in_version", ""
                ),
                "expected_remove_version": defaults.get(
                    "expected_remove_version", ""
                ),
            },
        )
    if "preview_api" in pkg_api:
        new_preview = copy_api(
            tracked=tracked,
            keys=[pkg, "preview_api"],
            src=pkg_api["preview_api"],
            defaults={
                "added_in_version": defaults.get("added_in_version", ""),
                "expected_stable_version": defaults.get(
                    "expected_stable_version", ""
                ),
            },
        )
    if "stable_api" in pkg_api:
        new_stable = copy_api(
            tracked=tracked,
            keys=[pkg, "stable_api"],
            src=pkg_api["stable_api"],
        )
    return new_deprecated, new_preview, new_stable

def api_update(tracked, src, copy_stable=False, defaults=None):
    for pkg, pkg_api in src.items():
        _, _, new_stable = compare_and_update(
            tracked, pkg, pkg_api, defaults=defaults
        )
        if new_stable and not copy_stable:
            return len(new_stable)
    return 0

def api_compare(tracked, src):
    problems = 0
    tmp = copy.deepcopy(tracked)
    for pkg, pkg_api in src.items():
        new_deprecated, new_preview, new_stable = compare_and_update(
            tmp, pkg, pkg_api
        )
        for dapi in new_deprecated:
            print("not tracked (deprecated):", pkg, dapi["name"])
            problems += 1
        for papi in new_preview:
            print("not tracked (preview):", pkg, papi["name"])
            problems += 1
        for sapi in new_stable:
            print("not tracked (stable):", pkg, sapi["name"])
            problems += 1
    for pkg, pkg_api in tmp.items():
        for api in pkg_api.get("deprecated_api", []):
            if not api.get("deprecated_in_version"):
                print("no deprecated_in_version set:", pkg, api["name"])
                problems += 1
        for api in pkg_api.get("preview_api", []):
            if not api.get("added_in_version"):
                print("no added_in_version set:", pkg, api["name"])
                problems += 1
            if not api.get("expected_stable_version"):
                print("no expected_stable_version set:", pkg, api["name"])
                problems += 1
    return problems

def api_fix_versions(tracked, values, pred=None):
    """Walks through tracked API and fixes any placeholder versions to real version numbers.
    """
    for pkg, pkg_api in tracked.items():
        for api in pkg_api.get("deprecated_api", []):
            if pred and not pred(pkg, api["name"]):
                print(f"Skipping {pkg}:{api['name']} due to filter")
                continue
            _vfix(pkg, "deprecated_in_version", api, values)
            _vfix(pkg, "expected_remove_version", api, values)
        for api in pkg_api.get("preview_api", []):
            if pred and not pred(pkg, api["name"]):
                print(f"Skipping {pkg}:{api['name']} due to filter")
                continue
            _vfix(pkg, "added_in_version", api, values)
            _vfix(pkg, "expected_stable_version", api, values)

def format_markdown(tracked, outfh):
    print("<!-- GENERATED FILE: DO NOT EDIT DIRECTLY -->", file=outfh)
    print("", file=outfh)
    print("# go-ceph API Stability", file=outfh)
    print("", file=outfh)
    for pkg, pkg_api in tracked.items():
        print(f"## Package: {pkg}", file=outfh)
        print("", file=outfh)
        if "preview_api" in pkg_api:
            print("### Preview APIs", file=outfh)
            print("", file=outfh)
            _table(
                pkg_api["preview_api"],
                columns=[
                    ("Name", "name"),
                    ("Added in Version", "added_in_version"),
                    ("Expected Stable Version", "expected_stable_version"),
                ],
                outfh=outfh,
            )
            print("", file=outfh)
        if "deprecated_api" in pkg_api:
            print("### Deprecated APIs", file=outfh)
            print("", file=outfh)
            _table(
                pkg_api["deprecated_api"],
                columns=[
                    ("Name", "name"),
                    ("Deprecated in Version", "deprecated_in_version"),
                    ("Expected Removal Version", "expected_remove_version"),
                ],
                outfh=outfh,
            )
            print("", file=outfh)

def _table(data, columns, outfh):
    for key, _ in columns:
        outfh.write(key)
        outfh.write(" | ")
    outfh.write("\n")
    for key, _ in columns:
        outfh.write("-" * len(key))
        outfh.write(" | ")
    outfh.write("\n")
    for entry in data:
        for _, dname in columns:
            outfh.write(entry[dname])
            outfh.write(" | ")
        outfh.write("\n")

def _setif(dct, key, value):
    if value:
        dct[key] = value

def _vfmt(x, y, z):
    return f"v{x}.{y}.{z}"

def _vfix(pkg, key, api, values):
    if api.get(key, "").startswith("$"):
        try:
            val = values[key]
        except KeyError:
            raise ValueError(f"missing {key} in values: {key} must be provided to fix apis")
        api[key] = val
        print(f"Updated {pkg}:{api['name']} {key}={values[key]}")

def _make_fix_filter(cli):
    pkgre = namere = None
    if cli.fix_filter_pkg:
        pkgre = re.compile(cli.fix_filter_pkg)
    if cli.fix_filter_func:
        namere = re.compile(cli.fix_filter_func)

    def f(pkg, fname):
        if pkgre and not pkgre.match(pkg):
            return False
        if namere and not namere.match(fname):
            return False
        return True

    return f

def tag_to_versions(cli, version_tag):
    # first: parse the tag
    if not version_tag.startswith("v"):
        raise ValueError(f"unexpected tag: {version_tag}")
    try:
        x, y, z = [int(val) for val in version_tag[1:].split(".")]
    except ValueError:
        raise ValueError(f"unexpected tag: {version_tag}")
    # set values according to the simple policy:
    # where version is X.Y.Z
    #  * added in: X+1
    #  * expected stable in: X+1+2
    #  * deprecated in: X+1
    # if they weren't manually specified
    if not cli.added_in_version:
        cli.added_in_version = _vfmt(x, y + 1, z)
    if not cli.stable_in_version:
        cli.stable_in_version = _vfmt(x, y + 3, z)
    if not cli.deprecated_in_version:
        cli.deprecated_in_version = _vfmt(x, y + 1, z)

def placeholder_versions(cli):
    if not cli.added_in_version:
        cli.added_in_version = "$NEXT_RELEASE"
    if not cli.stable_in_version:
        cli.stable_in_version = "$NEXT_RELEASE_STABLE"
    if not cli.deprecated_in_version:
        cli.deprecated_in_version = "$NEXT_RELEASE"

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--source",
        "-s",
        default="./_results/implements.json",
        help="json describing state of code",
    )
    parser.add_argument(
        "--current",
        "-c",
        default="./docs/api-status.json",
        help="json tracking current apis",
    )
    parser.add_argument(
        "--document",
        "-d",
        default="./docs/api-status.md",
        help="markdown file describing current apis",
    )
    parser.add_argument(
        "--mode",
        choices=("compare", "update", "write-doc", "fix-versions"),
        default="compare",
        help="either update current state or compare current state to source",
    )
    parser.add_argument(
        "--copy-stable-apis",
        action="store_true",
        help="allow copying of pre-existing stable APIs",
    )
    parser.add_argument(
        "--added-in-version",
        "-A",
        help="specify an added-in version for all new preview apis",
    )
    parser.add_argument(
        "--stable-in-version",
        "-S",
        help="specify a stable-in version for all new preview apis",
    )
    parser.add_argument(
        "--deprecated-in-version",
        "-D",
        help="specify a deprecated-in version for all newly deprecated apis",
    )
    parser.add_argument(
        "--remove-in-version",
        "-R",
        help="specify a version that this deprecated api is expected to be removed",
    )
    parser.add_argument(
        "--current-tag",
        "-t",
        help=(
            "Specify the current VCS tag. This will be used to automatically"
            " set version values if not otherwise specified."
        ),
    )
    parser.add_argument(
        "--placeholder-versions",
        action="store_true",
        help="Specify special placeholder values for version numbers.",
    )
    parser.add_argument(
        "--fix-filter-pkg",
        help="Specify a regular expression to filter on package names.",
    )
    parser.add_argument(
        "--fix-filter-func",
        help="Specify a regular expression to filter on function names.",
    )
    cli = parser.parse_args()

    api_src = read_json(cli.source) if cli.source else {}
    api_tracked = read_json(cli.current) if cli.current else {}

    if not api_src:
        print(
            f"error: no source data found (path: {cli.source})", file=sys.stderr
        )
        sys.exit(1)

    if cli.current_tag:
        tag_to_versions(cli, cli.current_tag)
    elif cli.placeholder_versions:
        if cli.mode == "fix-versions":
            raise ValueError("fix-versions requires real version numbers")
        placeholder_versions(cli)

    if cli.mode == "compare":
        # just compare the json files. useful for CI
        pcount = api_compare(api_tracked, api_src)
        if pcount:
            print(f"error: {pcount} problems detected", file=sys.stderr)
            sys.exit(1)
    elif cli.mode == "update":
        # update the current/tracked apis with those from the source
        defaults = {}
        _setif(defaults, "added_in_version", cli.added_in_version)
        _setif(defaults, "expected_stable_version", cli.stable_in_version)
        _setif(defaults, "deprecated_in_version", cli.deprecated_in_version)
        _setif(defaults, "expected_remove_version", cli.remove_in_version)
        pcount = api_update(
            api_tracked,
            api_src,
            copy_stable=cli.copy_stable_apis,
            defaults=defaults,
        )
        if pcount:
            print(f"error: {pcount} problems detected", file=sys.stderr)
            sys.exit(1)
        write_json(cli.current, api_tracked)
        write_markdown(cli.document, api_tracked)
    elif cli.mode == "fix-versions":
        values = {}
        _setif(values, "added_in_version", cli.added_in_version)
        _setif(values, "expected_stable_version", cli.stable_in_version)
        _setif(values, "deprecated_in_version", cli.deprecated_in_version)
        _setif(values, "expected_remove_version", cli.remove_in_version)
        api_fix_versions(api_tracked, values=values, pred=_make_fix_filter(cli))
        write_json(cli.current, api_tracked)
    elif cli.mode == "write-doc":
        write_markdown(cli.document, api_tracked)

if __name__ == "__main__":
    main()
<|endoftext|>