<commit_before># -*- mode: python; tab-width: 2; coding: utf8 -*-
#
# Copyright (C) 2015 Niklas Rosenstein
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
'''
`git_auth` - a Git authentication layer alternative
===================================================
'''

import os
import sys
import argparse
import collections
import errno
import itertools
import shlex
import shutil
import subprocess
import textwrap
import traceback
import re

__author__ = 'Niklas Rosenstein <rosensteinniklas(at)gmail.com>'
__version__ = '0.9.0'

ACCESS_READ = 1 << 0
ACCESS_WRITE = 1 << 1
ACCESS_MANAGE = 1 << 2

def get_subpath(path, parent):
  relpath = os.path.relpath(path, parent)
  if relpath == os.curdir or relpath.startswith(os.pardir):
    return None
  return relpath

def is_subpath(path, parent):
  return bool(get_subpath(path, parent))

def confirm(question):
  ''' Asks the specified *question* and requests the user to reply with
  yes or no. Returns True if yes was replied, False if no. Will ask the
  question again if an invalid reply was given. '''

  while True:
    reply = input(question + ' [y/n] ').strip().lower()
    if reply in ('yes', 'y'):
      return True
    elif reply in ('no', 'n'):
      return False
    else:
      print("Please reply with yes/y or no/n.")

# == Git Auth Layer ===========================================================
# =============================================================================

class AccessController(object):
  ''' This interface describes the controller to manage access to Git
  repositories. '''

  User = collections.namedtuple('User', 'name home shell_access')

  class UnknownUser(Exception): pass

  def get_user_info(self, session, user_name):
    ''' Return user information for the specified *user_name* or raise
    `AccessController.UnknownUser` if the user does not exist. '''

    raise self.UnknownUser(user_name)

  def get_access_info(self, session, user_name, path):
    ''' Return a bit mask that indicates the access privileges of the
    user with the specified *user_name* to the *path*. The *path* would
    usually be a sub path of the `repository_root` in the configuration. '''

    raise NotImplementedError

class SimpleAccessController(AccessController):
  ''' This simple implementation of the `AccessController` interface
  allows all registered users to manage their home directory via SSH. '''

  def __init__(self, has_root=False):
    super().__init__()
    self.has_root = has_root

  def get_user_info(self, session, user_name):
    if not re.match('[A-z0-9\-_]', user_name):
      raise self.UnknownUser(user_name)
    if self.has_root and user_name == 'root':
      return self.User(user_name, '/', True)
    return self.User(user_name, '/' + user_name, True)

  def get_access_info(self, session, user_name, path):
    if self.has_root and user_name == 'root':
      if is_subpath(path, session.config.repository_root):
        return ACCESS_READ | ACCESS_WRITE | ACCESS_MANAGE

    home = os.path.join(session.config.repository_root, user_name)
    if is_subpath(path, home):
      return ACCESS_READ | ACCESS_WRITE | ACCESS_MANAGE
    return 0

class GitAuthSession(object):
  ''' This is the central authentication and repository management class. '''

  def __init__(self, user, config=None):
    super().__init__()
    if config is None:
      import git_auth_config as config
    self.user = config.access_controller.get_user_info(self, user)
    self.config = config

  def command(self, command):
    ''' Execute the specified command list. The first element in the
    list is used as the command name and dispatched by the default
    commands and the additional commands in the configuration. The
    exit code of the command is returned. '''

    fname = 'command_' + command[0]
    command_func = None
    if fname in globals():
      command_func = globals()[fname]
    if hasattr(self.config, fname):
      command_func = getattr(self.config, fname)
    if not callable(command_func):
      print("unknown command:", command[0], file=sys.stderr)
    else:
      try:
        return command_func(self, command[1:])
      except SystemExit as exc:
        return exc.code
    return 255

  def commands(self):
    ''' Returns a dictionary of all available commands mapping the name
    to the command function that implements it. '''

    commands = {}
    chain = vars(self.config).items(), globals().items()
    for key, value in itertools.chain(*chain):
      if callable(value) and key.startswith('command_'):
        name = key[8:]
        commands[name] = value

    return commands

  def cmdloop(self, intro='git-auth$ '):
    ''' Enters the interactive shell. '''

    header = "git-auth v{0} - Copyright (C) 2015 {1}"
    print(header.format(__version__, __author__))

    while True:
      command = shlex.split(input(intro))
      if command:
        if command[0] == 'exit':
          break
        elif command[0] == '?':
          command[0] = 'help'
        self.command(command)

  def repo2path(self, repo_name):
    ''' Converts a relative repository name to an absolute path on the
    filesystem based on the repository root directory as specified in
    the configuration. '''

    if os.name == 'nt':
      repo_name = repo_name.replace('/', '\\')

    if not repo_name.endswith('.git'):
      repo_name += '.git'
    path = os.path.join(self.config.repository_root, repo_name)
    path = os.path.normpath(path)
    return path

  def path2repo(self, path):
    ''' Converts the specified absolute path to a relative repository
    name. The original path can be obtained with `repo2path()`. The
    path is returned even if the repository does not exist. The `.git`
    suffix is removed from the path. None will be returned if the
    path does not end with `.git` or is not inside the repository
    root directory. '''

    if not path.endswith('.git'):
      return None
    path = os.path.normpath(path[:-4])
    relpath = get_subpath(path, self.config.repository_root)
    if not relpath:
      return None

    if os.name == 'nt':
      relpath = relpath.replace('\\', '/')
    return relpath

  def get_access_info(self, path):
    ''' Wrapper for the `AccessController.get_access_info()` function
    that uses this session and the current user name. Returns a bit
    field with the access privileges of the current user to *path*. '''

    controller = self.config.access_controller
    return controller.get_access_info(self, self.user.name, path)

  def repositories(self):
    ''' Iterate over all repositories. Yields tuples in the form of
    `(repo_name, path)`. '''

    if not os.path.isdir(self.config.repository_root):
      return
    for dirpath, dirs, files in os.walk(self.config.repository_root):
      for dirname in dirs:
        if dirname != '.git' and dirname.endswith('.git'):
          repo_path = os.path.join(dirpath, dirname)
          repo_name = self.path2repo(repo_path)
          yield (repo_name, repo_path)

# == Command Functions ========================================================
# =============================================================================

def command_repo(session, args):
  ''' Manage repositories. '''

  parser = argparse.ArgumentParser(prog='repo')
  subparser = parser.add_subparsers(dest='cmd')
  create_p = subparser.add_parser('create')
  create_p.add_argument('name')
  rename_p = subparser.add_parser('rename')
  rename_p.add_argument('old')
  rename_p.add_argument('new')
  delete_p = subparser.add_parser('delete')
  delete_p.add_argument('repo')
  delete_p.add_argument('-f', '--force', action='store_true')
  list_p = subparser.add_parser('list')
  args = parser.parse_args(args)

  if not args.cmd:
    parser.print_usage()
    return 0

  if args.cmd == 'create':
    path = session.repo2path(args.name)
    if not session.get_access_info(path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.name))
      return errno.EPERM

    # Make sure that none of the parent directories is a repository.
    if any(x.endswith('.git') for x in path.split(os.sep)[:-1]):
      print("error: can not create repository inside repository")
      return errno.EPERM

    if os.path.exists(path):
      print("error: repository {!r} already exists.".format(args.name))
      return errno.EEXIST

    res = subprocess.call(['git', 'init', '--bare', path])
    if res != 0:
      print("error: repository could not be created.")
    return res
  elif args.cmd == 'rename':
    old_path = session.repo2path(args.old)
    new_path = session.repo2path(args.new)
    if not session.get_access_info(old_path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.old))
      return errno.EPERM
    if not session.get_access_info(new_path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.new))
      return errno.EPERM

    # Make sure that none of the parent directories is a repository.
    if any(x.endswith('.git') for x in new_path.split(os.sep)[:-1]):
      print("error: can not create repository inside repository")
      return errno.EPERM

    if not os.path.exists(old_path):
      print("error: repository {!r} does not exist".format(args.old))
      return errno.ENOENT
    if os.path.exists(new_path):
      print("error: repository {!r} already exists.".format(args.new))
      return errno.EEXIST

    try:
      os.rename(old_path, new_path)
    except (OSError, IOError) as exc:
      print("error:", exc)
      return exc.errno
    return 0
  elif args.cmd == 'delete':
    path = session.repo2path(args.repo)
    if not session.get_access_info(path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.repo))
      return errno.EPERM
    if not os.path.exists(path):
      print("error: repository {!r} does not exist".format(args.repo))
      return errno.ENOENT
    if not os.path.isdir(path):
      print("fatal error: repository {!r} is not a directory".format(args.repo))
      return errno.ENOENT

    if not args.force:
      if not confirm('do you really want to delete this repository?'):
        return 0
    print("deleting repository {!r}...".format(args.repo), end=' ')
    try:
      shutil.rmtree(path)
    except (OSError, IOError) as exc:
      print("error.")
      print(exc)
      return exc.errno
    else:
      print("done.")
    return 0
  elif args.cmd == 'list':
    for repo_name, path in session.repositories():
      info = session.get_access_info(path)
      flags = list('---')
      if info & ACCESS_READ:
        flags[0] = 'r'
      if info & ACCESS_WRITE:
        flags[1] = 'w'
      if info & ACCESS_MANAGE:
        flags[2] = 'm'
      if info:
        print(''.join(flags), ': ', repo_name, sep='')
    return 0

  print("error: command not handled.")
  return 255

def command_help(session, args):
  ''' Show this help. '''

  print("Available commands:")
  print()
  for key, func in sorted(session.commands().items(), key=lambda x: x[0]):
    print(key)
    if func.__doc__:
      for line in textwrap.wrap(textwrap.dedent(func.__doc__)):
        print("  ", line, sep='')

# == Main =====================================================================
# =============================================================================

def get_argument_parser():
  parser = argparse.ArgumentParser(description='''
    git_auth v{0} - Copyright (C) 2015 Niklas Rosenstein''')
  parser.add_argument('user')
  parser.add_argument('command', nargs='...')
  return parser

def main():
  parser = get_argument_parser()
  args = parser.parse_args()
  session = GitAuthSession(args.user)

  if not args.command:
    ssh_command = os.environ.pop('SSH_ORIGINAL_COMMAND', None)
    if ssh_command:
      args.command = shlex.split(ssh_command)

  if args.command:
    if args.command[0] in ('git-receive-pack', 'git-upload-pack'):
      return subprocess.call(args.command)

  # Users without manage privileges can't go past this line.
  if not session.user.shell_access:
    print("You are not privileged for SSH Access.")
    return errno.EPERM

  if args.command:
    return session.command(args.command)

  session.cmdloop()
  return 0
<commit_msg>fixed error when renaming repository to a parent directory that didn't exist.<commit_after># -*- mode: python; tab-width: 2; coding: utf8 -*-
#
# Copyright (C) 2015 Niklas Rosenstein
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
# either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
'''
`git_auth` - a Git authentication layer alternative
===================================================
'''

import os
import sys
import argparse
import collections
import errno
import itertools
import shlex
import shutil
import subprocess
import textwrap
import traceback
import re

__author__ = 'Niklas Rosenstein <rosensteinniklas(at)gmail.com>'
__version__ = '0.9.0'

ACCESS_READ = 1 << 0
ACCESS_WRITE = 1 << 1
ACCESS_MANAGE = 1 << 2

def get_subpath(path, parent):
  relpath = os.path.relpath(path, parent)
  if relpath == os.curdir or relpath.startswith(os.pardir):
    return None
  return relpath

def is_subpath(path, parent):
  return bool(get_subpath(path, parent))

def confirm(question):
  ''' Asks the specified *question* and requests the user to reply with
  yes or no. Returns True if yes was replied, False if no. Will ask the
  question again if an invalid reply was given. '''

  while True:
    reply = input(question + ' [y/n] ').strip().lower()
    if reply in ('yes', 'y'):
      return True
    elif reply in ('no', 'n'):
      return False
    else:
      print("Please reply with yes/y or no/n.")

# == Git Auth Layer ===========================================================
# =============================================================================

class AccessController(object):
  ''' This interface describes the controller to manage access to Git
  repositories. '''

  User = collections.namedtuple('User', 'name home shell_access')

  class UnknownUser(Exception): pass

  def get_user_info(self, session, user_name):
    ''' Return user information for the specified *user_name* or raise
    `AccessController.UnknownUser` if the user does not exist. '''

    raise self.UnknownUser(user_name)

  def get_access_info(self, session, user_name, path):
    ''' Return a bit mask that indicates the access privileges of the
    user with the specified *user_name* to the *path*. The *path* would
    usually be a sub path of the `repository_root` in the configuration. '''

    raise NotImplementedError

class SimpleAccessController(AccessController):
  ''' This simple implementation of the `AccessController` interface
  allows all registered users to manage their home directory via SSH. '''

  def __init__(self, has_root=False):
    super().__init__()
    self.has_root = has_root

  def get_user_info(self, session, user_name):
    if not re.match('[A-z0-9\-_]', user_name):
      raise self.UnknownUser(user_name)
    if self.has_root and user_name == 'root':
      return self.User(user_name, '/', True)
    return self.User(user_name, '/' + user_name, True)

  def get_access_info(self, session, user_name, path):
    if self.has_root and user_name == 'root':
      if is_subpath(path, session.config.repository_root):
        return ACCESS_READ | ACCESS_WRITE | ACCESS_MANAGE

    home = os.path.join(session.config.repository_root, user_name)
    if is_subpath(path, home):
      return ACCESS_READ | ACCESS_WRITE | ACCESS_MANAGE
    return 0

class GitAuthSession(object):
  ''' This is the central authentication and repository management class. '''

  def __init__(self, user, config=None):
    super().__init__()
    if config is None:
      import git_auth_config as config
    self.user = config.access_controller.get_user_info(self, user)
    self.config = config

  def command(self, command):
    ''' Execute the specified command list. The first element in the
    list is used as the command name and dispatched by the default
    commands and the additional commands in the configuration. The
    exit code of the command is returned. '''

    fname = 'command_' + command[0]
    command_func = None
    if fname in globals():
      command_func = globals()[fname]
    if hasattr(self.config, fname):
      command_func = getattr(self.config, fname)
    if not callable(command_func):
      print("unknown command:", command[0], file=sys.stderr)
    else:
      try:
        return command_func(self, command[1:])
      except SystemExit as exc:
        return exc.code
    return 255

  def commands(self):
    ''' Returns a dictionary of all available commands mapping the name
    to the command function that implements it. '''

    commands = {}
    chain = vars(self.config).items(), globals().items()
    for key, value in itertools.chain(*chain):
      if callable(value) and key.startswith('command_'):
        name = key[8:]
        commands[name] = value

    return commands

  def cmdloop(self, intro='git-auth$ '):
    ''' Enters the interactive shell. '''

    header = "git-auth v{0} - Copyright (C) 2015 {1}"
    print(header.format(__version__, __author__))

    while True:
      command = shlex.split(input(intro))
      if command:
        if command[0] == 'exit':
          break
        elif command[0] == '?':
          command[0] = 'help'
        self.command(command)

  def repo2path(self, repo_name):
    ''' Converts a relative repository name to an absolute path on the
    filesystem based on the repository root directory as specified in
    the configuration. '''

    if os.name == 'nt':
      repo_name = repo_name.replace('/', '\\')

    if not repo_name.endswith('.git'):
      repo_name += '.git'
    path = os.path.join(self.config.repository_root, repo_name)
    path = os.path.normpath(path)
    return path

  def path2repo(self, path):
    ''' Converts the specified absolute path to a relative repository
    name. The original path can be obtained with `repo2path()`. The
    path is returned even if the repository does not exist. The `.git`
    suffix is removed from the path. None will be returned if the
    path does not end with `.git` or is not inside the repository
    root directory. '''

    if not path.endswith('.git'):
      return None
    path = os.path.normpath(path[:-4])
    relpath = get_subpath(path, self.config.repository_root)
    if not relpath:
      return None

    if os.name == 'nt':
      relpath = relpath.replace('\\', '/')
    return relpath

  def get_access_info(self, path):
    ''' Wrapper for the `AccessController.get_access_info()` function
    that uses this session and the current user name. Returns a bit
    field with the access privileges of the current user to *path*. '''

    controller = self.config.access_controller
    return controller.get_access_info(self, self.user.name, path)

  def repositories(self):
    ''' Iterate over all repositories. Yields tuples in the form of
    `(repo_name, path)`. '''

    if not os.path.isdir(self.config.repository_root):
      return
    for dirpath, dirs, files in os.walk(self.config.repository_root):
      for dirname in dirs:
        if dirname != '.git' and dirname.endswith('.git'):
          repo_path = os.path.join(dirpath, dirname)
          repo_name = self.path2repo(repo_path)
          yield (repo_name, repo_path)

# == Command Functions ========================================================
# =============================================================================

def command_repo(session, args):
  ''' Manage repositories. '''

  parser = argparse.ArgumentParser(prog='repo')
  subparser = parser.add_subparsers(dest='cmd')
  create_p = subparser.add_parser('create')
  create_p.add_argument('name')
  rename_p = subparser.add_parser('rename')
  rename_p.add_argument('old')
  rename_p.add_argument('new')
  delete_p = subparser.add_parser('delete')
  delete_p.add_argument('repo')
  delete_p.add_argument('-f', '--force', action='store_true')
  list_p = subparser.add_parser('list')
  args = parser.parse_args(args)

  if not args.cmd:
    parser.print_usage()
    return 0

  if args.cmd == 'create':
    path = session.repo2path(args.name)
    if not session.get_access_info(path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.name))
      return errno.EPERM

    # Make sure that none of the parent directories is a repository.
    if any(x.endswith('.git') for x in path.split(os.sep)[:-1]):
      print("error: can not create repository inside repository")
      return errno.EPERM

    if os.path.exists(path):
      print("error: repository {!r} already exists.".format(args.name))
      return errno.EEXIST

    res = subprocess.call(['git', 'init', '--bare', path])
    if res != 0:
      print("error: repository could not be created.")
    return res
  elif args.cmd == 'rename':
    old_path = session.repo2path(args.old)
    new_path = session.repo2path(args.new)
    if not session.get_access_info(old_path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.old))
      return errno.EPERM
    if not session.get_access_info(new_path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.new))
      return errno.EPERM

    # Make sure that none of the parent directories is a repository.
    if any(x.endswith('.git') for x in new_path.split(os.sep)[:-1]):
      print("error: can not create repository inside repository")
      return errno.EPERM

    if not os.path.exists(old_path):
      print("error: repository {!r} does not exist".format(args.old))
      return errno.ENOENT
    if os.path.exists(new_path):
      print("error: repository {!r} already exists.".format(args.new))
      return errno.EEXIST

    try:
      # Make sure the parent of the new target directory exists.
      parent = os.path.dirname(new_path)
      if not os.path.exists(parent):
        os.makedirs(parent)
      os.rename(old_path, new_path)
    except (OSError, IOError) as exc:
      print("error:", exc)
      return exc.errno
    return 0
  elif args.cmd == 'delete':
    path = session.repo2path(args.repo)
    if not session.get_access_info(path) & ACCESS_MANAGE:
      print("error: manage permission to {!r} denied".format(args.repo))
      return errno.EPERM
    if not os.path.exists(path):
      print("error: repository {!r} does not exist".format(args.repo))
      return errno.ENOENT
    if not os.path.isdir(path):
      print("fatal error: repository {!r} is not a directory".format(args.repo))
      return errno.ENOENT

    if not args.force:
      if not confirm('do you really want to delete this repository?'):
        return 0
    print("deleting repository {!r}...".format(args.repo), end=' ')
    try:
      shutil.rmtree(path)
    except (OSError, IOError) as exc:
      print("error.")
      print(exc)
      return exc.errno
    else:
      print("done.")
    return 0
  elif args.cmd == 'list':
    for repo_name, path in session.repositories():
      info = session.get_access_info(path)
      flags = list('---')
      if info & ACCESS_READ:
        flags[0] = 'r'
      if info & ACCESS_WRITE:
        flags[1] = 'w'
      if info & ACCESS_MANAGE:
        flags[2] = 'm'
      if info:
        print(''.join(flags), ': ', repo_name, sep='')
    return 0

  print("error: command not handled.")
  return 255

def command_help(session, args):
  ''' Show this help. '''

  print("Available commands:")
  print()
  for key, func in sorted(session.commands().items(), key=lambda x: x[0]):
    print(key)
    if func.__doc__:
      for line in textwrap.wrap(textwrap.dedent(func.__doc__)):
        print("  ", line, sep='')

# == Main =====================================================================
# =============================================================================

def get_argument_parser():
  parser = argparse.ArgumentParser(description='''
    git_auth v{0} - Copyright (C) 2015 Niklas Rosenstein''')
  parser.add_argument('user')
  parser.add_argument('command', nargs='...')
  return parser

def main():
  parser = get_argument_parser()
  args = parser.parse_args()
  session = GitAuthSession(args.user)

  if not args.command:
    ssh_command = os.environ.pop('SSH_ORIGINAL_COMMAND', None)
    if ssh_command:
      args.command = shlex.split(ssh_command)

  if args.command:
    if args.command[0] in ('git-receive-pack', 'git-upload-pack'):
      return subprocess.call(args.command)

  # Users without manage privileges can't go past this line.
  if not session.user.shell_access:
    print("You are not privileged for SSH Access.")
    return errno.EPERM

  if args.command:
    return session.command(args.command)

  session.cmdloop()
  return 0
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Usage:  program.py <customer>

"""

import requests
import json
import os.path

base_url = "http://localhost:9292"

files = [
  f"{os.path.dirname(__file__)}/bel2.0-example-statements.bel",
  f"{os.path.dirname(__file__)}/bel2_document_examples.bel"
]

def send_request(bel):
    # Issue #134
    # GET http://localhost:9292/api/expressions/rxn(reactants(a(CHEBI:superoxide)),products(a(CHEBI:%22hydrogen%20peroxide%22),%20a(CHEBI:%20%22oxygen%22))/validation

    try:
        response = requests.get(
            url=f"{base_url}/api/expressions/{bel}/validation",
        )
        try:
            r = response.json()
        except:
            r = None

        # print(f"Status {response.status_code}  Response: {r}")
        return (response.status_code, r)

    except requests.exceptions.RequestException:
        # return (response.status_code, response.json())
        print(f"Error {response.status_code}, {bel}")

def run_examples():
    results = []
    cnt = error_cnt = success_cnt = 0
    for fn in files:
        with open(fn, 'r') as f:
            for bel in f:
                cnt += 1
                bel = bel.strip()

                print(f"Running bel: {bel}")

                (status, msg) = send_request(bel)
                if status != 200:
                    error_cnt += 1
                    results.append((status, bel, msg))
                else:
                    success_cnt += 1

    print(f"Total: {cnt}  Success: {success_cnt}  Errors: {error_cnt}")
    with open('test_results.json', 'w') as f:
        json.dump(results, f, indent=4)

def main():
    run_examples()

if __name__ == '__main__':
    main()

<commit_msg>use requests's quote method to handle path quoting<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Usage:  program.py <customer>

"""

import requests
import json
import os.path

base_url = "http://localhost:9292"

files = [
  f"{os.path.dirname(__file__)}/bel2.0-example-statements.bel",
  f"{os.path.dirname(__file__)}/bel2_document_examples.bel"
]

def send_request(bel):
    # Issue #134
    # GET http://localhost:9292/api/expressions/rxn(reactants(a(CHEBI:superoxide)),products(a(CHEBI:%22hydrogen%20peroxide%22),%20a(CHEBI:%20%22oxygen%22))/validation

    try:
        response = requests.get(
            url=f"{base_url}/api/expressions/{requests.utils.quote(bel)}/validation",
        )
        try:
            r = response.json()
        except:
            r = None

        # print(f"Status {response.status_code}  Response: {r}")
        return (response.status_code, r)

    except requests.exceptions.RequestException:
        # return (response.status_code, response.json())
        print(f"Error {response.status_code}, {bel}")

def run_examples():
    results = []
    cnt = error_cnt = success_cnt = 0
    for fn in files:
        with open(fn, 'r') as f:
            for bel in f:
                cnt += 1
                bel = bel.strip()

                print(f"Running bel: {bel}")

                (status, msg) = send_request(bel)
                if status != 200:
                    error_cnt += 1
                    results.append((status, bel, msg))
                else:
                    success_cnt += 1

    print(f"Total: {cnt}  Success: {success_cnt}  Errors: {error_cnt}")
    with open('test_results.json', 'w') as f:
        json.dump(results, f, indent=4)

def main():
    run_examples()

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in or l \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<commit_msg>fix parse_test_results.py typo<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output in stead of stdout/stderr
def style_output(o):
	ret = ''
	subtle = False
	for l in o.split('\n'):
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or 'TEST_EQUAL_ERROR' in l or l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG'):
			ret += '%s\n' % l
		elif '**passed**' in l:
			ret += '%s\n' % l
		elif ': error: ' in l or ': fatal error: ' in l or ' : fatal error ' in l or \
			'failed to write output file' in l or ') : error C' in l or \
			' : error LNK' in l or ': undefined reference to ' in l or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l:
			ret += '%s\n' % l
		elif ': warning: ' in l or ') : warning C' in l or \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l:
			ret += '%s\n' % l
		elif l == '====== END OUTPUT ======' and not subtle:
			ret += '%s\n' % l
			subtle = True
		else:
			ret += '%s\n' % l
	if subtle: ret += ''
	return ret

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

%s
' % \
		(test_name.encode('utf-8'), style_output(data).encode('utf-8'))

	print >>html, ''
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except:
			print '\nFAILED TO LOAD "%s"\n' %f
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		for f in tests:
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*9 - 5, f)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				print >>html, '%s	' % toolset
				for f in platforms[p][toolset]:
					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						if details['status'] == 0: c = 'passed'
						else: c = 'failed'
						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])

				print >>html, '
'
				idx += 1
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<|endoftext|>


<commit_before>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output instead of stdout/stderr
def style_output(logfile, outfile):
	subtle = False
	for l in logfile.split('\n'):
		l = l.encode('utf-8')
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or \
			'TEST_EQUAL_ERROR' in l or \
			'"ERROR: "' in l or \
			l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG') or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l or \
			'points to uninitialised byte(s)' in l:
			print >>outfile, '%s' % l
		elif '**passed**' in l:
			print >>outfile, '%s' % l
		elif ': error: ' in l or \
			';1;31merror: ' in l or \
			': fatal error: ' in l or \
			' : fatal error ' in l or \
			'failed to write output file' in l or \
			') : error C' in l or \
			' : error LNK' in l or \
			': undefined reference to ' in l:
			print >>outfile, '%s' % l
		elif ': warning: ' in l or \
			') : warning C' in l or \
			'0;1;35mwarning: ' in l or \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l or \
			'bytes inside a block of size' in l:
			print >>outfile, '%s' % l.strip()
		elif l == '====== END OUTPUT ======' and not subtle:
			print >>outfile, '%s' % l
			subtle = True
		else:
			print >>outfile, '%s' % l
	if subtle: print >>outfile, ''

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

' % test_name.encode('utf-8')
	style_output(data, html)

	print >>html, '
'
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except Exception, e:
			print '\nFAILED TO LOAD "%s": %s\n' % (f, e)
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		features = tests.keys()
		features = sorted(features, key=lambda x: len(tests[x]))

		for f in features:
			title = f
			if len(tests[f]) < 10: title = '#'
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*6 - 5, title)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				log_dir = 'logs-%s-%d' % (branch_name, r)
				if not os.path.exists(log_dir):
					os.mkdir(log_dir)
				details_name = os.path.join(log_dir, '%s-%s.html' % (p, toolset))
				details_file = open(details_name, 'w+')

				print >>details_file, '''<title>%s %s [%s]</title>
					''' % (p, toolset, branch_name)
				print >>html, '%s	' % (details_name, toolset)

				deferred_end_table = False
				for f in features:
					title = f
					if len(tests[f]) < 10: title = '#'

					if title != '#':
						if deferred_end_table:
							print >>details_file, '

'
							print >>details_file, '
%s
' % title
							deferred_end_table = False
						else:
							print >>details_file, '
'
							print >>details_file, '
%s
' % title
					elif not deferred_end_table:
						print >>details_file, '
'
						print >>details_file, '
%s
' % title

					if not f in platforms[p][toolset]:
						print f
						for i in range(len(tests[f])):
							print >>html, '	' % (f)
						continue

					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						exitcode = details['status']

						if exitcode == 0:
							error_state = 'passed'
							c = 'passed'
						elif exitcode == 222:
							error_state = 'valgrind error'
							c = 'valgrind-error'
						elif exitcode == 139 or \
							exitcode == 138:
							error_state = 'crash'
							c = 'crash'
						elif exitcode == -1073740777:
							error_state = 'timeout'
							c = 'timeout'
						elif exitcode == 333 or \
							exitcode == 77:
							error_code = 'test-failed'
							c = 'failed'
						else:
							error_state = 'compile-failed (%d)' % exitcode
							c = 'compile-failed'

						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						print >>details_file, '
%s [%s]
' % (c, os.path.split(log_name)[1], t, error_state)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])
					if title != '#':
						print >>details_file, '
'
						deferred_end_table = False
					else:
						deferred_end_table = True

				if deferred_end_table:
					print >>details_file, '
'

				print >>html, '
'
				idx += 1
				print >>details_file, ''
				details_file.close()
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<commit_msg>remove unnecessary print<commit_after>#!/bin/python

# Copyright (c) 2013, Arvid Norberg
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above copyright
#       notice, this list of conditions and the following disclaimer in
#       the documentation and/or other materials provided with the distribution.
#     * Neither the name of the author nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

# This is meant to be run from the root directory of the repo. It will
# look for the .regression.yml file and expect a regression_tests directory
# with results from test runs previously produced by run_tests.py

import os
import sys
import glob
import json

# TODO: different parsers could be run on output from different actions
# if we would use the xml output instead of stdout/stderr
def style_output(logfile, outfile):
	subtle = False
	for l in logfile.split('\n'):
		l = l.encode('utf-8')
		l = l.replace('<', '<')
		l = l.replace('>', '>')
		if 'TEST_CHECK' in l or \
			'TEST_EQUAL_ERROR' in l or \
			'"ERROR: "' in l or \
			l.startswith('EXIT STATUS: ') or \
			' second time limit exceeded' in l or l.startswith('signal: SIG') or \
			'jump or move depends on uninitialised value(s)' in l or \
			'Invalid read of size' in l or \
			'Invalid write of size' in l or \
			'Use of uninitialised value of size' in l or \
			'Uninitialised byte(s) found during' in l or \
			'points to uninitialised byte(s)' in l:
			print >>outfile, '%s' % l
		elif '**passed**' in l:
			print >>outfile, '%s' % l
		elif ': error: ' in l or \
			';1;31merror: ' in l or \
			': fatal error: ' in l or \
			' : fatal error ' in l or \
			'failed to write output file' in l or \
			') : error C' in l or \
			' : error LNK' in l or \
			': undefined reference to ' in l:
			print >>outfile, '%s' % l
		elif ': warning: ' in l or \
			') : warning C' in l or \
			'0;1;35mwarning: ' in l or \
			'Uninitialised value was created by a' in l or \
			'bytes after a block of size' in l or \
			'bytes inside a block of size' in l:
			print >>outfile, '%s' % l.strip()
		elif l == '====== END OUTPUT ======' and not subtle:
			print >>outfile, '%s' % l
			subtle = True
		else:
			print >>outfile, '%s' % l
	if subtle: print >>outfile, ''

def modification_time(file):
	mtime = 0
	try:
		mtime = os.stat(file).st_mtime
	except: pass
	return mtime

def save_log_file(log_name, project_name, branch_name, test_name, timestamp, data):

	if not os.path.exists(os.path.split(log_name)[0]):
		os.mkdir(os.path.split(log_name)[0])

	try:
		# if the log file already exists, and it's newer than
		# the source, no need to re-parse it
		mtime = os.stat(log_name).st_mtime
		if mtime >= timestamp: return
	except: pass

	html = open(log_name, 'w+')
	print >>html, '''<title>%s - %s</title>

%s - %s
''' % (project_name, branch_name, project_name, branch_name)
	print >>html, '
%s

' % test_name.encode('utf-8')
	style_output(data, html)

	print >>html, '
'
	html.close()
	sys.stdout.write('.')
	sys.stdout.flush()

def parse_tests(rev_dir):

	# this contains mappings from platforms to
	# the next layer of dictionaries. The next
	# layer contains a mapping of toolsets to
	# dictionaries the next layer of dictionaries.
	# those dictionaries contain a mapping from
	# feature-sets to the next layer of dictionaries.
	# the next layer contains a mapping from
	# tests to information about those tests, such
	# as whether it passed and the output from the
	# command
	# example:

	# {
	#   darwin: {
	#     clang-4.2.1: {
	#       ipv6=off: {
	#         test_primitives: {
	#           output: ...
	#           status: 1
	#         }
	#       }
	#     }
	#   }
	# }

	platforms = {}

	tests = {}

	for f in glob.glob(os.path.join(rev_dir, '*.json')):
		platform_toolset = os.path.split(f)[1].split('.json')[0].split('#')
		try:
			j = json.loads(open(f, 'rb').read())
			timestamp = os.stat(f).st_mtime
		except Exception, e:
			print '\nFAILED TO LOAD "%s": %s\n' % (f, e)
			continue

		platform = platform_toolset[0]
		toolset = platform_toolset[1]

		for cfg in j:
			test_name = cfg.split('|')[0]
			features = cfg.split('|')[1]

			if not features in tests:
				tests[features] = set()

			tests[features].add(test_name)

			if not platform in platforms:
				platforms[platform] = {}

			if not toolset in platforms[platform]:
				platforms[platform][toolset] = {}

			if not features in platforms[platform][toolset]:
				platforms[platform][toolset][features] = {}

			platforms[platform][toolset][features][test_name] = j[cfg]
			platforms[platform][toolset][features][test_name]['timestamp'] = timestamp

	return (platforms, tests)

# TODO: remove this dependency by encoding it in the output files
# this script should work from outside of the repo, just having
# access to the shared folder
project_name = 'libtorrent'

# maps branch name to latest rev
revs = {}

os.chdir('regression_tests')

for rev in os.listdir('.'):
	try:
		branch = rev.split('-')[0]
		if branch == 'logs': continue
		r = int(rev.split('-')[1])
		if not branch in revs:
			revs[branch] = r
		else:
			if r > revs[branch]:
				revs[branch] = r
	except:
		print 'ignoring %s' % rev

if revs == {}:
	print 'no test files found'
	sys.exit(1)

print 'latest versions'
for b in revs:
	print '%s\t%d' % (b, revs[b])

for branch_name in revs:

	latest_rev = revs[branch_name]

	html_file = '%s.html' % branch_name

	html = open(html_file, 'w+')

	print >>html, '''<title>regression tests, %s</title>

%s - %s
''' % (project_name, project_name, branch_name)

	print >>html, '
'

	num_printed_revs = 0;
	for r in range(latest_rev, latest_rev - 40, -1):
		sys.stdout.write('.')
		sys.stdout.flush()

		rev_dir = '%s-%d' % (branch_name, r)
		(platforms, tests) = parse_tests(rev_dir)

		if len(tests) + len(platforms) == 0: continue

		print >>html, '
revision %d	' % r

		features = tests.keys()
		features = sorted(features, key=lambda x: len(tests[x]))

		for f in features:
			title = f
			if len(tests[f]) < 10: title = '#'
			print >>html, '%s	' % (len(tests[f]), len(tests[f])*6 - 5, title)
		print >>html, '
'

		for p in platforms:
			print >>html, '
%s	' % (len(platforms[p]), p)
			idx = 0
			for toolset in platforms[p]:
				if idx > 0: print >>html, '
'
				log_dir = 'logs-%s-%d' % (branch_name, r)
				if not os.path.exists(log_dir):
					os.mkdir(log_dir)
				details_name = os.path.join(log_dir, '%s-%s.html' % (p, toolset))
				details_file = open(details_name, 'w+')

				print >>details_file, '''<title>%s %s [%s]</title>
					''' % (p, toolset, branch_name)
				print >>html, '%s	' % (details_name, toolset)

				deferred_end_table = False
				for f in features:
					title = f
					if len(tests[f]) < 10: title = '#'

					if title != '#':
						if deferred_end_table:
							print >>details_file, '

'
							print >>details_file, '
%s
' % title
							deferred_end_table = False
						else:
							print >>details_file, '
'
							print >>details_file, '
%s
' % title
					elif not deferred_end_table:
						print >>details_file, '
'
						print >>details_file, '
%s
' % title

					if not f in platforms[p][toolset]:
						for i in range(len(tests[f])):
							print >>html, '	' % (f)
						continue

					for t in platforms[p][toolset][f]:
						details = platforms[p][toolset][f][t]
						exitcode = details['status']

						if exitcode == 0:
							error_state = 'passed'
							c = 'passed'
						elif exitcode == 222:
							error_state = 'valgrind error'
							c = 'valgrind-error'
						elif exitcode == 139 or \
							exitcode == 138:
							error_state = 'crash'
							c = 'crash'
						elif exitcode == -1073740777:
							error_state = 'timeout'
							c = 'timeout'
						elif exitcode == 333 or \
							exitcode == 77:
							error_code = 'test-failed'
							c = 'failed'
						else:
							error_state = 'compile-failed (%d)' % exitcode
							c = 'compile-failed'

						log_name = os.path.join('logs-%s-%d' % (branch_name, r), p + '~' + toolset + '~' + t + '~' + f.replace(' ', '.') + '.html')
						print >>html, '	' % (t, f, c, log_name)
						print >>details_file, '
%s [%s]
' % (c, os.path.split(log_name)[1], t, error_state)
						save_log_file(log_name, project_name, branch_name, '%s - %s' % (t, f), int(details['timestamp']), details['output'])
					if title != '#':
						print >>details_file, '
'
						deferred_end_table = False
					else:
						deferred_end_table = True

				if deferred_end_table:
					print >>details_file, '
'

				print >>html, '
'
				idx += 1
				print >>details_file, ''
				details_file.close()
		num_printed_revs += 1
		if num_printed_revs >= 20: break

	print >>html, '
'
	html.close()

	print ''

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from operator import attrgetter
from openerp.osv import orm
from .adapter_util import BrowseAdapterSourceMixin
from .logistic_requisition_source import AGR_PROC

class logistic_requisition_line(orm.Model, BrowseAdapterSourceMixin):
    """Override to enabale to generation of source line"""

    _inherit = "logistic.requisition.line"

    def _map_agr_requisiton_to_source(self, cr, uid, line, context=None,
                                      qty=0, agreement=None, **kwargs):
        """Prepare data dict for source line using agreement as source
        :params line: browse record of origin requistion.line
        :params agreement: browse record of origin agreement
        :params qty: quantity to be set on source line
        :returns: dict to be used by Model.create"""
        res = {}
        direct_map = {
            'proposed_product_id': 'product_id',
            'requisition_line_id': 'id',
            'proposed_uom_id': 'requested_uom_id'}

        if not agreement:
            raise ValueError("Missing agreement")
        if not agreement.product_id.id == line.product_id.id:
            raise ValueError("Product mismatch for agreement and requisition line")
        res['unit_cost'] = agreement.get_price(qty)
        res['proposed_qty'] = qty
        res['agreement_id'] = agreement.id
        res['procurement_method'] = AGR_PROC
        res.update(self._direct_map(line, direct_map))
        return res

    def _map_requisition_to_source(self, cr, uid, line, context=None,
                                   qty=0, **kwargs):
        """Prepare data dict to generate source line using requisition as source
        :params line: browse record of origin requistion.line
        :params qty: quantity to be set on source line
        :returns: dict to be used by Model.create"""
        res = {}
        direct_map = {'proposed_product_id': 'product_id',
                      'requisition_line_id': 'id',
                      'proposed_uom_id': 'requested_uom_id'}
        res['unit_cost'] = 0.0
        res['proposed_qty'] = qty
        res['agreement_id'] = False
        res['procurement_method'] = 'procurement'
        res.update(self._direct_map(line, direct_map))
        return res

    def _generate_lines_from_agreements(self, cr, uid, container, line,
                                        agreements, qty, context=None):
        """Generate source lines for one requisition line using
        available agreements. We first look for cheapeast agreement.
        Then if no more quantity are available and there is still remaining needs
        we look for next cheapest agreement or return remianing qty
        :param container: iterator of agreements browse
        :param qty: quantity to be sourced
        :param line: origin requisition line
        :returns: remaining quantity to source
        """
        if not agreements:
            return qty
        agreements.sort(key=lambda x: x.get_price(qty))
        current_agr = agreements.pop(0)
        avail = current_agr.available_quantity
        if not avail:
            return qty
        avail_sold = avail - qty
        to_consume = qty if avail_sold >= 0 else avail

        source_id = self.make_source_line(cr, uid, line, force_qty=to_consume,
                                          agreement=current_agr, context=context)
        container.append(source_id)
        difference = qty - to_consume
        if difference:
            return self._generate_lines_from_agreements(cr, uid, container, line,
                                                        agreements, difference, context=context)
        else:
            return 0

    def _source_lines_for_agreements(self, cr, uid, line, agreements):
        """Generate source lines for one requisition line using
        available agreements. We first look for cheapeast agreement.
        Then if no more quantity are available and there is still remaining needs
        we look for next cheapest agreement or we create
        a tender source line
        """
        qty = line.requested_qty
        generated = []
        remaining_qty = self._generate_lines_from_agreements(cr, uid, generated,
                                                             line, agr_iter, qty)
        return (generated, remaining_qty)

    def make_source_line(self, cr, uid, line, force_qty=None, agreement=None, context=None):
        """Generate a source line for a tender from a requisition line
        :param line: browse record of origin logistic.request
        :param force_qty: if set this quantity will be used instead
        of requested quantity
        :returns: id of generated source line"""
        qty = force_qty if force_qty else line.requested_qty
        src_obj = self.pool['logistic.requisition.source']
        if agreement:
            return src_obj._make_source_line_from_origin(cr, uid, line,
                                                         self._map_agr_requisiton_to_source,
                                                         context=context, qty=qty,
                                                         agreement=agreement)
        else:
            return src_obj._make_source_line_from_origin(cr, uid, line,
                                                         self._map_requisition_to_source,
                                                         context=context, qty=qty)

    def _generate_source_line(self, cr, uid, line, context=None):
        """Generate one or n source line(s) per requisition line
        depending on the available resources. If there is framework agreement(s)
        running we generate one or n source line using agreements otherwise we generate one
        source line using tender process
        :param line: browse record of origin logistic.request
        :returns: list of generated source line ids"""
        if line.source_ids:
            return None
        agr_obj = self.pool['framework.agreement']
        date = line.requisition_id.date
        product_id = line.product_id.id
        agreements = agr_obj.get_all_product_agreements(cr, uid, product_id, date,
                                                        context=context)
        generated_lines = []
        if agreements:
            line_ids, missing_qty = self._source_lines_for_agreements(cr, uid, line, agreements)
            generated_lines.extend(line_ids)
            if missing_qty:
                generated_lines.append(self.make_source_line(cr, uid, line,
                                                             force_qty=missing_qty))
        else:
            generated_lines.append(self.make_source_line(cr, uid, line))

        return generated_lines

    def _do_confirm(self, cr, uid, ids, context=None):
        """Override to generate source lines from requision line.
        Please refer to _generate_source_line documentation"""
        res = super(logistic_requisition_line, self)._do_confirm(cr, uid, ids,
                                                                 context=context)
        for line_br in self.browse(cr, uid, ids, context=context):
            self._generate_source_line(cr, uid, line_br, context=context)
        return res
<commit_msg>[FIX] variable name<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Nicolas Bessi
#    Copyright 2013 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from operator import attrgetter
from openerp.osv import orm
from .adapter_util import BrowseAdapterSourceMixin
from .logistic_requisition_source import AGR_PROC

class logistic_requisition_line(orm.Model, BrowseAdapterSourceMixin):
    """Override to enabale to generation of source line"""

    _inherit = "logistic.requisition.line"

    def _map_agr_requisiton_to_source(self, cr, uid, line, context=None,
                                      qty=0, agreement=None, **kwargs):
        """Prepare data dict for source line using agreement as source
        :params line: browse record of origin requistion.line
        :params agreement: browse record of origin agreement
        :params qty: quantity to be set on source line
        :returns: dict to be used by Model.create"""
        res = {}
        direct_map = {
            'proposed_product_id': 'product_id',
            'requisition_line_id': 'id',
            'proposed_uom_id': 'requested_uom_id'}

        if not agreement:
            raise ValueError("Missing agreement")
        if not agreement.product_id.id == line.product_id.id:
            raise ValueError("Product mismatch for agreement and requisition line")
        res['unit_cost'] = agreement.get_price(qty)
        res['proposed_qty'] = qty
        res['agreement_id'] = agreement.id
        res['procurement_method'] = AGR_PROC
        res.update(self._direct_map(line, direct_map))
        return res

    def _map_requisition_to_source(self, cr, uid, line, context=None,
                                   qty=0, **kwargs):
        """Prepare data dict to generate source line using requisition as source
        :params line: browse record of origin requistion.line
        :params qty: quantity to be set on source line
        :returns: dict to be used by Model.create"""
        res = {}
        direct_map = {'proposed_product_id': 'product_id',
                      'requisition_line_id': 'id',
                      'proposed_uom_id': 'requested_uom_id'}
        res['unit_cost'] = 0.0
        res['proposed_qty'] = qty
        res['agreement_id'] = False
        res['procurement_method'] = 'procurement'
        res.update(self._direct_map(line, direct_map))
        return res

    def _generate_lines_from_agreements(self, cr, uid, container, line,
                                        agreements, qty, context=None):
        """Generate source lines for one requisition line using
        available agreements. We first look for cheapeast agreement.
        Then if no more quantity are available and there is still remaining needs
        we look for next cheapest agreement or return remianing qty
        :param container: iterator of agreements browse
        :param qty: quantity to be sourced
        :param line: origin requisition line
        :returns: remaining quantity to source
        """
        if not agreements:
            return qty
        agreements.sort(key=lambda x: x.get_price(qty))
        current_agr = agreements.pop(0)
        avail = current_agr.available_quantity
        if not avail:
            return qty
        avail_sold = avail - qty
        to_consume = qty if avail_sold >= 0 else avail

        source_id = self.make_source_line(cr, uid, line, force_qty=to_consume,
                                          agreement=current_agr, context=context)
        container.append(source_id)
        difference = qty - to_consume
        if difference:
            return self._generate_lines_from_agreements(cr, uid, container, line,
                                                        agreements, difference, context=context)
        else:
            return 0

    def _source_lines_for_agreements(self, cr, uid, line, agreements):
        """Generate source lines for one requisition line using
        available agreements. We first look for cheapeast agreement.
        Then if no more quantity are available and there is still remaining needs
        we look for next cheapest agreement or we create
        a tender source line
        """
        qty = line.requested_qty
        generated = []
        remaining_qty = self._generate_lines_from_agreements(cr, uid, generated,
                                                             line, agreements, qty)
        return (generated, remaining_qty)

    def make_source_line(self, cr, uid, line, force_qty=None, agreement=None, context=None):
        """Generate a source line for a tender from a requisition line
        :param line: browse record of origin logistic.request
        :param force_qty: if set this quantity will be used instead
        of requested quantity
        :returns: id of generated source line"""
        qty = force_qty if force_qty else line.requested_qty
        src_obj = self.pool['logistic.requisition.source']
        if agreement:
            return src_obj._make_source_line_from_origin(cr, uid, line,
                                                         self._map_agr_requisiton_to_source,
                                                         context=context, qty=qty,
                                                         agreement=agreement)
        else:
            return src_obj._make_source_line_from_origin(cr, uid, line,
                                                         self._map_requisition_to_source,
                                                         context=context, qty=qty)

    def _generate_source_line(self, cr, uid, line, context=None):
        """Generate one or n source line(s) per requisition line
        depending on the available resources. If there is framework agreement(s)
        running we generate one or n source line using agreements otherwise we generate one
        source line using tender process
        :param line: browse record of origin logistic.request
        :returns: list of generated source line ids"""
        if line.source_ids:
            return None
        agr_obj = self.pool['framework.agreement']
        date = line.requisition_id.date
        product_id = line.product_id.id
        agreements = agr_obj.get_all_product_agreements(cr, uid, product_id, date,
                                                        context=context)
        generated_lines = []
        if agreements:
            line_ids, missing_qty = self._source_lines_for_agreements(cr, uid, line, agreements)
            generated_lines.extend(line_ids)
            if missing_qty:
                generated_lines.append(self.make_source_line(cr, uid, line,
                                                             force_qty=missing_qty))
        else:
            generated_lines.append(self.make_source_line(cr, uid, line))

        return generated_lines

    def _do_confirm(self, cr, uid, ids, context=None):
        """Override to generate source lines from requision line.
        Please refer to _generate_source_line documentation"""
        res = super(logistic_requisition_line, self)._do_confirm(cr, uid, ids,
                                                                 context=context)
        for line_br in self.browse(cr, uid, ids, context=context):
            self._generate_source_line(cr, uid, line_br, context=context)
        return res
<|endoftext|>


<commit_before>load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

def repositories():
    # Node (https://nodejs.org/en/about/releases/)
    # Follow Node's maintainence schedule and support all LTS versions that are not end of life
    http_archive(
        name = "nodejs14_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "cc9d17834eb383565a3368f4222b825f341190813537c677973fc913dcc7bdd1",
        strip_prefix = "node-v14.19.3-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v14.19.3/node-v14.19.3-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs16_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "f78a49c0c9c2f546c3a44eb434c49a852125441422a1bcfc433dedc58d6a241c",
        strip_prefix = "node-v16.15.1-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v16.15.1/node-v16.15.1-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs18_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "4e3f5c72ec735aa23d52042be61e32e7279d26d7f05ebb5571c410e81d10c9a3",
        strip_prefix = "node-v18.4.0-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v18.4.0/node-v18.4.0-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs14_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "a1c837c7ec8a5ab0c4d5028695b05749adf216851fe0b84ef09a9c6eab86ba5d",
        strip_prefix = "node-v14.19.3-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v14.19.3/node-v14.19.3-linux-arm64.tar.gz"],
    )

    http_archive(
        name = "nodejs16_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "84db3f261a02c3d92558fb80a3b597b58175d713b8aa928f6b66e963340f1faf",
        strip_prefix = "node-v16.15.1-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v16.15.1/node-v16.15.1-linux-arm64.tar.gz"],
    )

    http_archive(
        name = "nodejs18_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "6926ab0721aee65b6a00f4858a307d736de5f48a511d7e5c6182f87887f76188",
        strip_prefix = "node-v18.4.0-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v18.4.0/node-v18.4.0-linux-arm64.tar.gz"],
    )
<commit_msg>Bump `nodejs14_amd64` to `14.20.0`<commit_after>load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

def repositories():
    # Node (https://nodejs.org/en/about/releases/)
    # Follow Node's maintainence schedule and support all LTS versions that are not end of life
    http_archive(
        name = "nodejs14_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "921680e244b813d6ffe06a4c22f87f05f39be635973c79486b2ded12a946cb37",
        strip_prefix = "node-v14.20.0-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs16_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "f78a49c0c9c2f546c3a44eb434c49a852125441422a1bcfc433dedc58d6a241c",
        strip_prefix = "node-v16.15.1-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v16.15.1/node-v16.15.1-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs18_amd64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "4e3f5c72ec735aa23d52042be61e32e7279d26d7f05ebb5571c410e81d10c9a3",
        strip_prefix = "node-v18.4.0-linux-x64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v18.4.0/node-v18.4.0-linux-x64.tar.gz"],
    )

    http_archive(
        name = "nodejs14_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "a1c837c7ec8a5ab0c4d5028695b05749adf216851fe0b84ef09a9c6eab86ba5d",
        strip_prefix = "node-v14.19.3-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v14.19.3/node-v14.19.3-linux-arm64.tar.gz"],
    )

    http_archive(
        name = "nodejs16_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "84db3f261a02c3d92558fb80a3b597b58175d713b8aa928f6b66e963340f1faf",
        strip_prefix = "node-v16.15.1-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v16.15.1/node-v16.15.1-linux-arm64.tar.gz"],
    )

    http_archive(
        name = "nodejs18_arm64",
        build_file = "//nodejs:BUILD.nodejs",
        sha256 = "6926ab0721aee65b6a00f4858a307d736de5f48a511d7e5c6182f87887f76188",
        strip_prefix = "node-v18.4.0-linux-arm64/",
        type = "tar.gz",
        urls = ["https://nodejs.org/dist/v18.4.0/node-v18.4.0-linux-arm64.tar.gz"],
    )
<|endoftext|>


<commit_before>
import mx.DateTime
from pyIEM import iemdb
i = iemdb.iemdb()
postgis = i['postgis']

tests = [
 'LSX.TO.11', 'CAE.TO.2', 'GSP.TO.5', 'PAH.SV.10', 'JKL.TO.4', 'LWX.TO.3',
 'RNK.TO.2', 'IND.TO.7', 'IND.TO.8', 'LMK.TO.12', 'LMK.TO.13', 'LMK.TO.14',
 'PAH.TO.17', 'PAH.TO.18', 'PAH.TO.19', 'PAH.TO.20', 'PAH.TO.21',
 'MEG.TO.20', 'ICT.TO.6', 'OAX.TO.8', 'DMX.TO.16', 'FSD.TO.2', 'SGF.TO.10',
 'LSX.TO.16', 'ILX.TO.2', 'RLX.TO.2', 'RLX.TO.3', 'MPX.TO.5', 'MPX.TO.6',
 'MPX.TO.7', 'FSD.TO.16', 'FSD.TO.17', 'FSD.TO.18'
]

sql = "SELECT wfo, min(eventid), max(eventid), phenomena from warnings_%s \
       WHERE phenomena IN ('MA','FF','SV','TO') and significance = 'W' \
       GROUP by wfo, phenomena" % (mx.DateTime.now().year, )
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
  sEvent = int(rs[i]['min'])
  eEvent = int(rs[i]['max'])
  phenomena = rs[i]['phenomena']
  wfo = rs[i]['wfo']

  for eventid in range(sEvent, eEvent):
    sql = "SELECT gtype, count(*) as c from warnings_%s WHERE wfo = '%s' \
           and phenomena = '%s' and eventid = '%s' and significance = 'W' \
           GROUP by gtype" % (mx.DateTime.now().year, wfo,\
           phenomena, eventid)
    rs2 = postgis.query(sql).dictresult()
    polyCount = 0
    cntyCount = 0
    for q in range(len(rs2)):
      if (rs2[q]['gtype'] == "P"): polyCount = int(rs2[q]['c'])
      if (rs2[q]['gtype'] == "C"): cntyCount = int(rs2[q]['c'])

    if (cntyCount == 0 and polyCount == 0):
      lookup = "%s.%s.%s" % (wfo, phenomena, eventid)
      if lookup not in tests:
        print "Warning Missing WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount == 0):
      print "SBW Missing     WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (cntyCount == 0):
      print "County Missing  WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount > 1):
      print "Duplicate SBW   WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
      sql = "DELETE from warnings_%s WHERE oid IN (\
         SELECT max(oid) as m from warnings_%s WHERE wfo = '%s' \
         and phenomena = '%s' and eventid = '%s' and significance = 'W' \
         and gtype = 'P')" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
      sql = "DELETE from warnings_%s WHERE oid IN (\
              select m from (\
select ugc,  max(oid) as m, count(oid) as c from warnings_%s WHERE wfo = '%s' \
and phenomena = '%s' and eventid = '%s' and significance = 'W' \
and gtype = 'C' GROUP by ugc) as foo WHERE c > 1)" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
<commit_msg>many more tests<commit_after>
import mx.DateTime
from pyIEM import iemdb
i = iemdb.iemdb()
postgis = i['postgis']

tests = [
 'LSX.TO.11', 'CAE.TO.2', 'GSP.TO.5', 'PAH.SV.10', 'JKL.TO.4', 'LWX.TO.3',
 'RNK.TO.2', 'IND.TO.7', 'IND.TO.8', 'LMK.TO.12', 'LMK.TO.13', 'LMK.TO.14',
 'PAH.TO.17', 'PAH.TO.18', 'PAH.TO.19', 'PAH.TO.20', 'PAH.TO.21',
 'MEG.TO.20', 'ICT.TO.6', 'OAX.TO.8', 'DMX.TO.16', 'FSD.TO.2', 'SGF.TO.10',
 'LSX.TO.16', 'ILX.TO.2', 'RLX.TO.2', 'RLX.TO.3', 'MPX.TO.5', 'MPX.TO.6',
 'MPX.TO.7', 'FSD.TO.16', 'FSD.TO.17', 'FSD.TO.18', 'ARX.TO.9', 'ARX.TO.10',
 'ARX.TO.11', 'GRB.TO.17', 'GRB.TO.18', 'GRB.TO.19', 'DLH.TO.2', 'DLH.TO.3',
 'MKX.TO.2', 'MKX.TO.3'
]

sql = "SELECT wfo, min(eventid), max(eventid), phenomena from warnings_%s \
       WHERE phenomena IN ('MA','FF','SV','TO') and significance = 'W' \
       GROUP by wfo, phenomena" % (mx.DateTime.now().year, )
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
  sEvent = int(rs[i]['min'])
  eEvent = int(rs[i]['max'])
  phenomena = rs[i]['phenomena']
  wfo = rs[i]['wfo']

  for eventid in range(sEvent, eEvent):
    sql = "SELECT gtype, count(*) as c from warnings_%s WHERE wfo = '%s' \
           and phenomena = '%s' and eventid = '%s' and significance = 'W' \
           GROUP by gtype" % (mx.DateTime.now().year, wfo,\
           phenomena, eventid)
    rs2 = postgis.query(sql).dictresult()
    polyCount = 0
    cntyCount = 0
    for q in range(len(rs2)):
      if (rs2[q]['gtype'] == "P"): polyCount = int(rs2[q]['c'])
      if (rs2[q]['gtype'] == "C"): cntyCount = int(rs2[q]['c'])

    if (cntyCount == 0 and polyCount == 0):
      lookup = "%s.%s.%s" % (wfo, phenomena, eventid)
      if lookup not in tests:
        print "Warning Missing WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount == 0):
      print "SBW Missing     WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (cntyCount == 0):
      print "County Missing  WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount > 1):
      print "Duplicate SBW   WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
      sql = "DELETE from warnings_%s WHERE oid IN (\
         SELECT max(oid) as m from warnings_%s WHERE wfo = '%s' \
         and phenomena = '%s' and eventid = '%s' and significance = 'W' \
         and gtype = 'P')" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
      sql = "DELETE from warnings_%s WHERE oid IN (\
              select m from (\
select ugc,  max(oid) as m, count(oid) as c from warnings_%s WHERE wfo = '%s' \
and phenomena = '%s' and eventid = '%s' and significance = 'W' \
and gtype = 'C' GROUP by ugc) as foo WHERE c > 1)" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
<|endoftext|>


<commit_before>from PIL import Image
from numpy import ones, zeros, linspace, argmin, nonzero, exp, array
# from numpy import column_stack
from numpy.random import rand

from .ModelFitter import ModelFitter
from src import Face

class GibbsSamplerFitter(ModelFitter):
    def __init__(self, image, dimensions=199, model=None, steps=None,
                 max_loops=1, determined_loops=0,
                 initial_face=None, callback=None):
        dimensions += 4
        self.__steps = ones(dimensions, dtype='i') if steps is None else steps
        self.__current_step = None
        self.__parameters = zeros(dimensions, dtype='f')

        self.__errors = None
        self.__values = None
        self.__loop = 0
        self.__max_loops = max_loops
        self.__determined_loops = determined_loops

        super(GibbsSamplerFitter, self).__init__(image, dimensions - 4, model,
                                                 initial_face, callback)
        self._dimensions += 4

    def start(self):
        self.__loop = 0
        self.request_normals(zeros(self._dimensions, dtype='f'), 'init')
        self.__get_parameter(0)

    def __get_parameter(self, i):
        self.__current_step = i

        if self.__steps[i] % 2 == 0:
            self.__values = linspace(-4, 4, self.__steps[i] + 1, dtype='f')
        else:
            self.__values = linspace(-4, 4, self.__steps[i] + 2, dtype='f')

        self.__errors = [None] * self.__values.size
        for i, value in enumerate(self.__values):
            self.__values[i] = value
            # parameters = self.__parameters.copy()
            # parameters[self.__current_step] = value
            # self.request_normals(parameters, i)
            self.request_normals((self.__current_step, value), i)

    def receive_image(self, image, index=None):
        if index == 'init':
            return
        elif index == 'pre':
            self.__get_parameter(self.__current_step + 1)
            return

        shadows = image

        if index is None:
            self.finish(normals, shadows)
            return

        self.__errors[index] = self.get_image_deviation(shadows)

        if None in self.__errors:
            return

        errors = array(self.__errors)
        max_error = errors.max()
        if self.__loop >= self.__determined_loops:
            X = exp(- errors / max_error).sum()
            v = rand()
            best_index = -1
            t = 0
            for i, error in enumerate(errors):
                e = exp(error / max_error) / X
                t += e
                if v <= e:
                    best_index = i
                    break
                else:
                    v -= e
            if best_index == -1:
                best_index = len(self.__errors) - 1
        else:
            best_index = argmin(self.__errors)

        self.__parameters[self.__current_step] = self.__values[best_index]
        print('{}.{}: Error of the best is {} ({})'.format(
            self.__loop, self.__current_step, self.__errors[best_index],
            min(self.__errors)))

        if self.__current_step + 1 == self._dimensions \
                and self.__loop + 1 >= self.__max_loops:
            self.request_normals(self.__parameters)
            return
        elif self.__current_step + 1 == self._dimensions:
            self.__current_step = 0
            self.__loop += 1
            self.request_normals((self._dimensions - 1,
                                  self.__values[best_index]), 'pre')
            return

        self.request_normals((self.__current_step, self.__values[best_index]),
                             'pre')

    def finish(self, normals, shadows):
        img = shadows[::-1]
        alpha = zeros(len(normals[:, 3]))
        alpha[nonzero(normals[:, 3])[0]] = 1.
        # data = (column_stack((img, img, img, alpha)) * 255).astype('i')
        image = Image.new('L', (500, 500))
        image.putdata((img*255).astype('i'))
        image.save('img.png')
        print('Finished')
<commit_msg>Requesting Face instead of normals in Gibbs Sampler<commit_after>from PIL import Image
from numpy import ones, zeros, linspace, argmin, nonzero, exp, array
# from numpy import column_stack
from numpy.random import rand

from .ModelFitter import ModelFitter
from src import Face

class GibbsSamplerFitter(ModelFitter):
    def __init__(self, image, dimensions=199, model=None, steps=None,
                 max_loops=1, determined_loops=0,
                 initial_face=None, callback=None):
        dimensions += 4
        self.__steps = ones(dimensions, dtype='i') if steps is None else steps
        self.__current_step = None
        self.__parameters = zeros(dimensions, dtype='f')

        self.__errors = None
        self.__values = None
        self.__loop = 0
        self.__max_loops = max_loops
        self.__determined_loops = determined_loops

        super(GibbsSamplerFitter, self).__init__(image, dimensions - 4, model,
                                                 initial_face, callback)
        self._dimensions += 4

    def start(self):
        self.__loop = 0

        self.__face = self._initial_face
        self.__parameters = zeros(self._dimensions, dtype='f')
        self.__parameters[-4] = self.__face.ambient_light
        self.__parameters[-3:] = self.__face.directed_light
        self.__parameters[:-4] = self.__face.coefficients

        self.request_face(self.__face, 'init')
        self.__get_parameter(0)

    def __get_parameter(self, i):
        self.__current_step = i

        if self.__steps[i] % 2 == 0:
            self.__values = linspace(-4, 4, self.__steps[i] + 1, dtype='f')
        else:
            self.__values = linspace(-4, 4, self.__steps[i] + 2, dtype='f')

        self.__errors = [None] * self.__values.size
        for i, value in enumerate(self.__values):
            self.__values[i] = value
            parameters = self.__parameters.copy()
            parameters[self.__current_step] = value
            self.request_face(Face.from_array(parameters), i)

    def receive_image(self, image, index=None):
        if index == 'init':
            return
        elif index == 'pre':
            self.__get_parameter(self.__current_step + 1)
            return

        shadows = image

        if index is None:
            self.finish(normals, shadows)
            return

        self.__errors[index] = self.get_image_deviation(shadows)

        if None in self.__errors:
            return

        errors = array(self.__errors)
        max_error = errors.max()
        if self.__loop >= self.__determined_loops:
            X = exp(- errors / max_error).sum()
            v = rand()
            best_index = -1
            t = 0
            for i, error in enumerate(errors):
                e = exp(error / max_error) / X
                t += e
                if v <= e:
                    best_index = i
                    break
                else:
                    v -= e
            if best_index == -1:
                best_index = len(self.__errors) - 1
        else:
            best_index = argmin(self.__errors)

        self.__parameters[self.__current_step] = self.__values[best_index]
        self.__face = Face.from_array(self.__parameters)
        print('{}.{}: Error of the best is {} ({})'.format(
            self.__loop, self.__current_step, self.__errors[best_index],
            min(self.__errors)))

        if self.__current_step + 1 == self._dimensions \
                and self.__loop + 1 >= self.__max_loops:
            self.request_face(self.__face)
            return
        elif self.__current_step + 1 == self._dimensions:
            self.__current_step = 0
            self.__loop += 1
            self.request_face(self.__face, 'pre')
            return

        self.request_face(self.__face, 'pre')

    def finish(self, normals, shadows):
        img = shadows[::-1]
        alpha = zeros(len(normals[:, 3]))
        alpha[nonzero(normals[:, 3])[0]] = 1.
        # data = (column_stack((img, img, img, alpha)) * 255).astype('i')
        image = Image.new('L', (500, 500))
        image.putdata((img*255).astype('i'))
        image.save('img.png')
        print('Finished')
<|endoftext|>


<commit_before><commit_msg>update to Inferbeddings class<commit_after><|endoftext|>