<commit_before># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r""""Converts music files to NoteSequence protos and writes TFRecord file.

Currently supports MIDI (.mid, .midi) and MusicXML (.xml, .mxl) files.

Example usage:
  $ bazel build magenta/scripts:convert_dir_to_note_sequences

  $ ./bazel-bin/magenta/scripts/convert_dir_to_note_sequences \
    --input_dir=/path/to/input/dir \
    --output_file=/path/to/tfrecord/file \
    --recursive
"""

import os

# internal imports
import tensorflow as tf

from magenta.music import midi_io
from magenta.music import musicxml_reader
from magenta.music import note_sequence_io

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('input_dir', None,
                           'Directory containing files to convert.')
tf.app.flags.DEFINE_string('output_file', None,
                           'Path to output TFRecord file. Will be overwritten '
                           'if it already exists.')
tf.app.flags.DEFINE_bool('recursive', False,
                         'Whether or not to recurse into subdirectories.')
tf.app.flags.DEFINE_string('log', 'INFO',
                           'The threshold for what messages will be logged '
                           'DEBUG, INFO, WARN, ERROR, or FATAL.')

def convert_directory(root_dir, sub_dir, sequence_writer, recursive=False):
  """Converts files to NoteSequences and writes to `sequence_writer`.

  Input files found in the specified directory specified by the combination of
  `root_dir` and `sub_dir` and converted to NoteSequence protos with the
  basename of `root_dir` as the collection_name, and the relative path to the
  file from `root_dir` as the filename. If `recursive` is true, recursively
  converts any subdirectories of the specified directory.

  Args:
    root_dir: A string specifying a root directory.
    sub_dir: A string specifying a path to a directory under `root_dir` in which
        to convert contents.
    sequence_writer: A NoteSequenceRecordWriter to write the resulting
        NoteSequence protos to.
    recursive: A boolean specifying whether or not recursively convert files
        contained in subdirectories of the specified directory.

  Returns:
    The number of NoteSequence protos written as an integer.
  """
  dir_to_convert = os.path.join(root_dir, sub_dir)
  tf.logging.info("Converting files in '%s'.", dir_to_convert)
  files_in_dir = tf.gfile.ListDirectory(os.path.join(dir_to_convert))
  recurse_sub_dirs = []
  sequences_written = 0
  sequences_skipped = 0
  for file_in_dir in files_in_dir:
    full_file_path = os.path.join(dir_to_convert, file_in_dir)
    if tf.gfile.IsDirectory(full_file_path):
      if recursive:
        recurse_sub_dirs.append(os.path.join(sub_dir, file_in_dir))
      continue

    if full_file_path.endswith('.mid') or full_file_path.endswith('.midi'):
      sequence = convert_midi(root_dir, sub_dir, full_file_path)
    elif full_file_path.endswith('.xml') or full_file_path.endswith('.mxl'):
      sequence = convert_musicxml(root_dir, sub_dir, full_file_path)
    else:
      tf.logging.info('Unable to find a converter for file %s', full_file_path)
      sequence = None

    if sequence is None:
      sequences_skipped += 1
      continue

    sequence_writer.write(sequence)
    sequences_written += 1
  tf.logging.info("Converted %d files in '%s'.", sequences_written,
                  dir_to_convert)
  tf.logging.info('Could not parse %d files.', sequences_skipped)
  for recurse_sub_dir in recurse_sub_dirs:
    sequences_written += convert_directory(
        root_dir, recurse_sub_dir, sequence_writer, recursive)
  return sequences_written

def convert_midi(root_dir, sub_dir, full_file_path):
  """Converts a midi file to a sequence proto.

  Args:
    root_dir: A string specifying the root directory for the files being
        converted.
    sub_dir: The directory being converted currently.
    full_file_path: the full path to the file to convert.

  Returns:
    Either a NoteSequence proto or None if the file could not be converted.
  """
  try:
    sequence = midi_io.midi_to_sequence_proto(
        tf.gfile.FastGFile(full_file_path, 'rb').read())
  except midi_io.MIDIConversionError as e:
    tf.logging.warning(
        'Could not parse MIDI file %s. It will be skipped. Error was: %s',
        full_file_path, e)
    return None
  sequence.collection_name = os.path.basename(root_dir)
  sequence.filename = os.path.join(sub_dir, os.path.basename(full_file_path))
  sequence.id = note_sequence_io.generate_note_sequence_id(
      sequence.filename, sequence.collection_name, 'midi')
  return sequence

def convert_musicxml(root_dir, sub_dir, full_file_path):
  """Converts a musicxml file to a sequence proto.

  Args:
    root_dir: A string specifying the root directory for the files being
        converted.
    sub_dir: The directory being converted currently.
    full_file_path: the full path to the file to convert.

  Returns:
    Either a NoteSequence proto or None if the file could not be converted.
  """
  try:
    sequence = musicxml_reader.musicxml_file_to_sequence_proto(full_file_path)
  except musicxml_reader.MusicXMLConversionError as e:
    tf.logging.warning(
        'Could not parse MusicXML file %s. It will be skipped. Error was: %s',
        full_file_path, e)
    return None
  sequence.collection_name = os.path.basename(root_dir)
  sequence.filename = os.path.join(sub_dir, os.path.basename(full_file_path))
  sequence.id = note_sequence_io.generate_note_sequence_id(
      sequence.filename, sequence.collection_name, 'musicxml')
  return sequence

def main(unused_argv):
  tf.logging.set_verbosity(FLAGS.log)

  if not FLAGS.input_dir:
    tf.logging.fatal('--input_dir required')
    return
  if not FLAGS.output_file:
    tf.logging.fatal('--output_file required')
    return

  input_dir = os.path.expanduser(FLAGS.input_dir)
  output_dir = os.path.dirname(output_file)
  output_file = os.path.expanduser(FLAGS.output_file)

  if output_dir:
    tf.gfile.MakeDirs(output_dir)

  with note_sequence_io.NoteSequenceRecordWriter(
      output_file) as sequence_writer:
    sequences_written = convert_directory(input_dir, '', sequence_writer,
                                          FLAGS.recursive)
    tf.logging.info("Wrote %d NoteSequence protos to '%s'", sequences_written,
                    output_file)

def console_entry_point():
  tf.app.run(main)

if __name__ == '__main__':
  console_entry_point()
<commit_msg>fix typo (#574)<commit_after># Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r""""Converts music files to NoteSequence protos and writes TFRecord file.

Currently supports MIDI (.mid, .midi) and MusicXML (.xml, .mxl) files.

Example usage:
  $ bazel build magenta/scripts:convert_dir_to_note_sequences

  $ ./bazel-bin/magenta/scripts/convert_dir_to_note_sequences \
    --input_dir=/path/to/input/dir \
    --output_file=/path/to/tfrecord/file \
    --recursive
"""

import os

# internal imports
import tensorflow as tf

from magenta.music import midi_io
from magenta.music import musicxml_reader
from magenta.music import note_sequence_io

FLAGS = tf.app.flags.FLAGS

tf.app.flags.DEFINE_string('input_dir', None,
                           'Directory containing files to convert.')
tf.app.flags.DEFINE_string('output_file', None,
                           'Path to output TFRecord file. Will be overwritten '
                           'if it already exists.')
tf.app.flags.DEFINE_bool('recursive', False,
                         'Whether or not to recurse into subdirectories.')
tf.app.flags.DEFINE_string('log', 'INFO',
                           'The threshold for what messages will be logged '
                           'DEBUG, INFO, WARN, ERROR, or FATAL.')

def convert_directory(root_dir, sub_dir, sequence_writer, recursive=False):
  """Converts files to NoteSequences and writes to `sequence_writer`.

  Input files found in the specified directory specified by the combination of
  `root_dir` and `sub_dir` and converted to NoteSequence protos with the
  basename of `root_dir` as the collection_name, and the relative path to the
  file from `root_dir` as the filename. If `recursive` is true, recursively
  converts any subdirectories of the specified directory.

  Args:
    root_dir: A string specifying a root directory.
    sub_dir: A string specifying a path to a directory under `root_dir` in which
        to convert contents.
    sequence_writer: A NoteSequenceRecordWriter to write the resulting
        NoteSequence protos to.
    recursive: A boolean specifying whether or not recursively convert files
        contained in subdirectories of the specified directory.

  Returns:
    The number of NoteSequence protos written as an integer.
  """
  dir_to_convert = os.path.join(root_dir, sub_dir)
  tf.logging.info("Converting files in '%s'.", dir_to_convert)
  files_in_dir = tf.gfile.ListDirectory(os.path.join(dir_to_convert))
  recurse_sub_dirs = []
  sequences_written = 0
  sequences_skipped = 0
  for file_in_dir in files_in_dir:
    full_file_path = os.path.join(dir_to_convert, file_in_dir)
    if tf.gfile.IsDirectory(full_file_path):
      if recursive:
        recurse_sub_dirs.append(os.path.join(sub_dir, file_in_dir))
      continue

    if full_file_path.endswith('.mid') or full_file_path.endswith('.midi'):
      sequence = convert_midi(root_dir, sub_dir, full_file_path)
    elif full_file_path.endswith('.xml') or full_file_path.endswith('.mxl'):
      sequence = convert_musicxml(root_dir, sub_dir, full_file_path)
    else:
      tf.logging.info('Unable to find a converter for file %s', full_file_path)
      sequence = None

    if sequence is None:
      sequences_skipped += 1
      continue

    sequence_writer.write(sequence)
    sequences_written += 1
  tf.logging.info("Converted %d files in '%s'.", sequences_written,
                  dir_to_convert)
  tf.logging.info('Could not parse %d files.', sequences_skipped)
  for recurse_sub_dir in recurse_sub_dirs:
    sequences_written += convert_directory(
        root_dir, recurse_sub_dir, sequence_writer, recursive)
  return sequences_written

def convert_midi(root_dir, sub_dir, full_file_path):
  """Converts a midi file to a sequence proto.

  Args:
    root_dir: A string specifying the root directory for the files being
        converted.
    sub_dir: The directory being converted currently.
    full_file_path: the full path to the file to convert.

  Returns:
    Either a NoteSequence proto or None if the file could not be converted.
  """
  try:
    sequence = midi_io.midi_to_sequence_proto(
        tf.gfile.FastGFile(full_file_path, 'rb').read())
  except midi_io.MIDIConversionError as e:
    tf.logging.warning(
        'Could not parse MIDI file %s. It will be skipped. Error was: %s',
        full_file_path, e)
    return None
  sequence.collection_name = os.path.basename(root_dir)
  sequence.filename = os.path.join(sub_dir, os.path.basename(full_file_path))
  sequence.id = note_sequence_io.generate_note_sequence_id(
      sequence.filename, sequence.collection_name, 'midi')
  return sequence

def convert_musicxml(root_dir, sub_dir, full_file_path):
  """Converts a musicxml file to a sequence proto.

  Args:
    root_dir: A string specifying the root directory for the files being
        converted.
    sub_dir: The directory being converted currently.
    full_file_path: the full path to the file to convert.

  Returns:
    Either a NoteSequence proto or None if the file could not be converted.
  """
  try:
    sequence = musicxml_reader.musicxml_file_to_sequence_proto(full_file_path)
  except musicxml_reader.MusicXMLConversionError as e:
    tf.logging.warning(
        'Could not parse MusicXML file %s. It will be skipped. Error was: %s',
        full_file_path, e)
    return None
  sequence.collection_name = os.path.basename(root_dir)
  sequence.filename = os.path.join(sub_dir, os.path.basename(full_file_path))
  sequence.id = note_sequence_io.generate_note_sequence_id(
      sequence.filename, sequence.collection_name, 'musicxml')
  return sequence

def main(unused_argv):
  tf.logging.set_verbosity(FLAGS.log)

  if not FLAGS.input_dir:
    tf.logging.fatal('--input_dir required')
    return
  if not FLAGS.output_file:
    tf.logging.fatal('--output_file required')
    return

  input_dir = os.path.expanduser(FLAGS.input_dir)
  output_file = os.path.expanduser(FLAGS.output_file)
  output_dir = os.path.dirname(output_file)

  if output_dir:
    tf.gfile.MakeDirs(output_dir)

  with note_sequence_io.NoteSequenceRecordWriter(
      output_file) as sequence_writer:
    sequences_written = convert_directory(input_dir, '', sequence_writer,
                                          FLAGS.recursive)
    tf.logging.info("Wrote %d NoteSequence protos to '%s'", sequences_written,
                    output_file)

def console_entry_point():
  tf.app.run(main)

if __name__ == '__main__':
  console_entry_point()
<|endoftext|>


<commit_before>from annotypes import Anno, add_call_types
from tornado.websocket import WebSocketHandler, WebSocketError

from malcolm.core import Part, json_decode, deserialize_object, Request, PathRequest,\
    json_encode, Subscribe, Unsubscribe, Delta, Update, Error, Response, FieldError
from malcolm.modules import builtin
from ..infos import HandlerInfo
from ..hooks import ReportHandlersHook, ALoop, UHandlerInfos, PublishHook, \
    APublished

# For some reason tornado doesn't make us implement all abstract methods
# noinspection PyAbstractClass
class MalcWebSocketHandler(WebSocketHandler):
    _server_part = None
    _loop = None

    def initialize(self, server_part=None, loop=None):
        self._server_part = server_part
        self._loop = loop

    def on_message(self, message):
        # called in tornado's thread
        msg_id = -1
        try:
            d = json_decode(message)

            try:
                msg_id = d['id']
            except KeyError:
                raise FieldError('id field not present in JSON message')

            request = deserialize_object(d, Request)
            request.set_callback(self.on_response)
            self._server_part.on_request(request)

        except Exception as e:
            self._server_part.log.exception("Error handling message from client")
            error = Error(msg_id, e)
            error_message = error.to_dict()
            self.write_message(json_encode(error_message))

    def on_response(self, response):
        # called from any thread
        self._loop.add_callback(
            self._server_part.on_response, response, self)

    # http://stackoverflow.com/q/24851207
    # TODO: remove this when the web gui is hosted from the box
    def check_origin(self, origin):
        return True

with Anno("Part name and subdomain name to host websocket on"):
    AName = str

class WebsocketServerPart(Part):
    def __init__(self, name="ws"):
        # type: (AName) -> None
        super(WebsocketServerPart, self).__init__(name)
        # {id: Subscribe}
        self._subscription_keys = {}
        # [mri]
        self._published = []
        # Hooks
        self.register_hooked(ReportHandlersHook, self.report_handlers)
        self.register_hooked(PublishHook, self.publish)

    @add_call_types
    def report_handlers(self, loop):
        # type: (ALoop) -> UHandlerInfos
        regexp = r"/%s" % self.name
        info = HandlerInfo(
            regexp, MalcWebSocketHandler, server_part=self, loop=loop)
        return info

    @add_call_types
    def publish(self, published):
        # type: (APublished) -> None
        # called from any thread
        self._published = published
        for request in self._subscription_keys.values():
            if request.path[0] == ".":
                self._notify_published(request)

    def on_request(self, request):
        # type: (Request) -> None
        # called from tornado thread
        if isinstance(request, PathRequest):
            if not request.path:
                raise ValueError("No path supplied")
        self.log.info("Request: %s", request)
        if isinstance(request, Subscribe):
            if request.generate_key() in self._subscription_keys.keys():
                raise FieldError("duplicate subscription ID on client")
            if request.path[0] == ".":
                # special entries
                assert request.path[1] == "blocks", \
                    "Don't know how to subscribe to %s" % (request.path,)
                self._notify_published(request)
            self._subscription_keys[request.generate_key()] = request
            if request.path[0] == ".":
                return

        if isinstance(request, Unsubscribe):
            subscribe = self._subscription_keys.pop(request.generate_key())
            mri = subscribe.path[0]
            if mri == ".":
                # service requests on ourself
                response, cb = subscribe.return_response()
                cb(response)
                return
        else:
            mri = request.path[0]
        self.registrar.report(builtin.infos.RequestInfo(request, mri))

    def on_response(self, response, websocket):
        # type: (Response, MalcWebSocketHandler) -> None
        # called from tornado thread
        message = json_encode(response)
        try:
            websocket.write_message(message)
        except WebSocketError:
            # The websocket is dead. If the response was a Delta or Update, then
            # unsubscribe so the local controller doesn't keep on trying to
            # respond
            if isinstance(response, (Delta, Update)):
                # Websocket is dead so we can clear the subscription key.
                # Subsequent updates may come in before the unsubscribe, but
                # ignore them as we can't do anything about it
                subscribe = self._subscription_keys.pop((websocket.on_response, response.id), None)
                if subscribe:
                    self.log.exception('WebSocket Error; unsubscribing from stale handle')
                    unsubscribe = Unsubscribe(response.id)
                    unsubscribe.set_callback(websocket.on_response)
                    self.registrar.report(builtin.infos.RequestInfo(
                        unsubscribe, subscribe.path[0]))

    def _notify_published(self, request):
        # type: (Request) -> None
        # called from any thread
        cb, response = request.update_response(self._published)
        cb(response)
<commit_msg>Change stale websocket errors into info<commit_after>from annotypes import Anno, add_call_types
from tornado.websocket import WebSocketHandler, WebSocketError

from malcolm.core import Part, json_decode, deserialize_object, Request, PathRequest,\
    json_encode, Subscribe, Unsubscribe, Delta, Update, Error, Response, FieldError
from malcolm.modules import builtin
from ..infos import HandlerInfo
from ..hooks import ReportHandlersHook, ALoop, UHandlerInfos, PublishHook, \
    APublished

# For some reason tornado doesn't make us implement all abstract methods
# noinspection PyAbstractClass
class MalcWebSocketHandler(WebSocketHandler):
    _server_part = None
    _loop = None

    def initialize(self, server_part=None, loop=None):
        self._server_part = server_part
        self._loop = loop

    def on_message(self, message):
        # called in tornado's thread
        msg_id = -1
        try:
            d = json_decode(message)

            try:
                msg_id = d['id']
            except KeyError:
                raise FieldError('id field not present in JSON message')

            request = deserialize_object(d, Request)
            request.set_callback(self.on_response)
            self._server_part.on_request(request)

        except Exception as e:
            self._server_part.log.exception("Error handling message from client")
            error = Error(msg_id, e)
            error_message = error.to_dict()
            self.write_message(json_encode(error_message))

    def on_response(self, response):
        # called from any thread
        self._loop.add_callback(
            self._server_part.on_response, response, self)

    # http://stackoverflow.com/q/24851207
    # TODO: remove this when the web gui is hosted from the box
    def check_origin(self, origin):
        return True

with Anno("Part name and subdomain name to host websocket on"):
    AName = str

class WebsocketServerPart(Part):
    def __init__(self, name="ws"):
        # type: (AName) -> None
        super(WebsocketServerPart, self).__init__(name)
        # {id: Subscribe}
        self._subscription_keys = {}
        # [mri]
        self._published = []
        # Hooks
        self.register_hooked(ReportHandlersHook, self.report_handlers)
        self.register_hooked(PublishHook, self.publish)

    @add_call_types
    def report_handlers(self, loop):
        # type: (ALoop) -> UHandlerInfos
        regexp = r"/%s" % self.name
        info = HandlerInfo(
            regexp, MalcWebSocketHandler, server_part=self, loop=loop)
        return info

    @add_call_types
    def publish(self, published):
        # type: (APublished) -> None
        # called from any thread
        self._published = published
        for request in self._subscription_keys.values():
            if request.path[0] == ".":
                self._notify_published(request)

    def on_request(self, request):
        # type: (Request) -> None
        # called from tornado thread
        if isinstance(request, PathRequest):
            if not request.path:
                raise ValueError("No path supplied")
        self.log.info("Request: %s", request)
        if isinstance(request, Subscribe):
            if request.generate_key() in self._subscription_keys.keys():
                raise FieldError("duplicate subscription ID on client")
            if request.path[0] == ".":
                # special entries
                assert request.path[1] == "blocks", \
                    "Don't know how to subscribe to %s" % (request.path,)
                self._notify_published(request)
            self._subscription_keys[request.generate_key()] = request
            if request.path[0] == ".":
                return

        if isinstance(request, Unsubscribe):
            subscribe = self._subscription_keys.pop(request.generate_key())
            mri = subscribe.path[0]
            if mri == ".":
                # service requests on ourself
                response, cb = subscribe.return_response()
                cb(response)
                return
        else:
            mri = request.path[0]
        self.registrar.report(builtin.infos.RequestInfo(request, mri))

    def on_response(self, response, websocket):
        # type: (Response, MalcWebSocketHandler) -> None
        # called from tornado thread
        message = json_encode(response)
        try:
            websocket.write_message(message)
        except WebSocketError:
            # The websocket is dead. If the response was a Delta or Update, then
            # unsubscribe so the local controller doesn't keep on trying to
            # respond
            if isinstance(response, (Delta, Update)):
                # Websocket is dead so we can clear the subscription key.
                # Subsequent updates may come in before the unsubscribe, but
                # ignore them as we can't do anything about it
                subscribe = self._subscription_keys.pop(
                    (websocket.on_response, response.id), None)
                if subscribe:
                    self.log.info(
                        'WebSocket Error; unsubscribing from stale handle')
                    unsubscribe = Unsubscribe(response.id)
                    unsubscribe.set_callback(websocket.on_response)
                    self.registrar.report(builtin.infos.RequestInfo(
                        unsubscribe, subscribe.path[0]))

    def _notify_published(self, request):
        # type: (Request) -> None
        # called from any thread
        cb, response = request.update_response(self._published)
        cb(response)
<|endoftext|>


<commit_before>'''
Created on 15/03/2010

@author: peio
'''
import os, shutil
from tempfile import NamedTemporaryFile

from franklin.backbone.analysis import (Analyzer, scrape_info_from_fname,
                                        LastAnalysisAnalyzer)
from franklin.mapping import map_reads
from franklin.utils.misc_utils import NamedTemporaryDir, VersionedPath
from franklin.backbone.specifications import BACKBONE_BASENAMES
from franklin.sam import (bam2sam, add_header_and_tags_to_sam, merge_sam,
                          sam2bam, sort_bam_sam, standardize_sam, realign_bam)

class SetAssemblyAsReferenceAnalyzer(Analyzer):
    'It sets the reference assembly as mapping reference'
    def run(self):
        '''It runs the analysis.'''
        contigs_path = self._get_input_fpaths()['contigs']
        contigs_ext = contigs_path.extension
        reference_dir = self._create_output_dirs()['result']
        reference_fpath = os.path.join(reference_dir,
                          BACKBONE_BASENAMES['mapping_reference'] + contigs_ext)
        if os.path.exists(reference_fpath):
            os.remove(reference_fpath)
        os.symlink(contigs_path.last_version, reference_fpath)

def _get_basename(fpath):
    'It returns the base name without path and extension'
    return os.path.splitext(os.path.basename(fpath))[0]

class MappingAnalyzer(Analyzer):
    'It performs the mapping of the sequences to the reference'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings['Mappers']
        inputs = self._get_input_fpaths()
        reads_fpaths = inputs['reads']
        reference_fpath = inputs['reference']
        output_dir = self._create_output_dirs(timestamped=True)['result']

        for read_fpath in reads_fpaths:
            read_info = scrape_info_from_fname(read_fpath)
            platform = read_info['pl']
            #which maper are we using for this platform
            mapper = settings['mapper_for_%s' % platform]
            out_bam_fpath = os.path.join(output_dir,
                                         read_fpath.basename + '.bam')
            mapping_parameters = {}
            if platform in ('454', 'sanger'):
                mapping_parameters['reads_length'] = 'long'
            else:
                mapping_parameters['reads_length'] = 'short'
            if not os.path.exists(out_bam_fpath):
                map_reads(mapper,
                          reads_fpath=read_fpath.last_version,
                          reference_fpath=reference_fpath.last_version,
                          out_bam_fpath=out_bam_fpath,
                          parameters = mapping_parameters)
        self._log({'analysis_finished':True})

class MergeBamAnalyzer(Analyzer):
    'It performs the merge of various bams into only one'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings
        project_path = settings['General_settings']['project_path']
        os.chdir(project_path)
        inputs          = self._get_input_fpaths()
        bam_paths      = inputs['bams']
        reference_path = inputs['reference']

        output_dir      = self._create_output_dirs()['result']
        merged_bam_path = VersionedPath(os.path.join(output_dir,
                                        BACKBONE_BASENAMES['merged_bam']))

        merged_bam_fpath = merged_bam_path.next_version

        #Do we have to add the default qualities to the sam file?
        #do we have characters different from ACTGN?
        add_qualities = ('Sam_processing' in settings and
                       'add_default_qualities' in settings['Sam_processing'] and
                       settings['Sam_processing']['add_default_qualities'])
        if add_qualities:
            default_sanger_quality = settings['Other_settings']['default_sanger_quality']
            default_sanger_quality = int(default_sanger_quality)
        else:
            default_sanger_quality = None
        temp_dir = NamedTemporaryDir()
        for bam_path in bam_paths:
            bam_basename = bam_path.basename
            temp_sam     =  NamedTemporaryFile(prefix='%s.' % bam_basename,
                                               suffix='.sam')
            sam_fpath    = os.path.join(temp_dir.name, bam_basename + '.sam')
            bam2sam(bam_path.last_version, temp_sam.name)
            sam_fhand = open(sam_fpath, 'w')
            # First we need to create the sam with added tags and headers
            add_header_and_tags_to_sam(temp_sam, sam_fhand)
            temp_sam.close()
            sam_fhand.close()
            #the standardization
            temp_sam2 = NamedTemporaryFile(prefix='%s.' % bam_basename,
                                           suffix='.sam', delete=False)
            standardize_sam(open(sam_fhand.name), temp_sam2,
                            default_sanger_quality,
                            add_def_qual=add_qualities,
                            only_std_char=True)
            temp_sam2.flush()
            shutil.move(temp_sam2.name, sam_fhand.name)

            temp_sam2.close()

        get_sam_fpaths = lambda dir_: [os.path.join(dir_, fname) for fname in os.listdir(dir_) if fname.endswith('.sam')]

        # Once the headers are ready we are going to merge
        sams = get_sam_fpaths(temp_dir.name)
        sams = [open(sam) for sam in sams]

        temp_sam = NamedTemporaryFile(suffix='.sam')
        reference_fhand = open(reference_path.last_version)
        merge_sam(sams, temp_sam, reference_fhand)
        reference_fhand.close()

        # close files
        for sam in sams:
            sam.close()
        # Convert sam into a bam,(Temporary)
        temp_bam = NamedTemporaryFile(suffix='.bam')
        sam2bam(temp_sam.name, temp_bam.name)

        # finally we need to order the bam
        sort_bam_sam(temp_bam.name, merged_bam_fpath)
        temp_bam.close()
        temp_sam.close()
        self._log({'analysis_finished':True})

class RealignBamAnalyzer(Analyzer):
    'It realigns the bam using GATK'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings
        project_path = settings['General_settings']['project_path']
        os.chdir(project_path)
        inputs    = self._get_input_fpaths()
        bam_path = inputs['bam']
        bam_fpath = bam_path.last_version
        reference_path = inputs['reference']

        #memory for the java programs
        if ('Other_settings' in settings and
            'java_memory' in settings['Other_settings']):
            java_mem = settings['Other_settings']['java_memory']
        else:
            java_mem = None

        #we need a temporary path
        temp_bam = NamedTemporaryFile(suffix='.bam')
        temp_bam_fpath = temp_bam.name
        temp_bam.close()

        #do the realigment
        realign_bam(bam_fpath=bam_fpath,
                    reference_fpath=reference_path.last_version,
                    out_bam_fpath=temp_bam_fpath,
                    java_memory=java_mem)
        #a new version for the original bam
        out_bam_fpath = bam_path.next_version
        shutil.move(temp_bam_fpath, out_bam_fpath)
        self._log({'analysis_finished':True})

DEFINITIONS = {
    'set_assembly_as_reference':
        {'inputs':{
                   'contigs':
                            {'directory': 'assembly_result',
                             'file': 'contigs'},
                   },
         'outputs':{'result':{'directory': 'mapping_reference'}},
         'analyzer': SetAssemblyAsReferenceAnalyzer,
        },
    'mapping':
        {'inputs':{
            'reads':
                {'directory': 'cleaned_reads',
                 'file_kinds': 'sequence_files'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mappings_by_readgroup'}},
         'analyzer': MappingAnalyzer,
        },
    'select_last_mapping':
        {'inputs':{'analyses_dir':{'directory': 'mappings'}},
         'outputs':{'result':{'directory': 'mapping_result',
                              'create':False}},
         'analyzer': LastAnalysisAnalyzer,
        },
    'merge_bam':
        {'inputs':{
            'bams':
                {'directory': 'mappings_by_readgroup',
                 'file_kinds': 'bam'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mapping_result'}},
         'analyzer': MergeBamAnalyzer,
        },
    'realign_bam':
        {'inputs':{
            'bam':
                {'directory': 'mapping_result',
                 'file': 'merged_bam'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mapping_result'}},
         'analyzer': RealignBamAnalyzer,
        },
}
<commit_msg>bufix: reference name without pint in the extension<commit_after>'''
Created on 15/03/2010

@author: peio
'''
import os, shutil
from tempfile import NamedTemporaryFile

from franklin.backbone.analysis import (Analyzer, scrape_info_from_fname,
                                        LastAnalysisAnalyzer)
from franklin.mapping import map_reads
from franklin.utils.misc_utils import NamedTemporaryDir, VersionedPath
from franklin.backbone.specifications import BACKBONE_BASENAMES
from franklin.sam import (bam2sam, add_header_and_tags_to_sam, merge_sam,
                          sam2bam, sort_bam_sam, standardize_sam, realign_bam)

class SetAssemblyAsReferenceAnalyzer(Analyzer):
    'It sets the reference assembly as mapping reference'
    def run(self):
        '''It runs the analysis.'''
        contigs_path = self._get_input_fpaths()['contigs']
        contigs_ext = contigs_path.extension
        reference_dir = self._create_output_dirs()['result']
        reference_fpath = os.path.join(reference_dir,
                          BACKBONE_BASENAMES['mapping_reference'] + '.' + \
                                                                    contigs_ext)
        if os.path.exists(reference_fpath):
            os.remove(reference_fpath)
        os.symlink(contigs_path.last_version, reference_fpath)

def _get_basename(fpath):
    'It returns the base name without path and extension'
    return os.path.splitext(os.path.basename(fpath))[0]

class MappingAnalyzer(Analyzer):
    'It performs the mapping of the sequences to the reference'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings['Mappers']
        inputs = self._get_input_fpaths()
        reads_fpaths = inputs['reads']
        reference_fpath = inputs['reference']
        output_dir = self._create_output_dirs(timestamped=True)['result']

        for read_fpath in reads_fpaths:
            read_info = scrape_info_from_fname(read_fpath)
            platform = read_info['pl']
            #which maper are we using for this platform
            mapper = settings['mapper_for_%s' % platform]
            out_bam_fpath = os.path.join(output_dir,
                                         read_fpath.basename + '.bam')
            mapping_parameters = {}
            if platform in ('454', 'sanger'):
                mapping_parameters['reads_length'] = 'long'
            else:
                mapping_parameters['reads_length'] = 'short'
            if not os.path.exists(out_bam_fpath):
                map_reads(mapper,
                          reads_fpath=read_fpath.last_version,
                          reference_fpath=reference_fpath.last_version,
                          out_bam_fpath=out_bam_fpath,
                          parameters = mapping_parameters)
        self._log({'analysis_finished':True})

class MergeBamAnalyzer(Analyzer):
    'It performs the merge of various bams into only one'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings
        project_path = settings['General_settings']['project_path']
        os.chdir(project_path)
        inputs          = self._get_input_fpaths()
        bam_paths      = inputs['bams']
        reference_path = inputs['reference']

        output_dir      = self._create_output_dirs()['result']
        merged_bam_path = VersionedPath(os.path.join(output_dir,
                                        BACKBONE_BASENAMES['merged_bam']))

        merged_bam_fpath = merged_bam_path.next_version

        #Do we have to add the default qualities to the sam file?
        #do we have characters different from ACTGN?
        add_qualities = ('Sam_processing' in settings and
                       'add_default_qualities' in settings['Sam_processing'] and
                       settings['Sam_processing']['add_default_qualities'])
        if add_qualities:
            default_sanger_quality = settings['Other_settings']['default_sanger_quality']
            default_sanger_quality = int(default_sanger_quality)
        else:
            default_sanger_quality = None
        temp_dir = NamedTemporaryDir()
        for bam_path in bam_paths:
            bam_basename = bam_path.basename
            temp_sam     =  NamedTemporaryFile(prefix='%s.' % bam_basename,
                                               suffix='.sam')
            sam_fpath    = os.path.join(temp_dir.name, bam_basename + '.sam')
            bam2sam(bam_path.last_version, temp_sam.name)
            sam_fhand = open(sam_fpath, 'w')
            # First we need to create the sam with added tags and headers
            add_header_and_tags_to_sam(temp_sam, sam_fhand)
            temp_sam.close()
            sam_fhand.close()
            #the standardization
            temp_sam2 = NamedTemporaryFile(prefix='%s.' % bam_basename,
                                           suffix='.sam', delete=False)
            standardize_sam(open(sam_fhand.name), temp_sam2,
                            default_sanger_quality,
                            add_def_qual=add_qualities,
                            only_std_char=True)
            temp_sam2.flush()
            shutil.move(temp_sam2.name, sam_fhand.name)

            temp_sam2.close()

        get_sam_fpaths = lambda dir_: [os.path.join(dir_, fname) for fname in os.listdir(dir_) if fname.endswith('.sam')]

        # Once the headers are ready we are going to merge
        sams = get_sam_fpaths(temp_dir.name)
        sams = [open(sam) for sam in sams]

        temp_sam = NamedTemporaryFile(suffix='.sam')
        reference_fhand = open(reference_path.last_version)
        merge_sam(sams, temp_sam, reference_fhand)
        reference_fhand.close()

        # close files
        for sam in sams:
            sam.close()
        # Convert sam into a bam,(Temporary)
        temp_bam = NamedTemporaryFile(suffix='.bam')
        sam2bam(temp_sam.name, temp_bam.name)

        # finally we need to order the bam
        sort_bam_sam(temp_bam.name, merged_bam_fpath)
        temp_bam.close()
        temp_sam.close()
        self._log({'analysis_finished':True})

class RealignBamAnalyzer(Analyzer):
    'It realigns the bam using GATK'
    def run(self):
        '''It runs the analysis.'''
        self._log({'analysis_started':True})
        settings = self._project_settings
        project_path = settings['General_settings']['project_path']
        os.chdir(project_path)
        inputs    = self._get_input_fpaths()
        bam_path = inputs['bam']
        bam_fpath = bam_path.last_version
        reference_path = inputs['reference']

        #memory for the java programs
        if ('Other_settings' in settings and
            'java_memory' in settings['Other_settings']):
            java_mem = settings['Other_settings']['java_memory']
        else:
            java_mem = None

        #we need a temporary path
        temp_bam = NamedTemporaryFile(suffix='.bam')
        temp_bam_fpath = temp_bam.name
        temp_bam.close()

        #do the realigment
        realign_bam(bam_fpath=bam_fpath,
                    reference_fpath=reference_path.last_version,
                    out_bam_fpath=temp_bam_fpath,
                    java_memory=java_mem)
        #a new version for the original bam
        out_bam_fpath = bam_path.next_version
        shutil.move(temp_bam_fpath, out_bam_fpath)
        self._log({'analysis_finished':True})

DEFINITIONS = {
    'set_assembly_as_reference':
        {'inputs':{
                   'contigs':
                            {'directory': 'assembly_result',
                             'file': 'contigs'},
                   },
         'outputs':{'result':{'directory': 'mapping_reference'}},
         'analyzer': SetAssemblyAsReferenceAnalyzer,
        },
    'mapping':
        {'inputs':{
            'reads':
                {'directory': 'cleaned_reads',
                 'file_kinds': 'sequence_files'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mappings_by_readgroup'}},
         'analyzer': MappingAnalyzer,
        },
    'select_last_mapping':
        {'inputs':{'analyses_dir':{'directory': 'mappings'}},
         'outputs':{'result':{'directory': 'mapping_result',
                              'create':False}},
         'analyzer': LastAnalysisAnalyzer,
        },
    'merge_bam':
        {'inputs':{
            'bams':
                {'directory': 'mappings_by_readgroup',
                 'file_kinds': 'bam'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mapping_result'}},
         'analyzer': MergeBamAnalyzer,
        },
    'realign_bam':
        {'inputs':{
            'bam':
                {'directory': 'mapping_result',
                 'file': 'merged_bam'},
            'reference':
                {'directory': 'mapping_reference',
                'file': 'mapping_reference'},
            },
         'outputs':{'result':{'directory': 'mapping_result'}},
         'analyzer': RealignBamAnalyzer,
        },
}
<|endoftext|>


<commit_before>#! /usr/bin/env python
import sys
from IPython.kernel import find_connection_file, KernelManager

def connect_to_kernel(file_name):
    cf = find_connection_file(file_name)
    km = KernelManager(connection_file=cf)
    km.load_connection_file()
    return km.client()

if __name__=='__main__':
    code = sys.argv[1]

    # Connect to last opened instance
    client = connect_to_kernel('')

    # Code is run on the shell channel
    # Execution is immediate and async
    client.start_channels()
    uuid = client.execute(code)

    # Block for a reply
    reply = client.get_shell_msg()
    status = reply['content']['status']

    if status == 'ok':
        prompt = reply['content']['execution_count']
        print 'Out [%s]: succeeded!' % prompt
    elif status == 'error':
        import re
        regex = re.compile('\x1b\[[0-9;m]*', re.UNICODE)

        print 'failed!'
        # strip the ansi color codes from the ultraTB traceback
        for line in reply['content']['traceback']:
            print regex.sub('', line)

<commit_msg>fix regex to strip ansi color codes<commit_after>#! /usr/bin/env python
import sys

def connect_to_kernel(file_name):
    from IPython.kernel import find_connection_file, KernelManager
    cf = find_connection_file(file_name)
    km = KernelManager(connection_file=cf)
    km.load_connection_file()
    return km.client()

if __name__=='__main__':
    code = sys.argv[1]

    # Connect to last opened instance
    print "connecting..."
    client = connect_to_kernel('')

    # Code is run on the shell channel
    # Execution is immediate and async
    client.start_channels()
    uuid = client.execute(code)

    # Block for a reply
    reply = client.get_shell_msg()
    status = reply['content']['status']

    if status == 'ok':
        prompt = reply['content']['execution_count']
        print 'Out [%s]: succeeded!' % prompt
    elif status == 'error':
        import re
        regex = re.compile('\x1b\[[0-9;]*m', re.UNICODE)

        print 'failed!'
        # strip the ansi color codes from the ultraTB traceback
        for line in reply['content']['traceback']:
            print regex.sub('', line)

<|endoftext|>


<commit_before>import numpy as np
from numpy.testing import assert_allclose
from nose.tools import raises

import menpo.io as mio
from menpo.landmark import labeller, ibug_68_trimesh
from menpo.transform import PiecewiseAffine, ThinPlateSplines
from menpo.fitmultilevel.aam import AAMBuilder
from menpo.fitmultilevel.featurefunctions import sparse_hog

# load images
filenames = ['breakingbad.jpg', 'einstein.jpg']
training_images = []
for i in range(2):
    im = mio.import_builtin_asset(filenames[i])
    im.crop_to_landmarks_proportion(0.1)
    labeller(im, 'PTS', ibug_68_trimesh)
    if im.n_channels == 3:
        im = im.as_greyscale(mode='luminosity')
    training_images.append(im)

# build aams
aam1 = AAMBuilder(feature_type=['igo', sparse_hog, None],
                  transform=PiecewiseAffine,
                  trilist=training_images[0].landmarks['ibug_68_trimesh'].
                  lms.trilist,
                  normalization_diagonal=150,
                  n_levels=3,
                  downscale=2,
                  scaled_shape_models=False,
                  pyramid_on_features=False,
                  max_shape_components=[1, 2, 3],
                  max_appearance_components=[3, 3, 3],
                  boundary=3,
                  interpolator='scipy').build(training_images, group='PTS')

aam2 = AAMBuilder(feature_type=None,
                  transform=ThinPlateSplines,
                  trilist=None,
                  normalization_diagonal=None,
                  n_levels=2,
                  downscale=1.2,
                  scaled_shape_models=True,
                  pyramid_on_features=False,
                  max_shape_components=None,
                  max_appearance_components=10,
                  boundary=0,
                  interpolator='scipy').build(training_images, group='PTS')

aam3 = AAMBuilder(feature_type='igo',
                  transform=ThinPlateSplines,
                  trilist=None,
                  normalization_diagonal=None,
                  n_levels=1,
                  downscale=3,
                  scaled_shape_models=True,
                  pyramid_on_features=True,
                  max_shape_components=[1],
                  max_appearance_components=10,
                  boundary=2,
                  interpolator='scipy').build(training_images, group='PTS')

@raises(ValueError)
def test_feature_type_exception():
    aam = AAMBuilder(feature_type=['igo', sparse_hog],
                     pyramid_on_features=False).build(training_images,
                                                      group='PTS')

@raises(ValueError)
def test_feature_type_with_pyramid_on_features_exception():
    aam = AAMBuilder(feature_type=['igo', sparse_hog]).build(training_images,
                                                             group='PTS')

@raises(ValueError)
def test_n_levels_exception():
    aam = AAMBuilder(n_levels=0).build(training_images,
                                       group='PTS')

@raises(ValueError)
def test_downscale_exception():
    aam = AAMBuilder(downscale=1).build(training_images,
                                        group='PTS')
    assert (aam.downscale == 1)
    aam = AAMBuilder(downscale=0).build(training_images,
                                        group='PTS')

@raises(ValueError)
def test_normalization_diagonal_exception():
    aam = AAMBuilder(normalization_diagonal=100).build(training_images,
                                                       group='PTS')
    assert (aam.appearance_models[0].n_features == 378)
    aam = AAMBuilder(normalization_diagonal=10).build(training_images,
                                                      group='PTS')

@raises(ValueError)
def test_max_shape_components_exception():
    aam = AAMBuilder(max_shape_components=[1, 0.2, 'a']).build(training_images,
                                                               group='PTS')

@raises(ValueError)
def test_max_appearance_components_exception():
    aam = AAMBuilder(max_appearance_components=[1, 2]).build(training_images,
                                                             group='PTS')

@raises(ValueError)
def test_boundary_exception():
    aam = AAMBuilder(boundary=-1).build(training_images, group='PTS')

def test_aam_1():
    assert (aam1.n_training_images == 2)
    assert (aam1.n_levels == 3)
    assert (aam1.downscale == 2)
    assert (aam1.feature_type[0] == 'igo' and aam1.feature_type[2] is None)
    assert (aam1.interpolator == 'scipy')
    assert_allclose(np.around(aam1.reference_shape.range()), (110., 102.))
    assert (not aam1.scaled_shape_models)
    assert (not aam1.pyramid_on_features)
    assert (np.all([aam1.shape_models[j].n_components == 1
                    for j in range(aam1.n_levels)]))
    assert (np.all([aam1.appearance_models[j].n_components == 1
                    for j in range(aam1.n_levels)]))
    assert_allclose([aam1.appearance_models[j].template_instance.n_channels
                     for j in range(aam1.n_levels)], (2, 36, 1))
    assert_allclose([aam1.appearance_models[j].components.shape[1]
                     for j in range(aam1.n_levels)], (13892, 250056, 6946))

def test_aam_2():
    assert (aam2.n_training_images == 2)
    assert (aam2.n_levels == 2)
    assert (aam2.downscale == 1.2)
    assert (aam2.feature_type[0] is None and aam2.feature_type[1] is None)
    assert (aam2.interpolator == 'scipy')
    assert_allclose(np.around(aam2.reference_shape.range()), (224., 207.))
    assert aam2.scaled_shape_models
    assert (not aam2.pyramid_on_features)
    assert (np.all([aam2.shape_models[j].n_components == 1
                    for j in range(aam2.n_levels)]))
    assert (np.all([aam2.appearance_models[j].n_components == 1
                    for j in range(aam2.n_levels)]))
    assert (np.all([aam2.appearance_models[j].template_instance.n_channels == 1
                    for j in range(aam2.n_levels)]))
    assert_allclose([aam2.appearance_models[j].components.shape[1]
                     for j in range(aam2.n_levels)], (20447, 29443))

def test_aam_3():
    assert (aam3.n_training_images == 2)
    assert (aam3.n_levels == 1)
    assert (aam3.downscale == 3)
    assert (aam3.feature_type[0] is 'igo' and len(aam3.feature_type) == 1)
    assert (aam3.interpolator == 'scipy')
    assert_allclose(np.around(aam3.reference_shape.range()), (224., 207.))
    assert aam3.scaled_shape_models
    assert aam3.pyramid_on_features
    assert (np.all([aam3.shape_models[j].n_components == 1
                    for j in range(aam3.n_levels)]))
    assert (np.all([aam3.appearance_models[j].n_components == 1
                    for j in range(aam3.n_levels)]))
    assert (np.all([aam3.appearance_models[j].template_instance.n_channels == 2
                    for j in range(aam3.n_levels)]))
    assert_allclose([aam3.appearance_models[j].components.shape[1]
                     for j in range(aam3.n_levels)], (58886))
<commit_msg>adds patch based aam builder test<commit_after>import numpy as np
from numpy.testing import assert_allclose
from nose.tools import raises

import menpo.io as mio
from menpo.landmark import labeller, ibug_68_trimesh
from menpo.transform import PiecewiseAffine, ThinPlateSplines
from menpo.fitmultilevel.aam import AAMBuilder, PatchBasedAAMBuilder
from menpo.fitmultilevel.featurefunctions import sparse_hog

# load images
filenames = ['breakingbad.jpg', 'einstein.jpg']
training_images = []
for i in range(2):
    im = mio.import_builtin_asset(filenames[i])
    im.crop_to_landmarks_proportion(0.1)
    labeller(im, 'PTS', ibug_68_trimesh)
    if im.n_channels == 3:
        im = im.as_greyscale(mode='luminosity')
    training_images.append(im)

# build aams
aam1 = AAMBuilder(feature_type=['igo', sparse_hog, None],
                  transform=PiecewiseAffine,
                  trilist=training_images[0].landmarks['ibug_68_trimesh'].
                  lms.trilist,
                  normalization_diagonal=150,
                  n_levels=3,
                  downscale=2,
                  scaled_shape_models=False,
                  pyramid_on_features=False,
                  max_shape_components=[1, 2, 3],
                  max_appearance_components=[3, 3, 3],
                  boundary=3,
                  interpolator='scipy').build(training_images, group='PTS')

aam2 = AAMBuilder(feature_type=None,
                  transform=ThinPlateSplines,
                  trilist=None,
                  normalization_diagonal=None,
                  n_levels=2,
                  downscale=1.2,
                  scaled_shape_models=True,
                  pyramid_on_features=False,
                  max_shape_components=None,
                  max_appearance_components=10,
                  boundary=0,
                  interpolator='scipy').build(training_images, group='PTS')

aam3 = AAMBuilder(feature_type='igo',
                  transform=ThinPlateSplines,
                  trilist=None,
                  normalization_diagonal=None,
                  n_levels=1,
                  downscale=3,
                  scaled_shape_models=True,
                  pyramid_on_features=True,
                  max_shape_components=[1],
                  max_appearance_components=10,
                  boundary=2,
                  interpolator='scipy').build(training_images, group='PTS')

aam4 = PatchBasedAAMBuilder(feature_type='lbp',
                            patch_shape=(10, 13),
                            normalization_diagonal=200,
                            n_levels=2,
                            downscale=1.2,
                            scaled_shape_models=True,
                            pyramid_on_features=True,
                            max_shape_components=1,
                            max_appearance_components=None,
                            boundary=2,
                            interpolator='scipy').build(training_images,
                                                        group='PTS')

@raises(ValueError)
def test_feature_type_exception():
    aam = AAMBuilder(feature_type=['igo', sparse_hog],
                     pyramid_on_features=False).build(training_images,
                                                      group='PTS')

@raises(ValueError)
def test_feature_type_with_pyramid_on_features_exception():
    aam = AAMBuilder(feature_type=['igo', sparse_hog]).build(training_images,
                                                             group='PTS')

@raises(ValueError)
def test_n_levels_exception():
    aam = AAMBuilder(n_levels=0).build(training_images,
                                       group='PTS')

@raises(ValueError)
def test_downscale_exception():
    aam = AAMBuilder(downscale=1).build(training_images,
                                        group='PTS')
    assert (aam.downscale == 1)
    aam = AAMBuilder(downscale=0).build(training_images,
                                        group='PTS')

@raises(ValueError)
def test_normalization_diagonal_exception():
    aam = AAMBuilder(normalization_diagonal=100).build(training_images,
                                                       group='PTS')
    assert (aam.appearance_models[0].n_features == 378)
    aam = AAMBuilder(normalization_diagonal=10).build(training_images,
                                                      group='PTS')

@raises(ValueError)
def test_max_shape_components_exception():
    aam = AAMBuilder(max_shape_components=[1, 0.2, 'a']).build(training_images,
                                                               group='PTS')

@raises(ValueError)
def test_max_appearance_components_exception():
    aam = AAMBuilder(max_appearance_components=[1, 2]).build(training_images,
                                                             group='PTS')

@raises(ValueError)
def test_boundary_exception():
    aam = AAMBuilder(boundary=-1).build(training_images, group='PTS')

def test_aam_1():
    assert (aam1.n_training_images == 2)
    assert (aam1.n_levels == 3)
    assert (aam1.downscale == 2)
    assert (aam1.feature_type[0] == 'igo' and aam1.feature_type[2] is None)
    assert (aam1.interpolator == 'scipy')
    assert_allclose(np.around(aam1.reference_shape.range()), (110., 102.))
    assert (not aam1.scaled_shape_models)
    assert (not aam1.pyramid_on_features)
    assert (np.all([aam1.shape_models[j].n_components == 1
                    for j in range(aam1.n_levels)]))
    assert (np.all([aam1.appearance_models[j].n_components == 1
                    for j in range(aam1.n_levels)]))
    assert_allclose([aam1.appearance_models[j].template_instance.n_channels
                     for j in range(aam1.n_levels)], (2, 36, 1))
    assert_allclose([aam1.appearance_models[j].components.shape[1]
                     for j in range(aam1.n_levels)], (13892, 250056, 6946))

def test_aam_2():
    assert (aam2.n_training_images == 2)
    assert (aam2.n_levels == 2)
    assert (aam2.downscale == 1.2)
    assert (aam2.feature_type[0] is None and aam2.feature_type[1] is None)
    assert (aam2.interpolator == 'scipy')
    assert_allclose(np.around(aam2.reference_shape.range()), (224., 207.))
    assert aam2.scaled_shape_models
    assert (not aam2.pyramid_on_features)
    assert (np.all([aam2.shape_models[j].n_components == 1
                    for j in range(aam2.n_levels)]))
    assert (np.all([aam2.appearance_models[j].n_components == 1
                    for j in range(aam2.n_levels)]))
    assert (np.all([aam2.appearance_models[j].template_instance.n_channels == 1
                    for j in range(aam2.n_levels)]))
    assert_allclose([aam2.appearance_models[j].components.shape[1]
                     for j in range(aam2.n_levels)], (20447, 29443))

def test_aam_3():
    assert (aam3.n_training_images == 2)
    assert (aam3.n_levels == 1)
    assert (aam3.downscale == 3)
    assert (aam3.feature_type[0] is 'igo' and len(aam3.feature_type) == 1)
    assert (aam3.interpolator == 'scipy')
    assert_allclose(np.around(aam3.reference_shape.range()), (224., 207.))
    assert aam3.scaled_shape_models
    assert aam3.pyramid_on_features
    assert (np.all([aam3.shape_models[j].n_components == 1
                    for j in range(aam3.n_levels)]))
    assert (np.all([aam3.appearance_models[j].n_components == 1
                    for j in range(aam3.n_levels)]))
    assert (np.all([aam3.appearance_models[j].template_instance.n_channels == 2
                    for j in range(aam3.n_levels)]))
    assert_allclose([aam3.appearance_models[j].components.shape[1]
                     for j in range(aam3.n_levels)], 58886)

def test_aam_4():
    assert (aam4.n_training_images == 2)
    assert (aam4.n_levels == 2)
    assert (aam4.downscale == 1.2)
    assert (aam4.feature_type[0] is 'lbp')
    assert (aam4.interpolator == 'scipy')
    assert_allclose(np.around(aam4.reference_shape.range()), (147., 136.))
    assert aam4.scaled_shape_models
    assert aam4.pyramid_on_features
    assert (np.all([aam4.shape_models[j].n_components == 1
                    for j in range(aam4.n_levels)]))
    assert (np.all([aam4.appearance_models[j].n_components == 1
                    for j in range(aam4.n_levels)]))
    assert (np.all([aam4.appearance_models[j].template_instance.n_channels == 4
                    for j in range(aam4.n_levels)]))
    assert_allclose([aam4.appearance_models[j].components.shape[1]
                     for j in range(aam4.n_levels)], (22792, 25336))
<|endoftext|>


<commit_before>#!/usr/bin/env python
from djeasytests.testsetup import TestSetup, default_settings

default_settings.update(dict(
    ROOT_URLCONF='shop_example.urls',
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'shopexampledb.sqlite',
        }
    },
    INSTALLED_APPS = [
        'shop_example',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        'django.contrib.sites',
        'django.contrib.staticfiles',
        'shop',
        'shop_categories',
        'mptt',
        'treeadmin',
        'south',
        'tinymce'
    ],
    SHOP_PRODUCT_MODEL = 'shop_example.models.product.Product',
    SHOP_ADDRESS_MODEL = 'shop_example.models.address.Address',
    SHOP_CATEGORIES_CATEGORY_MODEL = 'shop_example.models.category.Category',
    SHOP_PAYMENT_BACKENDS = [
        'shop.payment.backends.pay_on_delivery.PayOnDeliveryBackend'
    ],
    SHOP_SHIPPING_BACKENDS = [
        'shop.shipping.backends.flat_rate.FlatRateShipping'
    ]
))

testsetup = TestSetup(appname='djnetaxept', default_settings=default_settings)

if __name__ == '__main__':
    testsetup.run('shell')<commit_msg>shop_example<commit_after>#!/usr/bin/env python
from djeasytests.testsetup import TestSetup, default_settings

default_settings.update(dict(
    ROOT_URLCONF='shop_example.urls',
    DATABASES = {
        'default': {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': 'shopexampledb.sqlite',
        }
    },
    INSTALLED_APPS = [
        'shop_example',
        'django.contrib.auth',
        'django.contrib.contenttypes',
        'django.contrib.sessions',
        'django.contrib.admin',
        'django.contrib.sites',
        'django.contrib.staticfiles',
        'shop',
        'shop_categories',
        'mptt',
        'treeadmin',
        'south',
        'tinymce'
    ],
    SHOP_PRODUCT_MODEL = 'shop_example.models.product.Product',
    SHOP_ADDRESS_MODEL = 'shop_example.models.address.Address',
    SHOP_CATEGORIES_CATEGORY_MODEL = 'shop_example.models.category.Category',
    SHOP_PAYMENT_BACKENDS = [
        'shop.payment.backends.pay_on_delivery.PayOnDeliveryBackend'
    ],
    SHOP_SHIPPING_BACKENDS = [
        'shop.shipping.backends.flat_rate.FlatRateShipping'
    ]
))

testsetup = TestSetup(appname='shop_example', default_settings=default_settings)

if __name__ == '__main__':
    testsetup.run('shell')<|endoftext|>


<commit_before>'''
The mixin code for ModelAdmin to link to the SchemaModel objects in models.py.
'''
from metashare.utils import verify_subclass, get_class_by_name
from metashare.repository.supermodel import SchemaModel
from metashare.repository.editor.editorutils import encode_as_inline
from django.db.models.fields import FieldDoesNotExist
from metashare.repository.editor.widgets import ComboWidget, MultiComboWidget
from metashare.repository.models import languageInfoType_model, metadataInfoType_model, \
documentInfoType_model, annotationInfoType_model

# inputInfoType_model, \
# outputInfoType_model,

# Fields that need the ComboWidget/MultiComboWidget with autocomplete functionality
# to use with languageId,languageName pairs.
LANGUAGE_ID_NAME_FIELDS = {
   # inputInfoType_model:
   #     {'type': 'multiple', 'id': "languageId", 'name': "languageName"},
   # outputInfoType_model:
   #     {'type': 'multiple', 'id': "languageId", 'name': "languageName"},
   languageInfoType_model:
       {'type': 'single', 'id': "languageId", 'name': "languageName"},
   metadataInfoType_model:
       {'type': 'multiple', 'id': "metadataLanguageId", 'name': "metadataLanguageName"},
   documentInfoType_model:
       {'type': 'single', 'id': "documentLanguageId", 'name': "documentLanguageName"},
   annotationInfoType_model:
       {'type': 'single', 'id': "tagsetLanguageId", 'name': "tagsetLanguageName"},
}

class SchemaModelLookup(object):
    show_tabbed_fieldsets = False

    def is_field(self, name):
        return not self.is_inline(name)

    def is_inline(self, name):
        return name.endswith('_set')

    def is_required_field(self, name):
        """
        Checks whether the field with the given name is a required field.
        """
        # pylint: disable-msg=E1101
        _fields = self.model.get_fields()
        return name in _fields['required']

    def is_visible_as_normal_field(self, field_name, exclusion_list):
        return self.is_field(field_name) and field_name not in exclusion_list

    def is_visible_as_inline(self, field_name, include_inlines, inlines):
        return include_inlines and (field_name in inlines or self.is_inline(field_name))

    def get_excluded_fields(self):
        # pylint: disable-msg=E1101
        if hasattr(self, 'exclude') and self.exclude is not None:
            return tuple(self.exclude)
        return ()

    def get_hidden_fields(self):
        # pylint: disable-msg=E1101
        if hasattr(self, 'hidden_fields') and self.hidden_fields is not None:
            return tuple(self.hidden_fields)
        return ()

    def get_non_editable_fields(self):
        # pylint: disable-msg=E1101
        _readonly = ()
        if hasattr(self, 'readonly_fields') and self.readonly_fields is not None:
            _readonly += tuple(self.readonly_fields)
        _fields = self.model.get_fields_flat()
        for _field_name in _fields:
            try:
                if not self.model._meta.get_field(_field_name).editable:
                    _readonly += (_field_name, )
            except FieldDoesNotExist:
                pass
        return _readonly

    def build_fieldsets_from_schema_plain(self, include_inlines=False, inlines=()):
        """
        Builds fieldsets using SchemaModel.get_fields().
        """
        # pylint: disable-msg=E1101
        verify_subclass(self.model, SchemaModel)

        exclusion_list = set(self.get_excluded_fields() + self.get_hidden_fields() + self.get_non_editable_fields())

        # pylint: disable-msg=E1101
        _fields = self.model.get_fields_flat()
        _visible_fields = []

        for _field_name in _fields:
            if self.is_visible_as_normal_field(_field_name, exclusion_list):
                _visible_fields.append(_field_name)
            elif self.is_visible_as_inline(_field_name, include_inlines, inlines):
                _visible_fields.append(encode_as_inline(_field_name))

        _fieldsets = ((None,
            {'fields': _visible_fields + list(self.get_hidden_fields())}
        ),)
        return _fieldsets

    def build_fieldsets_from_schema_tabbed(self, include_inlines=False, inlines=()):
        """
        Builds fieldsets using SchemaModel.get_fields(),
        for tabbed viewing (required/recommended/optional).
        """
        # pylint: disable-msg=E1101
        verify_subclass(self.model, SchemaModel)

        exclusion_list = set(self.get_excluded_fields() + self.get_hidden_fields() + self.get_non_editable_fields())

        _fieldsets = []
        # pylint: disable-msg=E1101
        _fields = self.model.get_fields()

        for _field_status in ('required', 'recommended', 'optional'):
            _visible_fields = []
            _visible_fields_verbose_names = []

            for _field_name in _fields[_field_status]:
                if self.is_visible_as_normal_field(_field_name, exclusion_list):
                    _visible_fields.append(_field_name)
                    is_visible = True
                elif self.is_visible_as_inline(_field_name, include_inlines, inlines):
                    _visible_fields.append(encode_as_inline(_field_name))
                    is_visible = True
                else:
                    is_visible = False

                if is_visible:
                    _visible_fields_verbose_names.append(self.model.get_verbose_name(_field_name))

            if len(_visible_fields) > 0:
                _detail = ', '.join(_visible_fields_verbose_names)
                _caption = '{0} information: {1}'.format(_field_status.capitalize(), _detail)
                _fieldset = {'fields': _visible_fields}
                _fieldsets.append((_caption, _fieldset))

        _hidden_fields = self.get_hidden_fields()
        if _hidden_fields:
            _fieldsets.append((None, {'fields': _hidden_fields, 'classes':('display_none', )}))

        return _fieldsets

    def build_fieldsets_from_schema(self, include_inlines=False, inlines=()):
        if self.show_tabbed_fieldsets:
            return self.build_fieldsets_from_schema_tabbed(include_inlines, inlines)
        return self.build_fieldsets_from_schema_plain(include_inlines, inlines)

    def get_fieldsets(self, request, obj=None):
        return self.build_fieldsets_from_schema()

    def get_fieldsets_with_inlines(self, request, obj=None):
        # pylint: disable-msg=E1101
        inline_names = [inline.parent_fk_name for inline in self.inline_instances if hasattr(inline, 'parent_fk_name')]
        return self.build_fieldsets_from_schema(include_inlines=True, inlines=inline_names)

    def get_inline_classes(self, model, status):
        '''
        For the inlines listed in the given SchemaModel's __schema_fields__,
        provide the list of matching inline classes.
        '''
        result = []
        parent_model_class_name = model._meta.object_name
        for rec_path, rec_accessor, rec_status in model.__schema_fields__:
            if status == rec_status and self.is_inline(rec_accessor):
                if '/' in rec_path:
                    slashpos = rec_path.rfind('/')
                    rec_name = rec_path[(slashpos+1):]
                else:
                    rec_name = rec_path
                # pylint: disable-msg=E1101
                if hasattr(self, 'custom_one2many_inlines') and rec_name in self.custom_one2many_inlines:
                    inline_class = self.custom_one2many_inlines[rec_name]
                else:
                    one_model_class_name = model.__schema_classes__[rec_name]
                    inline_class = self.get_inline_class_from_model_class_name(one_model_class_name, parent_model_class_name)
                result.append(inline_class)
        return result

    def get_inline_class_from_model_class_name(self, model_class_name, parent_model_class_name=None):
        '''
        For a model class object such as identificationInfoType_model,
        return the class object extending SchemaModelInline which
        can be used to render this model inline in the admin interface.
        Raises an AttributeError if no suitable class can be found.

        This expects the following naming convention:
        - Take the model class name and remove the suffix 'Type_model';
        - append '_model_inline'.
        '''
        suffix_length = len("Type_model")
        modelname_without_suffix = model_class_name[:-suffix_length]
        inline_class_name = modelname_without_suffix + "_model_inline"
        if parent_model_class_name:
            context_specific_name = inline_class_name + "_" + parent_model_class_name
            try:
                return get_class_by_name('metashare.repository.admin', context_specific_name)
            except AttributeError:
                pass
        return get_class_by_name('metashare.repository.admin', inline_class_name)

    def add_lang_widget(self, db_field):
        # pylint: disable-msg=E1101
        model_cls = self.model().__class__
        widget_dict = {}
        if model_cls in LANGUAGE_ID_NAME_FIELDS:
            item = LANGUAGE_ID_NAME_FIELDS[model_cls]
            if item['type'] == 'single':
                attrs = {}
                attrs['id_field'] = item['id']
                attrs['name_field'] = item['name']
                if db_field.name == item['id']:
                    widget_dict.update({'widget': ComboWidget(field_type='id', attrs=attrs)})
                elif db_field.name == item['name']:
                    widget_dict.update({'widget': ComboWidget(field_type='name', attrs=attrs)})
            elif item['type'] == 'multiple':
                attrs = {}
                attrs['id_field'] = item['id']
                attrs['name_field'] = item['name']
                if db_field.name == item['name']:
                    prev_widget = db_field.widget
                    widget_id = prev_widget.widget_id
                    max_length = prev_widget.max_length
                    widget_dict.update({'widget': MultiComboWidget(field_type='name', attrs=attrs, widget_id=widget_id, max_length=max_length)})
                elif db_field.name == item['id']:
                    prev_widget = db_field.widget
                    widget_id = prev_widget.widget_id
                    max_length = prev_widget.max_length
                    widget_dict.update({'widget': MultiComboWidget(field_type='id', attrs=attrs, widget_id=widget_id, max_length=max_length)})

        return widget_dict

    def add_lang_templ_params(self, inline_admin_formset):
        model_cls = inline_admin_formset.formset.form.Meta.model().__class__
        if model_cls in LANGUAGE_ID_NAME_FIELDS:
            item = LANGUAGE_ID_NAME_FIELDS[model_cls]
            inline_admin_formset.has_lang = True
            inline_admin_formset.lang_id = item['id']
            inline_admin_formset.lang_name = item['name']
<commit_msg>change language selection to dropdown list<commit_after>'''
The mixin code for ModelAdmin to link to the SchemaModel objects in models.py.
'''
from metashare.utils import verify_subclass, get_class_by_name
from metashare.repository.supermodel import SchemaModel
from metashare.repository.editor.editorutils import encode_as_inline
from django.db.models.fields import FieldDoesNotExist
from metashare.repository.editor.widgets import ComboWidget, MultiComboWidget
from metashare.repository.models import languageInfoType_model, metadataInfoType_model, \
documentInfoType_model, annotationInfoType_model

# inputInfoType_model, \
# outputInfoType_model,

# Fields that need the ComboWidget/MultiComboWidget with autocomplete functionality
# to use with languageId,languageName pairs.
LANGUAGE_ID_NAME_FIELDS = {
   # inputInfoType_model:
   #     {'type': 'multiple', 'id': "languageId", 'name': "languageName"},
   # outputInfoType_model:
   #     {'type': 'multiple', 'id': "languageId", 'name': "languageName"},
   # languageInfoType_model:
   #     {'type': 'single', 'id': "languageId", 'name': "languageName"},
   metadataInfoType_model:
       {'type': 'multiple', 'id': "metadataLanguageId", 'name': "metadataLanguageName"},
   documentInfoType_model:
       {'type': 'single', 'id': "documentLanguageId", 'name': "documentLanguageName"},
   annotationInfoType_model:
       {'type': 'single', 'id': "tagsetLanguageId", 'name': "tagsetLanguageName"},
}

class SchemaModelLookup(object):
    show_tabbed_fieldsets = False

    def is_field(self, name):
        return not self.is_inline(name)

    def is_inline(self, name):
        return name.endswith('_set')

    def is_required_field(self, name):
        """
        Checks whether the field with the given name is a required field.
        """
        # pylint: disable-msg=E1101
        _fields = self.model.get_fields()
        return name in _fields['required']

    def is_visible_as_normal_field(self, field_name, exclusion_list):
        return self.is_field(field_name) and field_name not in exclusion_list

    def is_visible_as_inline(self, field_name, include_inlines, inlines):
        return include_inlines and (field_name in inlines or self.is_inline(field_name))

    def get_excluded_fields(self):
        # pylint: disable-msg=E1101
        if hasattr(self, 'exclude') and self.exclude is not None:
            return tuple(self.exclude)
        return ()

    def get_hidden_fields(self):
        # pylint: disable-msg=E1101
        if hasattr(self, 'hidden_fields') and self.hidden_fields is not None:
            return tuple(self.hidden_fields)
        return ()

    def get_non_editable_fields(self):
        # pylint: disable-msg=E1101
        _readonly = ()
        if hasattr(self, 'readonly_fields') and self.readonly_fields is not None:
            _readonly += tuple(self.readonly_fields)
        _fields = self.model.get_fields_flat()
        for _field_name in _fields:
            try:
                if not self.model._meta.get_field(_field_name).editable:
                    _readonly += (_field_name, )
            except FieldDoesNotExist:
                pass
        return _readonly

    def build_fieldsets_from_schema_plain(self, include_inlines=False, inlines=()):
        """
        Builds fieldsets using SchemaModel.get_fields().
        """
        # pylint: disable-msg=E1101
        verify_subclass(self.model, SchemaModel)

        exclusion_list = set(self.get_excluded_fields() + self.get_hidden_fields() + self.get_non_editable_fields())

        # pylint: disable-msg=E1101
        _fields = self.model.get_fields_flat()
        _visible_fields = []

        for _field_name in _fields:
            if self.is_visible_as_normal_field(_field_name, exclusion_list):
                _visible_fields.append(_field_name)
            elif self.is_visible_as_inline(_field_name, include_inlines, inlines):
                _visible_fields.append(encode_as_inline(_field_name))

        _fieldsets = ((None,
            {'fields': _visible_fields + list(self.get_hidden_fields())}
        ),)
        return _fieldsets

    def build_fieldsets_from_schema_tabbed(self, include_inlines=False, inlines=()):
        """
        Builds fieldsets using SchemaModel.get_fields(),
        for tabbed viewing (required/recommended/optional).
        """
        # pylint: disable-msg=E1101
        verify_subclass(self.model, SchemaModel)

        exclusion_list = set(self.get_excluded_fields() + self.get_hidden_fields() + self.get_non_editable_fields())

        _fieldsets = []
        # pylint: disable-msg=E1101
        _fields = self.model.get_fields()

        for _field_status in ('required', 'recommended', 'optional'):
            _visible_fields = []
            _visible_fields_verbose_names = []

            for _field_name in _fields[_field_status]:
                if self.is_visible_as_normal_field(_field_name, exclusion_list):
                    _visible_fields.append(_field_name)
                    is_visible = True
                elif self.is_visible_as_inline(_field_name, include_inlines, inlines):
                    _visible_fields.append(encode_as_inline(_field_name))
                    is_visible = True
                else:
                    is_visible = False

                if is_visible:
                    _visible_fields_verbose_names.append(self.model.get_verbose_name(_field_name))

            if len(_visible_fields) > 0:
                _detail = ', '.join(_visible_fields_verbose_names)
                _caption = '{0} information: {1}'.format(_field_status.capitalize(), _detail)
                _fieldset = {'fields': _visible_fields}
                _fieldsets.append((_caption, _fieldset))

        _hidden_fields = self.get_hidden_fields()
        if _hidden_fields:
            _fieldsets.append((None, {'fields': _hidden_fields, 'classes':('display_none', )}))

        return _fieldsets

    def build_fieldsets_from_schema(self, include_inlines=False, inlines=()):
        if self.show_tabbed_fieldsets:
            return self.build_fieldsets_from_schema_tabbed(include_inlines, inlines)
        return self.build_fieldsets_from_schema_plain(include_inlines, inlines)

    def get_fieldsets(self, request, obj=None):
        return self.build_fieldsets_from_schema()

    def get_fieldsets_with_inlines(self, request, obj=None):
        # pylint: disable-msg=E1101
        inline_names = [inline.parent_fk_name for inline in self.inline_instances if hasattr(inline, 'parent_fk_name')]
        return self.build_fieldsets_from_schema(include_inlines=True, inlines=inline_names)

    def get_inline_classes(self, model, status):
        '''
        For the inlines listed in the given SchemaModel's __schema_fields__,
        provide the list of matching inline classes.
        '''
        result = []
        parent_model_class_name = model._meta.object_name
        for rec_path, rec_accessor, rec_status in model.__schema_fields__:
            if status == rec_status and self.is_inline(rec_accessor):
                if '/' in rec_path:
                    slashpos = rec_path.rfind('/')
                    rec_name = rec_path[(slashpos+1):]
                else:
                    rec_name = rec_path
                # pylint: disable-msg=E1101
                if hasattr(self, 'custom_one2many_inlines') and rec_name in self.custom_one2many_inlines:
                    inline_class = self.custom_one2many_inlines[rec_name]
                else:
                    one_model_class_name = model.__schema_classes__[rec_name]
                    inline_class = self.get_inline_class_from_model_class_name(one_model_class_name, parent_model_class_name)
                result.append(inline_class)
        return result

    def get_inline_class_from_model_class_name(self, model_class_name, parent_model_class_name=None):
        '''
        For a model class object such as identificationInfoType_model,
        return the class object extending SchemaModelInline which
        can be used to render this model inline in the admin interface.
        Raises an AttributeError if no suitable class can be found.

        This expects the following naming convention:
        - Take the model class name and remove the suffix 'Type_model';
        - append '_model_inline'.
        '''
        suffix_length = len("Type_model")
        modelname_without_suffix = model_class_name[:-suffix_length]
        inline_class_name = modelname_without_suffix + "_model_inline"
        if parent_model_class_name:
            context_specific_name = inline_class_name + "_" + parent_model_class_name
            try:
                return get_class_by_name('metashare.repository.admin', context_specific_name)
            except AttributeError:
                pass
        return get_class_by_name('metashare.repository.admin', inline_class_name)

    def add_lang_widget(self, db_field):
        # pylint: disable-msg=E1101
        model_cls = self.model().__class__
        widget_dict = {}
        if model_cls in LANGUAGE_ID_NAME_FIELDS:
            item = LANGUAGE_ID_NAME_FIELDS[model_cls]
            if item['type'] == 'single':
                attrs = {}
                attrs['id_field'] = item['id']
                attrs['name_field'] = item['name']
                if db_field.name == item['id']:
                    widget_dict.update({'widget': ComboWidget(field_type='id', attrs=attrs)})
                elif db_field.name == item['name']:
                    widget_dict.update({'widget': ComboWidget(field_type='name', attrs=attrs)})
            elif item['type'] == 'multiple':
                attrs = {}
                attrs['id_field'] = item['id']
                attrs['name_field'] = item['name']
                if db_field.name == item['name']:
                    prev_widget = db_field.widget
                    widget_id = prev_widget.widget_id
                    max_length = prev_widget.max_length
                    widget_dict.update({'widget': MultiComboWidget(field_type='name', attrs=attrs, widget_id=widget_id, max_length=max_length)})
                elif db_field.name == item['id']:
                    prev_widget = db_field.widget
                    widget_id = prev_widget.widget_id
                    max_length = prev_widget.max_length
                    widget_dict.update({'widget': MultiComboWidget(field_type='id', attrs=attrs, widget_id=widget_id, max_length=max_length)})

        return widget_dict

    def add_lang_templ_params(self, inline_admin_formset):
        model_cls = inline_admin_formset.formset.form.Meta.model().__class__
        if model_cls in LANGUAGE_ID_NAME_FIELDS:
            item = LANGUAGE_ID_NAME_FIELDS[model_cls]
            inline_admin_formset.has_lang = True
            inline_admin_formset.lang_id = item['id']
            inline_admin_formset.lang_name = item['name']
<|endoftext|>


<commit_before>from .base import SprintMetric
from ..settings import conf
import json

class DailyUserStoryThroughput(SprintMetric):
	def _calculate_score(self, sprint=conf.sprints[-1]):
		SCORE_COLUMN = 'USperDev'
		UPPER_BOUND = 10
		LOWER_BOUND = 0.2

		if isinstance(self.results, str):
			results = json.loads(self.results)
			print('GOT A STRING, WANTED A DICT')
		else:
			results = self.results
		results = results[sprint]

		score_index = results['columns'].index(SCORE_COLUMN)
		value = results['rows'][0][score_index]

		if value > UPPER_BOUND or value < LOWER_BOUND:
			return 25
		r = 80 + (value*5)
		return r if r<=100 else 100
<commit_msg>Added team to daily user story throughput<commit_after>from .base import SprintMetric
from ..settings import conf
import json

class DailyUserStoryThroughput(SprintMetric):
	def _calculate_score(self, sprint, team):
		SCORE_COLUMN = 'USperDev'
		UPPER_BOUND = 10
		LOWER_BOUND = 0.2

		value = self.get_value(sprint, team, SCORE_COLUMN)
		if value is None:
			value = 0

		if value > UPPER_BOUND or value < LOWER_BOUND:
			return 25
		r = 80 + (value*5)
		return r if r<=100 else 100
<|endoftext|>