<commit_before>from wq.db.patterns import models
import swapper
from django.db.models.signals import post_save
from django.core.exceptions import ImproperlyConfigured
from django.dispatch import receiver
from django.conf import settings
from collections import OrderedDict
from .compat import clone_field

swapper.set_app_prefix('vera', 'WQ')
MODELS = swapper.get_model_names(
    'vera', ('Site', 'Event', 'Report', 'ReportStatus', 'Parameter', 'Result')
)

VALID_REPORT_ORDER = getattr(settings, "WQ_VALID_REPORT_ORDER", ('-entered',))
# Base classes for Site-Event-Report-Attribute-Value pattern
# Extend these when swapping out default implementation (below)

class BaseSite(models.NaturalKeyModel):
    @property
    def valid_events(self):
        events = self.event_set.filter(
            report__status__is_valid=True
        ).values_list('pk', flat=True)
        # FIXME: events may be duplicated
        return self.event_set.filter(pk__in=events)

    class Meta:
        abstract = True

class BaseEvent(models.NaturalKeyModel):
    site = models.ForeignKey(MODELS['Site'], null=True, blank=True)

    @property
    def valid_reports(self):
        return self.report_set.filter(status__is_valid=True).order_by(
            *VALID_REPORT_ORDER
        )

    @property
    def vals(self):
        return OrderedDict([
            (a.type.natural_key()[0], a.value)
            for a in self.results
        ])

    # Valid results for this event
    @property
    def results(self):
        Result = swapper.load_model('vera', 'Result')
        Parameter = swapper.load_model('vera', 'Parameter')
        # ORDER BY parameter, then valid report order
        order = (nest_ordering('type', Parameter._meta.ordering)
                 + ['type__id']
                 + nest_ordering('report', VALID_REPORT_ORDER))
        # DISTINCT ON parameter, collapsing multiple reports into one
        distinct = (nest_ordering('type', Parameter._meta.ordering, True)
                    + ['type__id'])
        results = Result.objects.filter(
            report__in=self.valid_reports,
            empty=False
        ).order_by(*order).distinct(*distinct)
        return results

    @property
    def is_valid(self):
        return self.valid_reports.count() > 0

    class Meta:
        abstract = True

class ReportManager(models.RelatedModelManager):
    def create_report(self, event_key, values, **kwargs):
        Event = swapper.load_model('vera', 'Event')
        kwargs['event'] = Event.objects.find(*event_key)
        report = self.create(**kwargs)
        report.vals = values
        return report

class ValidReportManager(ReportManager):
    def get_queryset(self):
        qs = super(ValidReportManager, self)
        return qs.filter(status__is_valid=True).order_by(*VALID_REPORT_ORDER)

class BaseReport(models.RelatedModel):
    event = models.ForeignKey(MODELS['Event'])
    entered = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL, null=True, blank=True,

        # Needed for model swapping in tests
        related_name="%(app_label)s_%(class)s",
    )
    status = models.ForeignKey(MODELS['ReportStatus'], null=True, blank=True)

    objects = ReportManager()
    valid_objects = ValidReportManager()

    @property
    def is_valid(self):
        return self.status and self.status.is_valid

    @property
    def vals(self):
        vals = OrderedDict()
        for result in self.results.all():
            vals[result.type.natural_key()[0]] = result.value
        return vals

    @vals.setter
    def vals(self, vals):
        Parameter = swapper.load_model('vera', 'Parameter')
        keys = [(key,) for key in vals.keys()]
        params, success = Parameter.objects.resolve_keys(keys)
        if not success:
            missing = [
                name for name, params in params.items() if params is None
            ]
            raise TypeError(
                "Could not identify one or more parameters: %s!"
                % missing
            )

        for key, param in params.items():
            result, is_new = self.results.get_or_create(type=param)
            result.value = vals[key[0]]
            result.save()

    def __unicode__(self):
        if self.pk is not None:
            return "%s according to %s" % (self.event, self.user)
        else:
            return "New Report"

    class Meta:
        abstract = True
        ordering = VALID_REPORT_ORDER

class BaseReportStatus(models.NaturalKeyModel):
    slug = models.SlugField()
    name = models.CharField(max_length=255)
    is_valid = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        abstract = True
        unique_together = [['slug']]

class BaseParameter(models.IdentifiedRelatedModel):
    name = models.CharField(max_length=255)
    is_numeric = models.BooleanField(default=False)
    units = models.CharField(max_length=50, null=True, blank=True)

    def __unicode__(self):
        if self.units:
            return u"%s (%s)" % (self.name, self.units)
        else:
            return self.name

    class Meta:
        abstract = True
        ordering = ('name',)

class BaseResult(models.Model):
    type = models.ForeignKey(MODELS['Parameter'])
    report = models.ForeignKey(MODELS['Report'], related_name='results')
    value_numeric = models.FloatField(null=True, blank=True)
    value_text = models.TextField(null=True, blank=True)
    empty = models.BooleanField(default=False, db_index=True)

    def is_empty(self, value):
        if value is None:
            return True
        if isinstance(value, basestring) and len(value.strip()) == 0:
            return True
        return False

    @property
    def value(self):
        if self.type.is_numeric:
            return self.value_numeric
        return self.value_text

    @value.setter
    def value(self, val):
        self.empty = self.is_empty(val)
        if self.type.is_numeric:
            if self.empty:
                self.value_numeric = None
            else:
                self.value_numeric = val
        else:
            self.value_text = val

    def __unicode__(self):
        return "%s -> %s: %s" % (self.report, self.type, self.value)

    class Meta:
        abstract = True
        ordering = ('type',)
        index_together = [
            ('type', 'report', 'empty'),
        ]

# Denormalize Events with Results from valid Reports
class EventResultManager(models.Manager):
    def set_for_event(self, event, delete=True):
        self.filter(event=event).delete()
        for result in event.results:
            er = self.model(
                event=event,
                result=result
            )
            er.save()

    def set_for_events(self, events, delete=True):
        for event in events:
            self.set_for_event(event, delete)

    def set_all(self):
        self.all().delete()
        Event = swapper.load_model('vera', 'Event')
        self.set_for_events(Event.objects.all(), delete=False)

class BaseEventResult(models.Model):
    """
    Denormalized event-result pairs (for valid reports)
    """
    id = models.PositiveIntegerField(primary_key=True)
    event = models.ForeignKey(MODELS['Event'])
    result = models.ForeignKey(MODELS['Result'])
    objects = EventResultManager()

    def __unicode__(self):
        if self.result_value_numeric is not None:
            value = self.result_value_numeric
        else:
            value = self.result_value_text
        return "%s -> %s: %s" % (
            self.event,
            self.result_type,
            value
        )

    def save(self, *args, **kwargs):
        """
        Denormalize all properties from the event and the result.
        """
        if self.event_id is None or self.result_id is None:
            return
        self.pk = self.result.pk

        def set_value(src, name):
            if field.primary_key:
                return
            obj = getattr(self, src)
            setattr(self, src + '_' + name, getattr(obj, name))

        for field in self.event._meta.fields:
            set_value('event', field.name)
        for field in self.result._meta.fields:
            set_value('result', field.name)
        super(BaseEventResult, self).save(*args, **kwargs)

    class Meta:
        abstract = True

def create_eventresult_model(event_cls, result_cls,
                             base=BaseEventResult, swappable=False):
    """
    **Here be magic**

    EventResult needs to have all of the fields from Event and Result.
    In order to DRY (and since either of these classes may be swapped), we
    add the fields dynamically here.  If neither Event or Result are swapped,
    this function will be called below, otherwise the user should call
    this function in their models.py.
    """

    app = 'vera'
    module = base.__module__
    for cls in event_cls, result_cls, base:
        if cls._meta.app_label != 'vera':
            app = cls._meta.app_label
            module = cls.__module__

    class Meta(base.Meta):
        db_table = 'wq_eventresult'
        unique_together = ('event', 'result_type')
        app_label = app

    if swappable:
        Meta.swappable = swappable

    attrs = {
        'Meta': Meta,
        '__module__': module
    }

    def add_field(prefix, field):
        if field.primary_key:
            return
        attrs[prefix + '_' + field.name] = clone_field(field)

    for field in event_cls._meta.fields:
        add_field('event', field)

    for field in result_cls._meta.fields:
        add_field('result', field)

    EventResult = type(
        'EventResult',
        (base,),
        attrs
    )

    # Disconnect any existing receiver
    post_save.disconnect(dispatch_uid="eventresult_receiver")

    @receiver(post_save, weak=False, dispatch_uid="eventresult_receiver")
    def handler(sender, instance=None, **kwargs):
        events = find_events(instance)
        if events:
            EventResult.objects.set_for_events(events)

    return EventResult

# Default implementation of the above classes, can be swapped
class Site(BaseSite):
    latitude = models.FloatField(null=True, blank=True)
    longitude = models.FloatField(null=True, blank=True)

    def __unicode__(self):
        return "%s, %s" % (round(self.latitude, 3), round(self.longitude, 3))

    class Meta(BaseSite.Meta):
        db_table = 'wq_site'
        swappable = swapper.swappable_setting('vera', 'Site')
        unique_together = ('latitude', 'longitude')

class Event(BaseEvent):
    date = models.DateField()

    def __unicode__(self):
        return "%s on %s" % (self.site, self.date)

    class Meta(BaseEvent.Meta):
        db_table = 'wq_event'
        swappable = swapper.swappable_setting('vera', 'Event')
        unique_together = ('site', 'date')
        ordering = ('-date',)

class Report(BaseReport):
    class Meta(BaseReport.Meta):
        db_table = 'wq_report'
        swappable = swapper.swappable_setting('vera', 'Report')

class ReportStatus(BaseReportStatus):
    class Meta(BaseReportStatus.Meta):
        verbose_name_plural = 'report statuses'
        db_table = 'wq_reportstatus'
        swappable = swapper.swappable_setting('vera', 'ReportStatus')

class Parameter(BaseParameter):
    class Meta(BaseParameter.Meta):
        db_table = 'wq_parameter'
        swappable = swapper.swappable_setting('vera', 'Parameter')

class Result(BaseResult):
    class Meta(BaseResult.Meta):
        db_table = 'wq_result'
        swappable = swapper.swappable_setting('vera', 'Result')

EventResult = create_eventresult_model(
    Event, Result, swappable=swapper.swappable_setting('vera', 'EventResult')
)

if ((swapper.is_swapped('vera', 'Event')
        or swapper.is_swapped('vera', 'Result'))
        and not swapper.is_swapped('vera', 'EventResult')):
    raise ImproperlyConfigured(
        "Event or Result was swapped but EventResult was not!"
    )

def nest_ordering(prefix, ordering, ignore_reverse=False):
    nest_order = []
    for field in ordering:
        reverse = ''
        if field.startswith('-'):
            field = field[1:]
            if not ignore_reverse:
                reverse = '-'
        nest_order.append(reverse + prefix + '__' + field)
    return nest_order

def find_events(instance):
    if isinstance(instance, BaseEvent):
        return [instance]
    if isinstance(instance, BaseReport):
        return [instance.event]
    if isinstance(instance, BaseResult):
        return [instance.report.event]
    return None
<commit_msg>make EventResult.result_value behave like Result.value<commit_after>from wq.db.patterns import models
import swapper
from django.db.models.signals import post_save
from django.core.exceptions import ImproperlyConfigured
from django.dispatch import receiver
from django.conf import settings
from collections import OrderedDict
from .compat import clone_field

swapper.set_app_prefix('vera', 'WQ')
MODELS = swapper.get_model_names(
    'vera', ('Site', 'Event', 'Report', 'ReportStatus', 'Parameter', 'Result')
)

VALID_REPORT_ORDER = getattr(settings, "WQ_VALID_REPORT_ORDER", ('-entered',))
# Base classes for Site-Event-Report-Attribute-Value pattern
# Extend these when swapping out default implementation (below)

class BaseSite(models.NaturalKeyModel):
    @property
    def valid_events(self):
        events = self.event_set.filter(
            report__status__is_valid=True
        ).values_list('pk', flat=True)
        # FIXME: events may be duplicated
        return self.event_set.filter(pk__in=events)

    class Meta:
        abstract = True

class BaseEvent(models.NaturalKeyModel):
    site = models.ForeignKey(MODELS['Site'], null=True, blank=True)

    @property
    def valid_reports(self):
        return self.report_set.filter(status__is_valid=True).order_by(
            *VALID_REPORT_ORDER
        )

    @property
    def vals(self):
        return OrderedDict([
            (a.type.natural_key()[0], a.value)
            for a in self.results
        ])

    # Valid results for this event
    @property
    def results(self):
        Result = swapper.load_model('vera', 'Result')
        Parameter = swapper.load_model('vera', 'Parameter')
        # ORDER BY parameter, then valid report order
        order = (nest_ordering('type', Parameter._meta.ordering)
                 + ['type__id']
                 + nest_ordering('report', VALID_REPORT_ORDER))
        # DISTINCT ON parameter, collapsing multiple reports into one
        distinct = (nest_ordering('type', Parameter._meta.ordering, True)
                    + ['type__id'])
        results = Result.objects.filter(
            report__in=self.valid_reports,
            empty=False
        ).order_by(*order).distinct(*distinct)
        return results

    @property
    def is_valid(self):
        return self.valid_reports.count() > 0

    class Meta:
        abstract = True

class ReportManager(models.RelatedModelManager):
    def create_report(self, event_key, values, **kwargs):
        Event = swapper.load_model('vera', 'Event')
        kwargs['event'] = Event.objects.find(*event_key)
        report = self.create(**kwargs)
        report.vals = values
        return report

class ValidReportManager(ReportManager):
    def get_queryset(self):
        qs = super(ValidReportManager, self)
        return qs.filter(status__is_valid=True).order_by(*VALID_REPORT_ORDER)

class BaseReport(models.RelatedModel):
    event = models.ForeignKey(MODELS['Event'])
    entered = models.DateTimeField(auto_now_add=True)
    user = models.ForeignKey(
        settings.AUTH_USER_MODEL, null=True, blank=True,

        # Needed for model swapping in tests
        related_name="%(app_label)s_%(class)s",
    )
    status = models.ForeignKey(MODELS['ReportStatus'], null=True, blank=True)

    objects = ReportManager()
    valid_objects = ValidReportManager()

    @property
    def is_valid(self):
        return self.status and self.status.is_valid

    @property
    def vals(self):
        vals = OrderedDict()
        for result in self.results.all():
            vals[result.type.natural_key()[0]] = result.value
        return vals

    @vals.setter
    def vals(self, vals):
        Parameter = swapper.load_model('vera', 'Parameter')
        keys = [(key,) for key in vals.keys()]
        params, success = Parameter.objects.resolve_keys(keys)
        if not success:
            missing = [
                name for name, params in params.items() if params is None
            ]
            raise TypeError(
                "Could not identify one or more parameters: %s!"
                % missing
            )

        for key, param in params.items():
            result, is_new = self.results.get_or_create(type=param)
            result.value = vals[key[0]]
            result.save()

    def __unicode__(self):
        if self.pk is not None:
            return "%s according to %s" % (self.event, self.user)
        else:
            return "New Report"

    class Meta:
        abstract = True
        ordering = VALID_REPORT_ORDER

class BaseReportStatus(models.NaturalKeyModel):
    slug = models.SlugField()
    name = models.CharField(max_length=255)
    is_valid = models.BooleanField(default=False)

    def __unicode__(self):
        return self.name

    class Meta:
        abstract = True
        unique_together = [['slug']]

class BaseParameter(models.IdentifiedRelatedModel):
    name = models.CharField(max_length=255)
    is_numeric = models.BooleanField(default=False)
    units = models.CharField(max_length=50, null=True, blank=True)

    def __unicode__(self):
        if self.units:
            return u"%s (%s)" % (self.name, self.units)
        else:
            return self.name

    class Meta:
        abstract = True
        ordering = ('name',)

class BaseResult(models.Model):
    type = models.ForeignKey(MODELS['Parameter'])
    report = models.ForeignKey(MODELS['Report'], related_name='results')
    value_numeric = models.FloatField(null=True, blank=True)
    value_text = models.TextField(null=True, blank=True)
    empty = models.BooleanField(default=False, db_index=True)

    def is_empty(self, value):
        if value is None:
            return True
        if isinstance(value, basestring) and len(value.strip()) == 0:
            return True
        return False

    @property
    def value(self):
        if self.type.is_numeric:
            return self.value_numeric
        return self.value_text

    @value.setter
    def value(self, val):
        self.empty = self.is_empty(val)
        if self.type.is_numeric:
            if self.empty:
                self.value_numeric = None
            else:
                self.value_numeric = val
        else:
            self.value_text = val

    def __unicode__(self):
        return "%s -> %s: %s" % (self.report, self.type, self.value)

    class Meta:
        abstract = True
        ordering = ('type',)
        index_together = [
            ('type', 'report', 'empty'),
        ]

# Denormalize Events with Results from valid Reports
class EventResultManager(models.Manager):
    def set_for_event(self, event, delete=True):
        self.filter(event=event).delete()
        for result in event.results:
            er = self.model(
                event=event,
                result=result
            )
            er.save()

    def set_for_events(self, events, delete=True):
        for event in events:
            self.set_for_event(event, delete)

    def set_all(self):
        self.all().delete()
        Event = swapper.load_model('vera', 'Event')
        self.set_for_events(Event.objects.all(), delete=False)

class BaseEventResult(models.Model):
    """
    Denormalized event-result pairs (for valid reports)
    """
    id = models.PositiveIntegerField(primary_key=True)
    event = models.ForeignKey(MODELS['Event'])
    result = models.ForeignKey(MODELS['Result'])
    objects = EventResultManager()

    def result_value(self):
        if self.result_type.is_numeric:
            return self.result_value_numeric
        return self.result_value_text

    def __unicode__(self):
        return "%s -> %s: %s" % (
            self.event,
            self.result_type,
            self.result_value
        )

    def save(self, *args, **kwargs):
        """
        Denormalize all properties from the event and the result.
        """
        if self.event_id is None or self.result_id is None:
            return
        self.pk = self.result.pk

        def set_value(src, name):
            if field.primary_key:
                return
            obj = getattr(self, src)
            setattr(self, src + '_' + name, getattr(obj, name))

        for field in self.event._meta.fields:
            set_value('event', field.name)
        for field in self.result._meta.fields:
            set_value('result', field.name)
        super(BaseEventResult, self).save(*args, **kwargs)

    class Meta:
        abstract = True

def create_eventresult_model(event_cls, result_cls,
                             base=BaseEventResult, swappable=False):
    """
    **Here be magic**

    EventResult needs to have all of the fields from Event and Result.
    In order to DRY (and since either of these classes may be swapped), we
    add the fields dynamically here.  If neither Event or Result are swapped,
    this function will be called below, otherwise the user should call
    this function in their models.py.
    """

    app = 'vera'
    module = base.__module__
    for cls in event_cls, result_cls, base:
        if cls._meta.app_label != 'vera':
            app = cls._meta.app_label
            module = cls.__module__

    class Meta(base.Meta):
        db_table = 'wq_eventresult'
        unique_together = ('event', 'result_type')
        app_label = app

    if swappable:
        Meta.swappable = swappable

    attrs = {
        'Meta': Meta,
        '__module__': module
    }

    def add_field(prefix, field):
        if field.primary_key:
            return
        attrs[prefix + '_' + field.name] = clone_field(field)

    for field in event_cls._meta.fields:
        add_field('event', field)

    for field in result_cls._meta.fields:
        add_field('result', field)

    EventResult = type(
        'EventResult',
        (base,),
        attrs
    )

    # Disconnect any existing receiver
    post_save.disconnect(dispatch_uid="eventresult_receiver")

    @receiver(post_save, weak=False, dispatch_uid="eventresult_receiver")
    def handler(sender, instance=None, **kwargs):
        events = find_events(instance)
        if events:
            EventResult.objects.set_for_events(events)

    return EventResult

# Default implementation of the above classes, can be swapped
class Site(BaseSite):
    latitude = models.FloatField(null=True, blank=True)
    longitude = models.FloatField(null=True, blank=True)

    def __unicode__(self):
        return "%s, %s" % (round(self.latitude, 3), round(self.longitude, 3))

    class Meta(BaseSite.Meta):
        db_table = 'wq_site'
        swappable = swapper.swappable_setting('vera', 'Site')
        unique_together = ('latitude', 'longitude')

class Event(BaseEvent):
    date = models.DateField()

    def __unicode__(self):
        return "%s on %s" % (self.site, self.date)

    class Meta(BaseEvent.Meta):
        db_table = 'wq_event'
        swappable = swapper.swappable_setting('vera', 'Event')
        unique_together = ('site', 'date')
        ordering = ('-date',)

class Report(BaseReport):
    class Meta(BaseReport.Meta):
        db_table = 'wq_report'
        swappable = swapper.swappable_setting('vera', 'Report')

class ReportStatus(BaseReportStatus):
    class Meta(BaseReportStatus.Meta):
        verbose_name_plural = 'report statuses'
        db_table = 'wq_reportstatus'
        swappable = swapper.swappable_setting('vera', 'ReportStatus')

class Parameter(BaseParameter):
    class Meta(BaseParameter.Meta):
        db_table = 'wq_parameter'
        swappable = swapper.swappable_setting('vera', 'Parameter')

class Result(BaseResult):
    class Meta(BaseResult.Meta):
        db_table = 'wq_result'
        swappable = swapper.swappable_setting('vera', 'Result')

EventResult = create_eventresult_model(
    Event, Result, swappable=swapper.swappable_setting('vera', 'EventResult')
)

if ((swapper.is_swapped('vera', 'Event')
        or swapper.is_swapped('vera', 'Result'))
        and not swapper.is_swapped('vera', 'EventResult')):
    raise ImproperlyConfigured(
        "Event or Result was swapped but EventResult was not!"
    )

def nest_ordering(prefix, ordering, ignore_reverse=False):
    nest_order = []
    for field in ordering:
        reverse = ''
        if field.startswith('-'):
            field = field[1:]
            if not ignore_reverse:
                reverse = '-'
        nest_order.append(reverse + prefix + '__' + field)
    return nest_order

def find_events(instance):
    if isinstance(instance, BaseEvent):
        return [instance]
    if isinstance(instance, BaseReport):
        return [instance.event]
    if isinstance(instance, BaseResult):
        return [instance.report.event]
    return None
<|endoftext|>


<commit_before>from moviepy.editor import *
from annotations import get_annotation_duration, get_annotations_added_duration, get_subtitle, get_marker

def bake_annotations(video_file, end_point, annotations):
    clip = VideoFileClip(video_file)
    annotated_video = generate_annotation_markings(clip, annotations)
    final_video = generate_video_pauses(annotated_video, annotations)
    final_video_audio = generate_pause_audio(clip.audio, annotations)
    final_video.set_audio(final_video_audio)
    final_video.write_videofile(end_point)

def generate_annotation_markings(video_clip, annotations):
    composite_clips = [video_clip]
    one_frame_time = 1 / video_clip.fps

    for annotation in annotations:
        txt_clip = get_subtitle(annotation, one_frame_time)
        marker = get_marker(annotation, one_frame_time, video_clip)
        composite_clips.append(txt_clip)
        composite_clips.append(marker)

    return CompositeVideoClip(composite_clips)

def generate_video_pauses(video_clip, annotations):
    """Takes in a regular video clip, and bakes in annotation pauses"""
    for annotation in reversed(annotations):
        pause_time = get_annotation_duration(annotation)
        current_annotation_time = annotation["time"] / 1000.0
        video_clip = video_clip.fx(vfx.freeze, t=current_annotation_time, freeze_duration=pause_time)

    return video_clip

def generate_pause_audio(original_audio, annotations):
    audio_clip = original_audio
    last_pause_time = 0
    audio_parts = []

    for annotation in annotations:
        pause_time = get_annotation_duration(annotation)
        current_annotation_time = annotation["time"] / 1000.0

        audio_part = audio_clip.subclip(last_pause_time, current_annotation_time)
        audio_part = audio_part.set_start(last_pause_time)
        audio_part = audio_part.set_end(current_annotation_time)
        audio_parts.append(audio_part)

        last_pause_time = current_annotation_time + pause_time

    comp = CompositeAudioClip(audio_parts)

    return comp

<commit_msg>Fix audio related crash on short clips<commit_after>from moviepy.editor import *
from annotations import get_annotation_duration, get_annotations_added_duration, get_subtitle, get_marker

def bake_annotations(video_file, end_point, annotations):
    clip = VideoFileClip(video_file)
    annotated_video = generate_annotation_markings(clip, annotations)
    final_video = generate_video_pauses(annotated_video, annotations)
    audio = clip.audio.set_duration(clip.audio.duration + get_annotations_added_duration(annotations))
    final_video_audio = generate_pause_audio(audio, annotations)
    final_video.set_audio(final_video_audio)
    final_video.write_videofile(end_point)

def generate_annotation_markings(video_clip, annotations):
    composite_clips = [video_clip]
    one_frame_time = 1 / video_clip.fps

    for annotation in annotations:
        txt_clip = get_subtitle(annotation, one_frame_time)
        marker = get_marker(annotation, one_frame_time, video_clip)
        composite_clips.append(txt_clip)
        composite_clips.append(marker)

    return CompositeVideoClip(composite_clips)

def generate_video_pauses(video_clip, annotations):
    """Takes in a regular video clip, and bakes in annotation pauses"""
    for annotation in reversed(annotations):
        pause_time = get_annotation_duration(annotation)
        current_annotation_time = annotation["time"] / 1000.0
        video_clip = video_clip.fx(vfx.freeze, t=current_annotation_time, freeze_duration=pause_time)

    return video_clip

def generate_pause_audio(original_audio, annotations):
    audio_clip = original_audio
    last_pause_time = 0
    audio_parts = []

    for annotation in annotations:
        pause_time = get_annotation_duration(annotation)
        current_annotation_time = annotation["time"] / 1000.0

        audio_part = audio_clip.subclip(last_pause_time, current_annotation_time)
        audio_part = audio_part.set_start(last_pause_time)
        audio_part = audio_part.set_end(current_annotation_time)
        audio_parts.append(audio_part)

        last_pause_time = current_annotation_time + pause_time

    comp = CompositeAudioClip(audio_parts)

    return comp

<|endoftext|>


<commit_before>#!/usr/bin/python
"""videothread.py: Threaded, stopable video player class
Author: Wes Modes (wmodes@gmail.com)
Copyright: 2017, MIT """

# -*- coding: iso-8859-15 -*-
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import threading
import subprocess
import sys
import signal
import os
import time

# local imports
import ffprobe

#
# constants
#

INTER_VIDEO_DELAY = 0.75

OMX_CMD = ['omxplayer', '--no-osd', '--no-keys', '--refresh', '--aspect-mode fill']
CONTENT_CMD = OMX_CMD + ['--layer %i', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']
LOOP_CMD = OMX_CMD + ['--layer %i', '--loop', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']
TRANSITION_CMD = OMX_CMD + ['--layer %i', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']

nullin = open(os.devnull, 'r')
nullout = open(os.devnull, 'w')

#
# globals
#

omx_layer_content = 1
omx_layer_loop = 3
omx_layer_transition = 5
omx_player = 1

class VideoThread(threading.Thread):
    """Thread class with a stop() method. The thread itself checks
    regularly for the stopped() condition."""
    _example = """
        The class takes a list containing one or more dictionaries containing
        data about the videos to be played:
            [{'name': "loop-idle1",         # if omitted, uses 'file'
                'file': "loop-idle1.mp4",   # looks in 'mediabase' for media
                'tags': 'loop',             # loop, transition, content, etc
                'start': 0.0,               # if omitted, assumes 0
                'length': 0.0,              # if omitted, assumes duration(filename)
                'disabled': True,           # if omitted, assumes False
             },]
        """

    def __init__(self, playlist=None, media_dir=".", debug=0):
        super(VideoThread, self).__init__()
        self._stop = threading.Event()
        # passed parameters
        self.playlist = playlist
        self.media_dir = media_dir
        # internal flags and vars
        self._debug_flag = debug
        self._last_debug_caller = None
        self._current_video = None
        self._player_pgid = None
        self._end_time = 0

    def set_sequence(self, playlist=None):
        self.playlist = playlist

    def stop(self):
        self._debug("Stop flag set")
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def debug(self, debug_flag):
        self._debug_flag = debug_flag

    def _debug(self, *args, **kwargs):
        """Produce debug message, indenting if from same calling function"""
        level = 1
        if ("level" in kwargs):
            level = kwargs["level"]
        elif("l" in kwargs):
            level = kwargs["l"]
        if (self._debug_flag >= level):
            text = " ".join(list(map(str, args)))
            # get name of calling function
            caller = sys._getframe(1).f_code.co_name
            if (caller == self._last_debug_caller):
                print "  debug: %s: %s" % (caller, text)
            else:
                print "debug: %s: %s" % (caller, text)
            # save last calling function
            self._last_debug_caller = caller

    def run(self):
        if not isinstance(self.playlist, list):
            raise ValueError(self._example)
        for video in self.playlist:
            if not self.stopped():
                self._start_video(video)

    def _start_video(self, video):
        """Starts a video. Takes a video object """
        # add media_dir to filename
        global omx_layer_content, omx_layer_loop, omx_layer_transition, omx_player
        filename = self.media_dir + '/' + video['file']
        # check to make sure we've passed the right thing
        if not isinstance(video, dict):
            raise ValueError(self._example)
        # set video name if we have it
        if 'name' in video:
            name = video['name']
        else:
            name = video['file']
        # skip this video if disabled in db
        if 'disabled' in video and video['disabled']:
            self._debug("Not played:", name, "disabled")
            return
        #debug messages
        self._debug("Starting %s in %s" % (name, self.media_dir))
        self._debug("Video data:", video)
        # get length
        filelength = self._get_length(filename)
        if ('length' not in video or video['length'] == 0.0):
            length = filelength
        else:
            length = video['length']
        # get start
        if 'start' in video:
            start = video['start']
        else:
            start = 0.0
        # if start is too large, set it to 0
        if (start >= filelength):
            start = 0.0
        # if length is too large, scale it back, unless loop
        if (('loop' not in video['tags']) and (start + length >= filelength)):
            length = filelength - start
        # store this for later
        self._current_video = video
        # debugging output
        self._debug("name: %s (%s)" % (name, filename))
        self._debug("tags: %s" % video['tags'])
        self._debug("start: %.1fs, end: %.1fs, len: %.1fs" %
                    (start, start+length, length))
        # each time we switch to a new video, we switch the layer
        # this will effectively toggle the 3 layer variables between (1,2), (3,4), and (5,6)
        omx_layer_content = 1 if (omx_layer_content != 1) else 2
        omx_layer_loop = 3 if (omx_layer_loop != 3) else 4
        omx_layer_transition = 5 if (omx_layer_transition != 5) else 6
        # we also toggle the virtual player
        omx_player = 1 if (omx_player != 1) else 2
        # build omxplayer command
        if ('loop' in video['tags']):
            my_cmd = " ".join(LOOP_CMD + [filename]) % (omx_layer_loop,
                              omx_player)
        elif ('transition' in video['tags']):
            my_cmd = " ".join(TRANSITION_CMD + ['--pos', str(start), filename]) % (omx_layer_transition,
                              omx_player)
        else:
            my_cmd = " ".join(CONTENT_CMD + ['--pos', str(start), filename]) % (omx_layer_content,
                              omx_player)
        self._debug("cmd:", my_cmd, l=2)
        # launch the player, saving the process handle
        # TODO: after debugging, replace 'if True' with 'try' and enable 'except'
        if True:
        # try:
            proc = None
            if (self._debug_flag >= 3):
                proc = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin)
            else:
                proc = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin, stdout=nullout)
            # save this process group id
            pgid = os.getpgid(proc.pid)
            self._player_pgid = pgid
            self._debug("Starting process: %i (%s)" % (pgid, name))
            # If we have a loop
            if ('loop' in video['tags']):
                self._debug("Looping %.2fs and setting kill timer for %s (pid %i)" %
                            (length - INTER_VIDEO_DELAY, name, pgid))
            # otherwise
            else:
                self._debug("Waiting %.2fs and setting kill timer for %s (pid %i)" %
                            (length - INTER_VIDEO_DELAY, name, pgid))
            # wait in a tight loop, checking if we've received stop event or time is over
            start_time = time.time()
            self._end_time = start_time + length - INTER_VIDEO_DELAY
            # when we get close to the end, we release the thread to start new vid
            while (not self.stopped() and
                   (time.time() <= self._end_time)):
                pass
            # then we set a timer to kill the old vid
            threading.Timer(INTER_VIDEO_DELAY,
                            self._stop_video, [pgid, name]).start()
        # except:
        #     self._debug("Unable to start video", name, l=0)

    def wait_for_end(self):
        """Wait for end of video in tight loop"""
        # wait until we get the timing info
        while not self._end_time:
            pass
        #self._debug("Waiting for end of video")
        # now wait until time expires
        while (not self.stopped() and
               (time.time() <= self._end_time)):
            pass

    def _stop_video(self, pgid, name):
        try:
            self._debug("Killing process %i (%s)" % (pgid, name))
            os.killpg(pgid, signal.SIGTERM)
            self._player_pgid = None
            self._current_video = None
        except OSError:
            self._debug("Couldn't terminate %i (%s)" % (pgid, name))
            pass

    def _get_length(self, filename):
        self._debug("Getting duration of %s" % filename)
        length = ffprobe.duration(filename)
        if length is None:
            length = 0
        return length

def main():
    media_dir = "media"
    films = [
          { "file" : "launch_party_timelapse.mp4",
            "tags" : [ "feature" ],
            "length" : 20,
          },
          { "file" : "secret_history_dorris_turner_what_does_the_river_mean_to_you.mp4",
            "tags" : [ "interview" ],
            "length" : 20,
          },
          { "file" : "la_crosse_boathouses_loop.mp4",
            "tags" : [ "loop" ],
            "length" : 200,
          },
        ]
    for film in films:
        print "Starting threaded sequence"
        video = VideoThread([film], media_dir, debug=2)
        video.start()
        print "Sequence started"
        video.wait_for_end()

if __name__ == "__main__":
    main()
<commit_msg>switch from fill to letterbox format<commit_after>#!/usr/bin/python
"""videothread.py: Threaded, stopable video player class
Author: Wes Modes (wmodes@gmail.com)
Copyright: 2017, MIT """

# -*- coding: iso-8859-15 -*-
# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

import threading
import subprocess
import sys
import signal
import os
import time

# local imports
import ffprobe

#
# constants
#

INTER_VIDEO_DELAY = 0.75

OMX_CMD = ['omxplayer', '--no-osd', '--no-keys', '--refresh', '--aspect-mode letterbox']
CONTENT_CMD = OMX_CMD + ['--layer %i', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']
LOOP_CMD = OMX_CMD + ['--layer %i', '--loop', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']
TRANSITION_CMD = OMX_CMD + ['--layer %i', '--dbus_name', 'org.mpris.MediaPlayer2.omxplayer%i']

nullin = open(os.devnull, 'r')
nullout = open(os.devnull, 'w')

#
# globals
#

omx_layer_content = 1
omx_layer_loop = 3
omx_layer_transition = 5
omx_player = 1

class VideoThread(threading.Thread):
    """Thread class with a stop() method. The thread itself checks
    regularly for the stopped() condition."""
    _example = """
        The class takes a list containing one or more dictionaries containing
        data about the videos to be played:
            [{'name': "loop-idle1",         # if omitted, uses 'file'
                'file': "loop-idle1.mp4",   # looks in 'mediabase' for media
                'tags': 'loop',             # loop, transition, content, etc
                'start': 0.0,               # if omitted, assumes 0
                'length': 0.0,              # if omitted, assumes duration(filename)
                'disabled': True,           # if omitted, assumes False
             },]
        """

    def __init__(self, playlist=None, media_dir=".", debug=0):
        super(VideoThread, self).__init__()
        self._stop = threading.Event()
        # passed parameters
        self.playlist = playlist
        self.media_dir = media_dir
        # internal flags and vars
        self._debug_flag = debug
        self._last_debug_caller = None
        self._current_video = None
        self._player_pgid = None
        self._end_time = 0

    def set_sequence(self, playlist=None):
        self.playlist = playlist

    def stop(self):
        self._debug("Stop flag set")
        self._stop.set()

    def stopped(self):
        return self._stop.isSet()

    def debug(self, debug_flag):
        self._debug_flag = debug_flag

    def _debug(self, *args, **kwargs):
        """Produce debug message, indenting if from same calling function"""
        level = 1
        if ("level" in kwargs):
            level = kwargs["level"]
        elif("l" in kwargs):
            level = kwargs["l"]
        if (self._debug_flag >= level):
            text = " ".join(list(map(str, args)))
            # get name of calling function
            caller = sys._getframe(1).f_code.co_name
            if (caller == self._last_debug_caller):
                print "  debug: %s: %s" % (caller, text)
            else:
                print "debug: %s: %s" % (caller, text)
            # save last calling function
            self._last_debug_caller = caller

    def run(self):
        if not isinstance(self.playlist, list):
            raise ValueError(self._example)
        for video in self.playlist:
            if not self.stopped():
                self._start_video(video)

    def _start_video(self, video):
        """Starts a video. Takes a video object """
        # add media_dir to filename
        global omx_layer_content, omx_layer_loop, omx_layer_transition, omx_player
        filename = self.media_dir + '/' + video['file']
        # check to make sure we've passed the right thing
        if not isinstance(video, dict):
            raise ValueError(self._example)
        # set video name if we have it
        if 'name' in video:
            name = video['name']
        else:
            name = video['file']
        # skip this video if disabled in db
        if 'disabled' in video and video['disabled']:
            self._debug("Not played:", name, "disabled")
            return
        #debug messages
        self._debug("Starting %s in %s" % (name, self.media_dir))
        self._debug("Video data:", video)
        # get length
        filelength = self._get_length(filename)
        if ('length' not in video or video['length'] == 0.0):
            length = filelength
        else:
            length = video['length']
        # get start
        if 'start' in video:
            start = video['start']
        else:
            start = 0.0
        # if start is too large, set it to 0
        if (start >= filelength):
            start = 0.0
        # if length is too large, scale it back, unless loop
        if (('loop' not in video['tags']) and (start + length >= filelength)):
            length = filelength - start
        # store this for later
        self._current_video = video
        # debugging output
        self._debug("name: %s (%s)" % (name, filename))
        self._debug("tags: %s" % video['tags'])
        self._debug("start: %.1fs, end: %.1fs, len: %.1fs" %
                    (start, start+length, length))
        # each time we switch to a new video, we switch the layer
        # this will effectively toggle the 3 layer variables between (1,2), (3,4), and (5,6)
        omx_layer_content = 1 if (omx_layer_content != 1) else 2
        omx_layer_loop = 3 if (omx_layer_loop != 3) else 4
        omx_layer_transition = 5 if (omx_layer_transition != 5) else 6
        # we also toggle the virtual player
        omx_player = 1 if (omx_player != 1) else 2
        # build omxplayer command
        if ('loop' in video['tags']):
            my_cmd = " ".join(LOOP_CMD + [filename]) % (omx_layer_loop,
                              omx_player)
        elif ('transition' in video['tags']):
            my_cmd = " ".join(TRANSITION_CMD + ['--pos', str(start), filename]) % (omx_layer_transition,
                              omx_player)
        else:
            my_cmd = " ".join(CONTENT_CMD + ['--pos', str(start), filename]) % (omx_layer_content,
                              omx_player)
        self._debug("cmd:", my_cmd, l=2)
        # launch the player, saving the process handle
        # TODO: after debugging, replace 'if True' with 'try' and enable 'except'
        if True:
        # try:
            proc = None
            if (self._debug_flag >= 3):
                proc = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin)
            else:
                proc = subprocess.Popen(my_cmd, shell=True, preexec_fn=os.setsid, stdin=nullin, stdout=nullout)
            # save this process group id
            pgid = os.getpgid(proc.pid)
            self._player_pgid = pgid
            self._debug("Starting process: %i (%s)" % (pgid, name))
            # If we have a loop
            if ('loop' in video['tags']):
                self._debug("Looping %.2fs and setting kill timer for %s (pid %i)" %
                            (length - INTER_VIDEO_DELAY, name, pgid))
            # otherwise
            else:
                self._debug("Waiting %.2fs and setting kill timer for %s (pid %i)" %
                            (length - INTER_VIDEO_DELAY, name, pgid))
            # wait in a tight loop, checking if we've received stop event or time is over
            start_time = time.time()
            self._end_time = start_time + length - INTER_VIDEO_DELAY
            # when we get close to the end, we release the thread to start new vid
            while (not self.stopped() and
                   (time.time() <= self._end_time)):
                pass
            # then we set a timer to kill the old vid
            threading.Timer(INTER_VIDEO_DELAY,
                            self._stop_video, [pgid, name]).start()
        # except:
        #     self._debug("Unable to start video", name, l=0)

    def wait_for_end(self):
        """Wait for end of video in tight loop"""
        # wait until we get the timing info
        while not self._end_time:
            pass
        #self._debug("Waiting for end of video")
        # now wait until time expires
        while (not self.stopped() and
               (time.time() <= self._end_time)):
            pass

    def _stop_video(self, pgid, name):
        try:
            self._debug("Killing process %i (%s)" % (pgid, name))
            os.killpg(pgid, signal.SIGTERM)
            self._player_pgid = None
            self._current_video = None
        except OSError:
            self._debug("Couldn't terminate %i (%s)" % (pgid, name))
            pass

    def _get_length(self, filename):
        self._debug("Getting duration of %s" % filename)
        length = ffprobe.duration(filename)
        if length is None:
            length = 0
        return length

def main():
    media_dir = "media"
    films = [
          { "file" : "launch_party_timelapse.mp4",
            "tags" : [ "feature" ],
            "length" : 20,
          },
          { "file" : "secret_history_dorris_turner_what_does_the_river_mean_to_you.mp4",
            "tags" : [ "interview" ],
            "length" : 20,
          },
          { "file" : "la_crosse_boathouses_loop.mp4",
            "tags" : [ "loop" ],
            "length" : 200,
          },
        ]
    for film in films:
        print "Starting threaded sequence"
        video = VideoThread([film], media_dir, debug=2)
        video.start()
        print "Sequence started"
        video.wait_for_end()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2012 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

from __future__ import print_function

"""WebSearch Admin Flask Blueprint"""

from flask import Blueprint, g, render_template, request, flash, redirect, \
    url_for, abort
from flask.ext.menu import register_menu
from invenio.ext.sqlalchemy import db
from ..models import Collection, CollectionCollection, \
        Collectionname, CollectionPortalbox, Portalbox
from invenio.base.i18n import _
from invenio.base.decorators import templated
from flask.ext.login import current_user, login_required
from invenio.ext.principal import permission_required
from invenio.base.i18n import language_list_long

# imports the necessary forms
from ..admin_forms import CollectionForm, TranslationsForm

not_guest = lambda: not current_user.is_guest

blueprint = Blueprint('websearch_admin', __name__,
                      url_prefix="/admin/websearch",
                      template_folder='../templates'
                      )

#breadcrumbs=[(_('Configure WebSearch'), 'websearch_admin.index')])

@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/index', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
@templated('search/admin_index.html')
@register_menu(blueprint, 'main.admin.websearch', _('Configure WebSearch'),
               order=50)
def index():
    """
    WebSearch admin interface with editable collection tree.
    """
    collection = Collection.query.get_or_404(1)
    orphans = Collection.query.filter(
        db.not_(db.or_(
            Collection.id.in_(db.session.query(CollectionCollection.id_son).subquery()),
            Collection.id.in_(db.session.query(CollectionCollection.id_dad).subquery())
        ))).all()

    return dict(collection=collection, orphans=orphans)

@blueprint.route('/modifycollectiontree', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
def modifycollectiontree():
    """
    Handler of the tree changing operations triggered by the drag and drop operation
    """

    # Get the requests parameters
    id_son = request.form.get('id_son', 0, type=int)
    id_dad = request.form.get('id_dad', 0, type=int)
    id_new_dad = request.form.get('id_new_dad', 0, type=int)
    score = request.form.get('score', 0, type=int)
    #if id_dad == id_new_dad:
    #    score = score + 1
    type = request.form.get('type', 'r')

    # Check if collection exits.
    Collection.query.get_or_404(id_son)

    if id_dad > 0:
        # Get only one record
        cc = CollectionCollection.query.filter(
            db.and_(
            CollectionCollection.id_son==id_son,
            CollectionCollection.id_dad==id_dad
            )).one()
        dad = Collection.query.get_or_404(id_dad)
        dad._collection_children.remove(cc)
        dad._collection_children.reorder()
    else:
        cc = CollectionCollection(
            id_dad=id_new_dad,
            id_son=id_son,
            type=type)
        db.session.add(cc)

    if id_new_dad == 0:
        db.session.delete(cc)
    else:
        new_dad = Collection.query.get_or_404(id_new_dad)
        cc.id_dad = id_new_dad
        try:
            descendants = Collection.query.get(id_son).descendants_ids
            ancestors = new_dad.ancestors_ids
            print(descendants, ancestors)
            if descendants & ancestors:
                raise
        except:
            ## Cycle has been detected.
            db.session.rollback()
            abort(406)
        new_dad._collection_children.reorder()
        new_dad._collection_children.insert(score, cc)

    #FIXME add dbrecs rebuild for modified trees.

    db.session.commit()
    return 'done'

@blueprint.route('/collectiontree', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def managecollectiontree():
    """
    Here is where managing the tree is possible
    """

    collection = Collection.query.get_or_404(1)
    orphans = Collection.query.filter(
            Collection.id != CollectionCollection.id_dad,
            id != CollectionCollection.id_son).get_or_404(1)

    return dict(collection=collection, orphans=orphans)

@blueprint.route('/collection/<name>', methods=['GET', 'POST'])
@blueprint.route('/collection/view/<name>', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def manage_collection(name):
    collection = Collection.query.filter(Collection.name==name).first_or_404()
    form = CollectionForm(request.form, obj = collection)

    # gets the collections translations
    translations = dict((x.ln,x.value) for x in  collection.collection_names)

    # Creating the translations form
    TranslationsFormFilled = TranslationsForm(language_list_long(), translations)
    translation_form = TranslationsFormFilled(request.form)
    #for x in  collection.collection_names:
    #    translation_form[x.ln](default = x.value)

    #translation_form.populate_obj(translations)

    return render_template('search/admin_collection.html', \
            collection = collection, form=form, \
            translation_form=translation_form)

@blueprint.route('/collection/update/<id>', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
def update(id):
    form = CollectionForm(request.form)
    if  request.method == 'POST':# and form.validate():
        collection = Collection.query.filter(Collection.id==id).first_or_404()
        form.populate_obj(collection)
        db.session.commit()
        flash(_('Collection was updated'), "info")
        return redirect(url_for('.index'))

@blueprint.route('/collection/new', methods=['GET', 'POST'])
@blueprint.route('/collection/add', methods=['GET', 'POST'])
#@login_required
@templated('search/admin_collection.html')
def create_collection():
    form = CollectionForm()
    return dict(form=form)

@blueprint.route('/collection/update_translations<id>', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
#@login_required
def update_translations(id):
    """
    updates translations if the value is altered or not void
    """

    collection = Collection.query.filter(Collection.id==id).first_or_404()

    for (lang, lang_long) in language_list_long():

        collection_name = Collectionname.query.filter(
                db.and_(Collectionname.id_collection==id,
            Collectionname.ln == lang, Collectionname.type=='ln')).first()

        if collection_name:
            if collection_name.value != request.form.get(lang):
                collection_name.value = request.form.get(lang)
                db.session.commit()
        else:
            if request.form.get(lang) != '':
                collection_name = Collectionname(collection, lang, \
                       'ln', request.form.get(lang))
                db.session.add(collection_name)
                db.session.commit()

    flash(_('Collection was updated on n languages:'), "info")
    return redirect(url_for('.manage_collection', name = collection.name))

@blueprint.route('/collection/manage_portalboxes_order', methods=['GET', 'POST'])
#@login_required
def manage_portalboxes_order():
    id_p = request.args.get('id', 0 , type=int)
    collection_id = request.args.get('id_collection', 0 , type=int)
    order = request.args.get('score', 0 , type=int)

    collection = Collection.query.filter(Collection.id==collection_id).first_or_404()

    portalbox = \
            CollectionPortalbox.query.filter(db.and_(
                CollectionPortalbox.id_portalbox==id_p,
                CollectionPortalbox.id_collection==collection_id)).first_or_404()

    position = portalbox.position
    p_order = portalbox.score

    db.session.delete(portalbox)

    #p = portalboxes.pop(portalbox)
    collection.portal_boxes_ln.set(CollectionPortalbox(collection_id, \
            id_p,\
            g.ln, position, p_order ), order )
    db.session.commit()

    return ''

@blueprint.route('/collection/edit_portalbox', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def edit_portalbox():
    portalbox = Portalbox.query.get(request.args.get_or_404('id', 0, type=int))
    return dict(portalbox = portalbox)
<commit_msg>base: admin drop-down menu fix<commit_after># -*- coding: utf-8 -*-
##
## This file is part of Invenio.
## Copyright (C) 2012, 2014 CERN.
##
## Invenio is free software; you can redistribute it and/or
## modify it under the terms of the GNU General Public License as
## published by the Free Software Foundation; either version 2 of the
## License, or (at your option) any later version.
##
## Invenio is distributed in the hope that it will be useful, but
## WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
## General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with Invenio; if not, write to the Free Software Foundation, Inc.,
## 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""WebSearch Admin Flask Blueprint."""

from __future__ import print_function

from flask import Blueprint, g, render_template, request, flash, redirect, \
    url_for, abort
from flask.ext.breadcrumbs import register_breadcrumb
from invenio.ext.sqlalchemy import db
from ..models import Collection, CollectionCollection, \
    Collectionname, CollectionPortalbox, Portalbox
from invenio.base.i18n import _
from invenio.base.decorators import templated
from flask.ext.login import current_user, login_required
from invenio.ext.principal import permission_required
from invenio.base.i18n import language_list_long

# imports the necessary forms
from ..admin_forms import CollectionForm, TranslationsForm

not_guest = lambda: not current_user.is_guest

blueprint = Blueprint('websearch_admin', __name__,
                      url_prefix="/admin/websearch",
                      template_folder='../templates'
                      )

#breadcrumbs=[(_('Configure WebSearch'), 'websearch_admin.index')])

@blueprint.route('/', methods=['GET', 'POST'])
@blueprint.route('/index', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
@templated('search/admin_index.html')
@register_breadcrumb(blueprint, 'admin.websearch_admin', _('WebSearch'))
def index():
    """WebSearch admin interface with editable collection tree."""
    collection = Collection.query.get_or_404(1)
    orphans = Collection.query.filter(
        db.not_(db.or_(
            Collection.id.in_(db.session.query(
                              CollectionCollection.id_son).subquery()),
            Collection.id.in_(db.session.query(
                              CollectionCollection.id_dad).subquery())
        ))).all()

    return dict(collection=collection, orphans=orphans)

@blueprint.route('/modifycollectiontree', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
def modifycollectiontree():
    """
    Handler of the tree changing operations triggered by the drag and drop operation.
    """
    # Get the requests parameters
    id_son = request.form.get('id_son', 0, type=int)
    id_dad = request.form.get('id_dad', 0, type=int)
    id_new_dad = request.form.get('id_new_dad', 0, type=int)
    score = request.form.get('score', 0, type=int)
    #if id_dad == id_new_dad:
    #    score = score + 1
    type = request.form.get('type', 'r')

    # Check if collection exits.
    Collection.query.get_or_404(id_son)

    if id_dad > 0:
        # Get only one record
        cc = CollectionCollection.query.filter(
            db.and_(
                CollectionCollection.id_son == id_son,
                CollectionCollection.id_dad == id_dad
            )).one()
        dad = Collection.query.get_or_404(id_dad)
        dad._collection_children.remove(cc)
        dad._collection_children.reorder()
    else:
        cc = CollectionCollection(
            id_dad=id_new_dad,
            id_son=id_son,
            type=type)
        db.session.add(cc)

    if id_new_dad == 0:
        db.session.delete(cc)
    else:
        new_dad = Collection.query.get_or_404(id_new_dad)
        cc.id_dad = id_new_dad
        try:
            descendants = Collection.query.get(id_son).descendants_ids
            ancestors = new_dad.ancestors_ids
            print(descendants, ancestors)
            if descendants & ancestors:
                raise
        except:
            ## Cycle has been detected.
            db.session.rollback()
            abort(406)
        new_dad._collection_children.reorder()
        new_dad._collection_children.insert(score, cc)

    #FIXME add dbrecs rebuild for modified trees.

    db.session.commit()
    return 'done'

@blueprint.route('/collectiontree', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def managecollectiontree():
    """Here is where managing the tree is possible."""
    collection = Collection.query.get_or_404(1)
    orphans = Collection.query.filter(
        Collection.id != CollectionCollection.id_dad,
        id != CollectionCollection.id_son).get_or_404(1)

    return dict(collection=collection, orphans=orphans)

@blueprint.route('/collection/<name>', methods=['GET', 'POST'])
@blueprint.route('/collection/view/<name>', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def manage_collection(name):
    collection = Collection.query.filter(Collection.name == name).first_or_404()
    form = CollectionForm(request.form, obj=collection)

    # gets the collections translations
    translations = dict((x.ln, x.value) for x in collection.collection_names)

    # Creating the translations form
    TranslationsFormFilled = TranslationsForm(language_list_long(),
                                              translations)
    translation_form = TranslationsFormFilled(request.form)
    #for x in  collection.collection_names:
    #    translation_form[x.ln](default = x.value)

    #translation_form.populate_obj(translations)

    return render_template('search/admin_collection.html',
                           collection=collection, form=form,
                           translation_form=translation_form)

@blueprint.route('/collection/update/<id>', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
def update(id):
    form = CollectionForm(request.form)
    if request.method == 'POST':  # and form.validate():
        collection = Collection.query.filter(Collection.id == id).first_or_404()
        form.populate_obj(collection)
        db.session.commit()
        flash(_('Collection was updated'), "info")
        return redirect(url_for('.index'))

@blueprint.route('/collection/new', methods=['GET', 'POST'])
@blueprint.route('/collection/add', methods=['GET', 'POST'])
#@login_required
@templated('search/admin_collection.html')
def create_collection():
    form = CollectionForm()
    return dict(form=form)

@blueprint.route('/collection/update_translations<id>', methods=['POST'])
@login_required
@permission_required('cfgwebsearch')
#@login_required
def update_translations(id):
    """Update translations if the value is altered or not void."""
    collection = Collection.query.filter(Collection.id == id).first_or_404()

    for (lang, lang_long) in language_list_long():

        collection_name = Collectionname.query.filter(
            db.and_(Collectionname.id_collection == id,
                    Collectionname.ln == lang, Collectionname.type == 'ln')).first()

        if collection_name:
            if collection_name.value != request.form.get(lang):
                collection_name.value = request.form.get(lang)
                db.session.commit()
        else:
            if request.form.get(lang) != '':
                collection_name = Collectionname(collection, lang,
                                                 'ln', request.form.get(lang))
                db.session.add(collection_name)
                db.session.commit()

    flash(_('Collection was updated on n languages:'), "info")
    return redirect(url_for('.manage_collection', name=collection.name))

@blueprint.route('/collection/manage_portalboxes_order', methods=['GET', 'POST'])
#@login_required
def manage_portalboxes_order():
    id_p = request.args.get('id', 0, type=int)
    collection_id = request.args.get('id_collection', 0, type=int)
    order = request.args.get('score', 0, type=int)

    collection = Collection.query.filter(Collection.id == collection_id).first_or_404()

    portalbox = \
        CollectionPortalbox.query.filter(db.and_(
            CollectionPortalbox.id_portalbox == id_p,
            CollectionPortalbox.id_collection == collection_id)).first_or_404()

    position = portalbox.position
    p_order = portalbox.score

    db.session.delete(portalbox)

    #p = portalboxes.pop(portalbox)
    collection.portal_boxes_ln.set(CollectionPortalbox(collection_id,
                                                       id_p,
                                                       g.ln, position,
                                                       p_order), order)
    db.session.commit()

    return ''

@blueprint.route('/collection/edit_portalbox', methods=['GET', 'POST'])
@login_required
@permission_required('cfgwebsearch')
def edit_portalbox():
    portalbox = Portalbox.query.get(request.args.get_or_404('id', 0, type=int))
    return dict(portalbox=portalbox)
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# Copyright 2012-2019 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

"""Adds functionality to make CBSS requests.

Technical specs see :ref:`weleup`.

.. autosummary::
   :toctree:

    mixins
    choicelists
    utils
    fixtures.cbss_demo
    fixtures.cbss
    fixtures.purposes
    fixtures.sectors

.. rubric:: Plugin configuration

"""

import six
from lino import ad
from django.utils.translation import ugettext_lazy as _

class Plugin(ad.Plugin):
    """The descriptor for this plugin. See
    :class:`lino.core.plugin.Plugin`.

    """
    verbose_name = _("CBSS")

    needs_plugins = ['lino_welfare.modlib.integ']

    cbss_live_requests = False
    """Whether executing requests should try to really connect to the
    CBSS.  Real requests would fail with a timeout if run from behind
    an IP address that is not registered at the :term:`CBSS`.

    """

    #~ cbss_environment = None
    cbss_environment = 'test'
    """
    Either `None` or one of 'test', 'acpt' or 'prod'.

    Setting this to `None` means that the cbss app is "inactive" even
    if installed.

    """

    def get_requirements(self, site):
        if six.PY2:
            yield 'suds'
        else:
            # yield 'suds-py3'
            # as long as https://github.com/cackharot/suds-py3/pull/40 is not
            # released, we must use the development version:
            yield 'git+https://github.com/karimabdelhakim/suds-py3#egg=suds-py3'

    def get_used_libs(self, html=None):
        try:
            import suds
            version = suds.__version__
        except ImportError:
            version = self.site.not_found_msg
        yield ("suds", version, "https://fedorahosted.org/suds/")

    def setup_main_menu(self, site, user_type, m):
        mg = site.plugins.integ
        m = m.add_menu(mg.app_label, mg.verbose_name)
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.MyIdentifyPersonRequests')
        m.add_action('cbss.MyManageAccessRequests')
        m.add_action('cbss.MyRetrieveTIGroupsRequests')

    def setup_config_menu(self, site, user_type, m):
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.Sectors')
        m.add_action('cbss.Purposes')

    def setup_explorer_menu(self, site, user_type, m):
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.AllIdentifyPersonRequests')
        m.add_action('cbss.AllManageAccessRequests')
        m.add_action('cbss.AllRetrieveTIGroupsRequests')
<commit_msg>fix url of suds-py3 library<commit_after># -*- coding: UTF-8 -*-
# Copyright 2012-2020 Rumma & Ko Ltd
# License: BSD (see file COPYING for details)

"""Adds functionality to make CBSS requests.

Technical specs see :ref:`weleup`.

.. autosummary::
   :toctree:

    mixins
    choicelists
    utils
    fixtures.cbss_demo
    fixtures.cbss
    fixtures.purposes
    fixtures.sectors

.. rubric:: Plugin configuration

"""

import six
from lino import ad
from django.utils.translation import ugettext_lazy as _

class Plugin(ad.Plugin):
    """The descriptor for this plugin. See
    :class:`lino.core.plugin.Plugin`.

    """
    verbose_name = _("CBSS")

    needs_plugins = ['lino_welfare.modlib.integ']

    cbss_live_requests = False
    """Whether executing requests should try to really connect to the
    CBSS.  Real requests would fail with a timeout if run from behind
    an IP address that is not registered at the :term:`CBSS`.

    """

    #~ cbss_environment = None
    cbss_environment = 'test'
    """
    Either `None` or one of 'test', 'acpt' or 'prod'.

    Setting this to `None` means that the cbss app is "inactive" even
    if installed.

    """

    def get_requirements(self, site):
        if six.PY2:
            yield 'suds'
        else:
            # yield 'suds-py3'
            # as long as https://github.com/cackharot/suds-py3/pull/40 is not
            # released, we must use the development version:
            yield 'git+https://github.com/karimabdelhakim/suds-py3#egg=suds-py3'

    def get_used_libs(self, html=None):
        try:
            import suds
            version = suds.__version__
        except ImportError:
            version = self.site.not_found_msg
        yield ("suds", version, "https://pypi.org/project/suds-py3/")

    def setup_main_menu(self, site, user_type, m):
        mg = site.plugins.integ
        m = m.add_menu(mg.app_label, mg.verbose_name)
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.MyIdentifyPersonRequests')
        m.add_action('cbss.MyManageAccessRequests')
        m.add_action('cbss.MyRetrieveTIGroupsRequests')

    def setup_config_menu(self, site, user_type, m):
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.Sectors')
        m.add_action('cbss.Purposes')

    def setup_explorer_menu(self, site, user_type, m):
        m = m.add_menu(self.app_label, self.verbose_name)
        m.add_action('cbss.AllIdentifyPersonRequests')
        m.add_action('cbss.AllManageAccessRequests')
        m.add_action('cbss.AllRetrieveTIGroupsRequests')
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Lint as: python3
"""Gradient-based attribution."""

import copy
import functools
from typing import Any, Iterable, List, Optional

from absl import logging
from lit_nlp.api import components as lit_components
from lit_nlp.api import dataset as lit_dataset
from lit_nlp.api import dtypes
from lit_nlp.api import model as lit_model
from lit_nlp.api import types
from lit_nlp.components.citrus import lime
from lit_nlp.components.citrus import utils as citrus_util
from lit_nlp.lib import utils

import numpy as np

JsonDict = types.JsonDict
Spec = types.Spec

def new_example(original_example: JsonDict, field: str, new_value: Any):
  """Deep copies the example and replaces `field` with `new_value`."""
  example = copy.deepcopy(original_example)
  example[field] = new_value
  return example

def _predict_fn(strings: Iterable[str], model: Any, original_example: JsonDict,
                text_key: str, pred_key: str):
  """Given raw strings, return probabilities. Used by `lime.explain`."""
  # Prepare example objects to be fed to the model for each sentence/string.
  input_examples = [new_example(original_example, text_key, s) for s in strings]

  # Get model predictions for the examples.
  model_outputs = model.predict(input_examples)
  outputs = np.array([output[pred_key] for output in model_outputs])
  # Make outputs 1D in case of regression or binary classification.
  if outputs.ndim == 2 and outputs.shape[1] == 1:
    outputs = np.squeeze(outputs, axis=1)
  # <float32>[len(strings)] or <float32>[len(strings), num_labels].
  return outputs

class LIME(lit_components.Interpreter):
  """Local Interpretable Model-agnostic Explanations (LIME)."""

  def __init__(self):
    pass

  def run(
      self,
      inputs: List[JsonDict],
      model: lit_model.Model,
      dataset: lit_dataset.Dataset,
      model_outputs: Optional[List[JsonDict]] = None,
      config: Optional[JsonDict] = None,
      kernel_width: int = 25,  # TODO(lit-dev): make configurable in UI.
      mask_string: str = '[MASK]',  # TODO(lit-dev): make configurable in UI.
      num_samples: int = 256,  # TODO(lit-dev): make configurable in UI.
      class_to_explain: Optional[int] = 1,  # TODO(lit-dev): b/173469699.
      seed: Optional[int] = None,  # TODO(lit-dev): make configurable in UI.
  ) -> Optional[List[JsonDict]]:
    """Run this component, given a model and input(s)."""

    # Find keys of input (text) segments to explain.
    # Search in the input spec, since it's only useful to look at ones that are
    # used by the model.
    text_keys = utils.find_spec_keys(model.input_spec(), types.TextSegment)
    if not text_keys:
      logging.warning('LIME requires text inputs.')
      return None
    logging.info('Found text fields for LIME attribution: %s', str(text_keys))

    # Find the key of output probabilities field(s).
    pred_keys = utils.find_spec_keys(
        model.output_spec(), (types.MulticlassPreds, types.RegressionScore))
    if not pred_keys:
      logging.warning('LIME did not find any supported output fields.')
      return None

    pred_key = pred_keys[0]  # TODO(lit-dev): configure which prob field to use.
    all_results = []

    # Explain each input.
    for input_ in inputs:
      # Dict[field name -> interpretations]
      result = {}
      predict_fn = functools.partial(
          _predict_fn, model=model, original_example=input_, pred_key=pred_key)

      # Explain each text segment in the input, keeping the others constant.
      for text_key in text_keys:
        input_string = input_[text_key]
        logging.info('Explaining: %s', input_string)

        # Perturbs the input string, gets model predictions, fits linear model.
        explanation = lime.explain(
            sentence=input_string,
            predict_fn=functools.partial(predict_fn, text_key=text_key),
            # `class_to_explain` is ignored when predict_fn output is a scalar.
            class_to_explain=class_to_explain,  # Index of the class to explain.
            num_samples=num_samples,
            tokenizer=str.split,
            mask_token=mask_string,
            kernel=functools.partial(
                lime.exponential_kernel, kernel_width=kernel_width),
            seed=seed)

        # Turn the LIME explanation into a list following original word order.
        scores = explanation.feature_importance
        # TODO(lit-dev): Move score normalization to the UI.
        scores = citrus_util.normalize_scores(scores)
        result[text_key] = dtypes.SalienceMap(input_string.split(), scores)

      all_results.append(result)

    return all_results
<commit_msg>Fix LIME failing for optional empty input fields.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Lint as: python3
"""Gradient-based attribution."""

import copy
import functools
from typing import Any, Iterable, List, Optional

from absl import logging
from lit_nlp.api import components as lit_components
from lit_nlp.api import dataset as lit_dataset
from lit_nlp.api import dtypes
from lit_nlp.api import model as lit_model
from lit_nlp.api import types
from lit_nlp.components.citrus import lime
from lit_nlp.components.citrus import utils as citrus_util
from lit_nlp.lib import utils

import numpy as np

JsonDict = types.JsonDict
Spec = types.Spec

def new_example(original_example: JsonDict, field: str, new_value: Any):
  """Deep copies the example and replaces `field` with `new_value`."""
  example = copy.deepcopy(original_example)
  example[field] = new_value
  return example

def _predict_fn(strings: Iterable[str], model: Any, original_example: JsonDict,
                text_key: str, pred_key: str):
  """Given raw strings, return probabilities. Used by `lime.explain`."""
  # Prepare example objects to be fed to the model for each sentence/string.
  input_examples = [new_example(original_example, text_key, s) for s in strings]

  # Get model predictions for the examples.
  model_outputs = model.predict(input_examples)
  outputs = np.array([output[pred_key] for output in model_outputs])
  # Make outputs 1D in case of regression or binary classification.
  if outputs.ndim == 2 and outputs.shape[1] == 1:
    outputs = np.squeeze(outputs, axis=1)
  # <float32>[len(strings)] or <float32>[len(strings), num_labels].
  return outputs

class LIME(lit_components.Interpreter):
  """Local Interpretable Model-agnostic Explanations (LIME)."""

  def __init__(self):
    pass

  def run(
      self,
      inputs: List[JsonDict],
      model: lit_model.Model,
      dataset: lit_dataset.Dataset,
      model_outputs: Optional[List[JsonDict]] = None,
      config: Optional[JsonDict] = None,
      kernel_width: int = 25,  # TODO(lit-dev): make configurable in UI.
      mask_string: str = '[MASK]',  # TODO(lit-dev): make configurable in UI.
      num_samples: int = 256,  # TODO(lit-dev): make configurable in UI.
      class_to_explain: Optional[int] = 1,  # TODO(lit-dev): b/173469699.
      seed: Optional[int] = None,  # TODO(lit-dev): make configurable in UI.
  ) -> Optional[List[JsonDict]]:
    """Run this component, given a model and input(s)."""

    # Find keys of input (text) segments to explain.
    # Search in the input spec, since it's only useful to look at ones that are
    # used by the model.
    text_keys = utils.find_spec_keys(model.input_spec(), types.TextSegment)
    if not text_keys:
      logging.warning('LIME requires text inputs.')
      return None
    logging.info('Found text fields for LIME attribution: %s', str(text_keys))

    # Find the key of output probabilities field(s).
    pred_keys = utils.find_spec_keys(
        model.output_spec(), (types.MulticlassPreds, types.RegressionScore))
    if not pred_keys:
      logging.warning('LIME did not find any supported output fields.')
      return None

    pred_key = pred_keys[0]  # TODO(lit-dev): configure which prob field to use.
    all_results = []

    # Explain each input.
    for input_ in inputs:
      # Dict[field name -> interpretations]
      result = {}
      predict_fn = functools.partial(
          _predict_fn, model=model, original_example=input_, pred_key=pred_key)

      # Explain each text segment in the input, keeping the others constant.
      for text_key in text_keys:
        input_string = input_[text_key]
        if not input_string:
          logging.info('Could not explain empty string for %s', text_key)
          continue
        logging.info('Explaining: %s', input_string)

        # Perturbs the input string, gets model predictions, fits linear model.
        explanation = lime.explain(
            sentence=input_string,
            predict_fn=functools.partial(predict_fn, text_key=text_key),
            # `class_to_explain` is ignored when predict_fn output is a scalar.
            class_to_explain=class_to_explain,  # Index of the class to explain.
            num_samples=num_samples,
            tokenizer=str.split,
            mask_token=mask_string,
            kernel=functools.partial(
                lime.exponential_kernel, kernel_width=kernel_width),
            seed=seed)

        # Turn the LIME explanation into a list following original word order.
        scores = explanation.feature_importance
        # TODO(lit-dev): Move score normalization to the UI.
        scores = citrus_util.normalize_scores(scores)
        result[text_key] = dtypes.SalienceMap(input_string.split(), scores)

      all_results.append(result)

    return all_results
<|endoftext|>