<commit_before>import re

with open('./latin-analyses.txt') as file_opened:
    string_raw = file_opened.read()
    string_rows = string_raw.splitlines()
    latin_analyses_dict = {}
    for row in string_rows[0:300]:
        headword = row.split('\t', 1)[0]
        #print(headword)
        analyses_string = row.split('\t', 1)[1]
        reg_bracket = re.compile('\{.*?\}')
        analyses = reg_bracket.findall(analyses_string)
        perseus_pos = {}
        for analysis in analyses:
            parts = analysis.split('\t')
            first = parts[0][1:]
            gloss = parts[1]
            third = parts[2][:-1]
            reg_digits = re.compile('\w+')
            perseus_headword_id = reg_digits.findall(first)[0]
            perseus_pos_id = reg_digits.findall(first)[1]
            persues_parsed = reg_digits.findall(first)[2]
            try:
                perseus_headword = reg_digits.findall(first)[3]
            except:
                pass
            #print(third)
            pos = third.split(' ')
            print(pos[0])
            '''
            if pos[0] == 'gerundive':
                perseus_pos['GERUNDIVE'] = pos[0]
            elif pos[0] == 'pres' or 'imperf' or 'fut' or 'perf' or 'plup':
                perseus_pos['VERB'] = pos[0]
            elif pos[0] == 'fem' or 'masc' or 'neut':
                perseus_pos['Noun'] = pos[0]
            '''

            #print(perseus_headword_id)
            #print(perseus_pos_id)
            #for part in parts:
            #print(part)
            #print(splitted)
            #reg_digits = re.compile('\d*')
            #zero = reg_digits.findall(splitted)
            #little_number = reg_digits.findall(splitted)[3]
            #print(zero)
            #print(little_number)

        '''
        reg_headword = re.compile('\w+')
        headword = reg_headword.findall(row)[0]
        print(headword)
        reg_bracket = re.compile('\{.*?\}')
        def_brackets = reg_bracket.findall(row)
        headword_dict = {}
        for bracket in def_brackets:
            #print(bracket)
            reg_digits = re.compile('\d*')
            big_number = reg_digits.findall(bracket)[1]
            little_number = reg_digits.findall(bracket)[3]
            #print(big_number)
            #print(little_number)
            reg_alphnum = re.compile('[a-zA-Z0-9_\,\-]*')
            keys = reg_alphnum.findall(bracket)[5]#this actually point to the headwords they belong to, I think; these needs to be parsed and looped
            #print(inflections)
            pos_def = reg_alphnum.findall(bracket)[6:]#this actually has the english def in there too, which needs to be removed
            #pos_filtered = [x for x in pos_def if x is not '']
            pos_def_filtered = list(filter(''.__ne__, pos_def))
            #print(pos_filtered)
            headword_dict['big_number'] = big_number
            headword_dict['little_number'] = little_number
            headword_dict['pos_def'] = pos_def_filtered
            print(headword_dict)
        '''
<commit_msg>pos[0] parsed<commit_after>import re

with open('./latin-analyses.txt') as file_opened:
    string_raw = file_opened.read()
    string_rows = string_raw.splitlines()
    latin_analyses_dict = {}
    for row in string_rows:
        headword = row.split('\t', 1)[0]
        #print(headword)
        analyses_string = row.split('\t', 1)[1]
        reg_bracket = re.compile('\{.*?\}')
        analyses = reg_bracket.findall(analyses_string)
        perseus_pos = {}
        for analysis in analyses:
            parts = analysis.split('\t')
            first = parts[0][1:]
            gloss = parts[1]
            third = parts[2][:-1]
            reg_digits = re.compile('\w+')
            perseus_headword_id = reg_digits.findall(first)[0]
            perseus_pos_id = reg_digits.findall(first)[1]
            persues_parsed = reg_digits.findall(first)[2]
            try:
                perseus_headword = reg_digits.findall(first)[3]
            except:
                pass
            #print(third)
            pos = third.split(' ')
            #print(pos[0])
            verb = {}
            if pos[0] in ('pres', 'imperf', 'fut', 'perf', 'plup', 'futperf'):
                verb['tense'] = pos[0]
                perseus_pos['type'] == 'verb'
                perseus_pos['pos'] == verb
            elif pos[0] == 'gerundive':
                pass
            elif pos[0] in ('fem', 'masc', 'neut', 'masc/fem/neut', 'masc/fem', 'masc/neut'):
                pass
            elif pos[0] == 'supine':
                pass
            elif pos[0] == 'indeclform':
                pass
            elif pos[0] == 'adverbial':
                pass
            elif pos[0] in ('nom', 'abl', 'gen','dat', 'nom/acc', 'nom/voc'):
                pass
            elif pos[0] == 'sg':
                pass #??? ex: ut, uter, altus, alter
            elif pos[0] == 'comp':
                pass # ex: diu, se
            elif pos[0] in ('subj', 'ind'):
                pass # ex: fio, impleo, deleo, compleo
            elif pos[0] == 'nu_movable':
                pass #only 1 ex: sum
            else:
                print(perseus_headword,pos)
            '''
            if pos[0] == 'gerundive':
                perseus_pos['GERUNDIVE'] = pos[0]
                #print('found gerundive')
            elif pos[0] == 'pres' or 'imperf' or 'fut' or 'perf' or 'plup' or 'futperf':
                perseus_pos['VERB'] = pos[0]
                print('found a verb')
            elif pos[0] == 'fem' or 'masc' or 'neut':
                perseus_pos['Noun'] = pos[0]
                #print('found a noun')
            elif pos[0] == 'fem' or 'masc' or 'neut':
                perseus_pos['Noun'] = pos[0]
            else:
                print(pos[0])
                #print('found nothing')
            '''

            #indeclform

            #print(perseus_headword_id)
            #print(perseus_pos_id)
            #for part in parts:
            #print(part)
            #print(splitted)
            #reg_digits = re.compile('\d*')
            #zero = reg_digits.findall(splitted)
            #little_number = reg_digits.findall(splitted)[3]
            #print(zero)
            #print(little_number)

        '''
        reg_headword = re.compile('\w+')
        headword = reg_headword.findall(row)[0]
        print(headword)
        reg_bracket = re.compile('\{.*?\}')
        def_brackets = reg_bracket.findall(row)
        headword_dict = {}
        for bracket in def_brackets:
            #print(bracket)
            reg_digits = re.compile('\d*')
            big_number = reg_digits.findall(bracket)[1]
            little_number = reg_digits.findall(bracket)[3]
            #print(big_number)
            #print(little_number)
            reg_alphnum = re.compile('[a-zA-Z0-9_\,\-]*')
            keys = reg_alphnum.findall(bracket)[5]#this actually point to the headwords they belong to, I think; these needs to be parsed and looped
            #print(inflections)
            pos_def = reg_alphnum.findall(bracket)[6:]#this actually has the english def in there too, which needs to be removed
            #pos_filtered = [x for x in pos_def if x is not '']
            pos_def_filtered = list(filter(''.__ne__, pos_def))
            #print(pos_filtered)
            headword_dict['big_number'] = big_number
            headword_dict['little_number'] = little_number
            headword_dict['pos_def'] = pos_def_filtered
            print(headword_dict)
        '''
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechSegmentGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import Adam
from scipy.spatial.distance import pdist
from .triplet_loss import TripletLoss
from pyannote.metrics.binary_classification import det_curve

class WTFTripletLoss(TripletLoss):
    """

    Parameters
    ----------
    variant : int, optional
        Loss variants. Defaults to 1.
    duration : float, optional
        Defautls to 3.2 seconds.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    sampling : {'all', 'hard', 'negative'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    trim : float, optional
        Do not use speech segments that are that close to the beginning/end of
        the annotated region. Useful when features are short-term normalized,
        as this can result in weird feature distribution at the boundaries.
        Defaults to 0s (i.e. do no trim).
    parallel : int, optional
        Number of prefetching background generators. Defaults to 1.
        Each generator will prefetch enough batches to cover a whole epoch.
        Set `parallel` to 0 to not use background generators.
    """

    def __init__(self, variant=1, duration=3.2, sampling='all',
                 per_label=3, per_fold=None, trim=0, parallel=1):

        super(WTFTripletLoss, self).__init__(
            duration=duration, metric='angular', clamp='sigmoid',
            sampling=sampling, per_label=per_label, per_fold=per_fold,
            trim=trim, parallel=parallel)

        self.variant = variant

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            epochs=1000, restart=0, gpu=False):

        import tensorboardX
        writer = tensorboardX.SummaryWriter(log_dir=log_dir)

        logging_callback = LoggingCallbackPytorch(log_dir=log_dir,
                                                  restart=restart > 0)

        batch_generator = SpeechSegmentGenerator(
            feature_extraction,
            per_label=self.per_label, per_fold=self.per_fold,
            duration=self.duration, trim=self.trim, parallel=self.parallel)
        batches = batch_generator(protocol, subset=subset)
        batch = next(batches)

        batches_per_epoch = batch_generator.batches_per_epoch

        if restart > 0:
            weights_pt = logging_callback.WEIGHTS_PT.format(
                log_dir=log_dir, epoch=restart)
            model.load_state_dict(torch.load(weights_pt))

        if gpu:
            model = model.cuda()

        model.internal = False

        parameters = list(model.parameters())

        if self.variant == 2:

            # rolling estimate of embedding norm distribution
            self.normalize_norm_ = nn.BatchNorm1d(
                1, eps=1e-5, momentum=0.1, affine=False)
            self.normalize_delta_ = nn.BatchNorm1d(
                1, eps=1e-5, momentum=0.1, affine=True)
            if gpu:
                self.normalize_norm_ = self.normalize_norm_.cuda()
                self.normalize_delta_ = self.normalize_delta_.cuda()
            parameters += list(self.normalize_norm_.parameters()) + \
                          list(self.normalize_delta_.parameters())

        optimizer = Adam(parameters)
        if restart > 0:
            optimizer_pt = logging_callback.OPTIMIZER_PT.format(
                log_dir=log_dir, epoch=restart)
            optimizer.load_state_dict(torch.load(optimizer_pt))
            if gpu:
                for state in optimizer.state.values():
                    for k, v in state.items():
                        if torch.is_tensor(v):
                            state[k] = v.cuda()

        epoch = restart if restart > 0 else -1
        while True:
            epoch += 1
            if epoch > epochs:
                break

            tloss_avg, closs_avg = 0., 0.

            if epoch % 10 == 0:
                positive, negative = [], []
                norms = []

            desc = 'Epoch #{0}'.format(epoch)
            for i in tqdm(range(batches_per_epoch), desc=desc):

                model.zero_grad()

                batch = next(batches)

                X = Variable(torch.from_numpy(
                    np.array(np.rollaxis(batch['X'], 0, 2),
                             dtype=np.float32)))

                if gpu:
                    X = X.cuda()

                fX = model(X)

                if epoch % 10 == 0:
                    if gpu:
                        fX_ = fX.data.cpu().numpy()
                    else:
                        fX_ = fX.data.numpy()
                    norms.append(np.linalg.norm(fX_, axis=1))

                # pre-compute pairwise distances
                distances = self.pdist(fX)

                if epoch % 10 == 0:
                    if gpu:
                        distances_ = distances.data.cpu().numpy()
                    else:
                        distances_ = distances.data.numpy()
                    is_positive = pdist(batch['y'].reshape((-1, 1)), metric='chebyshev') < 1
                    positive.append(distances_[np.where(is_positive)])
                    negative.append(distances_[np.where(~is_positive)])

                # sample triplets
                triplets = getattr(self, 'batch_{0}'.format(self.sampling))
                anchors, positives, negatives = triplets(batch['y'], distances)

                # compute triplet loss
                tlosses, deltas = self.triplet_loss(
                    distances, anchors, positives, negatives,
                    return_delta=True)

                tloss = torch.mean(tlosses)

                if self.variant == 1:
                    # compute wtf loss
                    closses = F.sigmoid(
                        F.softsign(deltas) * torch.norm(fX[anchors], 2, 1, keepdim=True))

                elif self.variant == 2:

                    anchor_norms = torch.norm(fX[anchors], 2, 1, keepdim=True)
                    anchor_norms = self.normalize_norm_(anchor_norms)
                    confidence = F.sigmoid(anchor_norms)

                    deltas = self.normalize_delta_(deltas)
                    correctness = F.sigmoid(-deltas)

                    closses = torch.abs(confidence - correctness)

                closs = torch.mean(closses)

                # log loss
                if gpu:
                    tloss_ = float(tloss.data.cpu().numpy())
                    closs_ = float(closs.data.cpu().numpy())
                else:
                    tloss_ = float(tloss.data.numpy())
                    closs_ = float(closs.data.numpy())
                tloss_avg += tloss_
                closs_avg += closs_

                loss = tloss + closs
                loss.backward()
                optimizer.step()

            tloss_avg /= batches_per_epoch
            writer.add_scalar('tloss', tloss_avg, global_step=epoch)

            closs_avg /= batches_per_epoch
            writer.add_scalar('closs', closs_avg, global_step=epoch)

            if epoch % 10 == 0:

                positive = np.hstack(positive)
                negative = np.hstack(negative)
                writer.add_histogram(
                    'embedding/pairwise_distance/positive', positive,
                    global_step=epoch, bins='auto')
                writer.add_histogram(
                    'embedding/pairwise_distance/negative', negative,
                    global_step=epoch, bins='auto')

                _, _, _, eer = det_curve(
                    np.hstack([np.ones(len(positive)), np.zeros(len(negative))]),
                    np.hstack([positive, negative]), distances=True)
                writer.add_scalar('eer', eer, global_step=epoch)

                norms = np.hstack(norms)
                writer.add_histogram(
                    'embedding/norm', norms,
                    global_step=epoch, bins='auto')

            logging_callback.model = model
            logging_callback.optimizer = optimizer
            logging_callback.on_epoch_end(epoch)
<commit_msg>feat: log loss<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2018 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechSegmentGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import Adam
from scipy.spatial.distance import pdist
from .triplet_loss import TripletLoss
from pyannote.metrics.binary_classification import det_curve

class WTFTripletLoss(TripletLoss):
    """

    Parameters
    ----------
    variant : int, optional
        Loss variants. Defaults to 1.
    duration : float, optional
        Defautls to 3.2 seconds.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    sampling : {'all', 'hard', 'negative'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    trim : float, optional
        Do not use speech segments that are that close to the beginning/end of
        the annotated region. Useful when features are short-term normalized,
        as this can result in weird feature distribution at the boundaries.
        Defaults to 0s (i.e. do no trim).
    parallel : int, optional
        Number of prefetching background generators. Defaults to 1.
        Each generator will prefetch enough batches to cover a whole epoch.
        Set `parallel` to 0 to not use background generators.
    """

    def __init__(self, variant=1, duration=3.2, sampling='all',
                 per_label=3, per_fold=None, trim=0, parallel=1):

        super(WTFTripletLoss, self).__init__(
            duration=duration, metric='angular', clamp='sigmoid',
            sampling=sampling, per_label=per_label, per_fold=per_fold,
            trim=trim, parallel=parallel)

        self.variant = variant

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            epochs=1000, restart=0, gpu=False):

        import tensorboardX
        writer = tensorboardX.SummaryWriter(log_dir=log_dir)

        logging_callback = LoggingCallbackPytorch(log_dir=log_dir,
                                                  restart=restart > 0)

        batch_generator = SpeechSegmentGenerator(
            feature_extraction,
            per_label=self.per_label, per_fold=self.per_fold,
            duration=self.duration, trim=self.trim, parallel=self.parallel)
        batches = batch_generator(protocol, subset=subset)
        batch = next(batches)

        batches_per_epoch = batch_generator.batches_per_epoch

        if restart > 0:
            weights_pt = logging_callback.WEIGHTS_PT.format(
                log_dir=log_dir, epoch=restart)
            model.load_state_dict(torch.load(weights_pt))

        if gpu:
            model = model.cuda()

        model.internal = False

        parameters = list(model.parameters())

        if self.variant == 2:

            # rolling estimate of embedding norm distribution
            self.normalize_norm_ = nn.BatchNorm1d(
                1, eps=1e-5, momentum=0.1, affine=False)
            self.normalize_delta_ = nn.BatchNorm1d(
                1, eps=1e-5, momentum=0.1, affine=True)
            if gpu:
                self.normalize_norm_ = self.normalize_norm_.cuda()
                self.normalize_delta_ = self.normalize_delta_.cuda()
            parameters += list(self.normalize_norm_.parameters()) + \
                          list(self.normalize_delta_.parameters())

        optimizer = Adam(parameters)
        if restart > 0:
            optimizer_pt = logging_callback.OPTIMIZER_PT.format(
                log_dir=log_dir, epoch=restart)
            optimizer.load_state_dict(torch.load(optimizer_pt))
            if gpu:
                for state in optimizer.state.values():
                    for k, v in state.items():
                        if torch.is_tensor(v):
                            state[k] = v.cuda()

        epoch = restart if restart > 0 else -1
        while True:
            epoch += 1
            if epoch > epochs:
                break

            loss_avg, tloss_avg, closs_avg = 0., 0., 0.

            if epoch % 10 == 0:
                positive, negative = [], []
                norms = []

            desc = 'Epoch #{0}'.format(epoch)
            for i in tqdm(range(batches_per_epoch), desc=desc):

                model.zero_grad()

                batch = next(batches)

                X = Variable(torch.from_numpy(
                    np.array(np.rollaxis(batch['X'], 0, 2),
                             dtype=np.float32)))

                if gpu:
                    X = X.cuda()

                fX = model(X)

                if epoch % 10 == 0:
                    if gpu:
                        fX_ = fX.data.cpu().numpy()
                    else:
                        fX_ = fX.data.numpy()
                    norms.append(np.linalg.norm(fX_, axis=1))

                # pre-compute pairwise distances
                distances = self.pdist(fX)

                if epoch % 10 == 0:
                    if gpu:
                        distances_ = distances.data.cpu().numpy()
                    else:
                        distances_ = distances.data.numpy()
                    is_positive = pdist(batch['y'].reshape((-1, 1)), metric='chebyshev') < 1
                    positive.append(distances_[np.where(is_positive)])
                    negative.append(distances_[np.where(~is_positive)])

                # sample triplets
                triplets = getattr(self, 'batch_{0}'.format(self.sampling))
                anchors, positives, negatives = triplets(batch['y'], distances)

                # compute triplet loss
                tlosses, deltas = self.triplet_loss(
                    distances, anchors, positives, negatives,
                    return_delta=True)

                tloss = torch.mean(tlosses)

                if self.variant == 1:
                    # compute wtf loss
                    closses = F.sigmoid(
                        F.softsign(deltas) * torch.norm(fX[anchors], 2, 1, keepdim=True))

                elif self.variant == 2:

                    anchor_norms = torch.norm(fX[anchors], 2, 1, keepdim=True)
                    anchor_norms = self.normalize_norm_(anchor_norms)
                    confidence = F.sigmoid(anchor_norms)

                    deltas = self.normalize_delta_(deltas)
                    correctness = F.sigmoid(-deltas)

                    closses = torch.abs(confidence - correctness)

                closs = torch.mean(closses)

                # log loss
                if gpu:
                    tloss_ = float(tloss.data.cpu().numpy())
                    closs_ = float(closs.data.cpu().numpy())
                else:
                    tloss_ = float(tloss.data.numpy())
                    closs_ = float(closs.data.numpy())
                tloss_avg += tloss_
                closs_avg += closs_
                loss_avg += tloss_ + closs_

                loss = tloss + closs
                loss.backward()
                optimizer.step()

            tloss_avg /= batches_per_epoch
            writer.add_scalar('tloss', tloss_avg, global_step=epoch)

            closs_avg /= batches_per_epoch
            writer.add_scalar('closs', closs_avg, global_step=epoch)

            loss_avg /= batches_per_epoch
            writer.add_scalar('loss', loss_avg, global_step=epoch)

            if epoch % 10 == 0:

                positive = np.hstack(positive)
                negative = np.hstack(negative)
                writer.add_histogram(
                    'embedding/pairwise_distance/positive', positive,
                    global_step=epoch, bins='auto')
                writer.add_histogram(
                    'embedding/pairwise_distance/negative', negative,
                    global_step=epoch, bins='auto')

                _, _, _, eer = det_curve(
                    np.hstack([np.ones(len(positive)), np.zeros(len(negative))]),
                    np.hstack([positive, negative]), distances=True)
                writer.add_scalar('eer', eer, global_step=epoch)

                norms = np.hstack(norms)
                writer.add_histogram(
                    'embedding/norm', norms,
                    global_step=epoch, bins='auto')

            logging_callback.model = model
            logging_callback.optimizer = optimizer
            logging_callback.on_epoch_end(epoch)
<|endoftext|>


<commit_before>import logging
import os

from ..analysis import file_index
from ..messages import MessageType, MessageHeader, Timestamp, message_type_to_class
from ..utils.time_range import TimeRange

class MixedLogReader(object):
    """!
    @brief Generator class for decoding FusionEngine messages contained in a mixed-content binary file.

    For real-time deserialization of an incoming binary data stream, see @ref FusionEngineDecoder.
    """
    logger = logging.getLogger('point_one.fusion_engine.parsers.mixed_log_reader')

    def __init__(self, input_file, warn_on_gaps: bool = False,
                 generate_index: bool = True, ignore_index: bool = False,
                 time_range: TimeRange = None, message_types: set = None,
                 return_header: bool = True, return_payload: bool = True,
                 return_bytes: bool = False, return_offset: bool = False):
        """!
        @brief Construct a new generator instance.

        Each call to @ref next() will return a tuple containing any/all of the message header, payload, serialized
        `bytes`, and byte offset depending on the values of the `return_*` parameters.

        @param input_file The path to an input file (`.p1log` or mixed-content binary file), or an open file-like
               object.
        @param warn_on_gaps If `True`, print warnings if gaps are detected in the FusionEngine message sequence numbers.
        @param generate_index If `True`, generate an index file if one does not exist for faster reading in the future.
               See @ref FileIndex for details.
        @param ignore_index If `True`, ignore the existing index file and read from the binary file directly. If
               `generate_index == True`, this will delete the existing file and create a new one.
        @param return_header If `True`, return the decoded @ref MessageHeader for each message.
        @param return_payload If `True`, parse and return the payload for each message as a subclass of @ref
               MessagePayload. Will return `None` if the payload cannot be parsed.
        @param return_bytes If `True`, return a `bytes` object containing the serialized message header and payload.
        @param return_offset If `True`, return the offset into the file (in bytes) at which the message began.
        """
        self.warn_on_gaps = warn_on_gaps

        self.return_header = return_header
        self.return_payload = return_payload
        self.return_bytes = return_bytes
        self.return_offset = return_offset

        self.time_range = time_range
        self.message_types = message_types

        self.valid_count = 0
        self.message_counts = {}
        self.prev_sequence_number = None
        self.total_bytes_read = 0

        # Open the file to be read.
        if isinstance(input_file, str):
            self.input_file = open(input_file, 'rb')
        else:
            self.input_file = input_file

        # Open the companion index file if one exists.
        input_path = self.input_file.name
        self.index_path = file_index.FileIndex.get_path(input_path)
        self.next_index_elem = 0
        if ignore_index:
            self.index = None
            self.index_builder = file_index.FileIndexBuilder() if generate_index else None
        else:
            if os.path.exists(self.index_path):
                try:
                    self.index = file_index.FileIndex(index_path=self.index_path, data_path=input_path,
                                                      delete_on_error=generate_index)
                    self.index = self.index[self.message_types][self.time_range]
                    self.index_builder = None
                except ValueError as e:
                    self.logger.error("Error loading index file: %s" % str(e))
                    self.index = None
                    self.index_builder = file_index.FileIndexBuilder() if generate_index else None

        if self.index_builder is not None:
            self.logger.debug("Generating index file '%s'." % self.index_path)

    def have_index(self):
        return self.index is not None

    def get_index(self):
        return self.index

    def generating_index(self):
        return self.index_builder is not None

    def get_bytes_read(self):
        return self.total_bytes_read

    def next(self):
        while True:
            if not self._advance_to_next_sync():
                # End of file.
                self.logger.debug('EOF reached.')
                break

            offset_bytes = self.input_file.tell()
            self.total_bytes_read = offset_bytes
            self.logger.trace('Reading candidate message @ %d (0x%x).' % (offset_bytes, offset_bytes), depth=2)

            # Read the next message header.
            data = self.input_file.read(MessageHeader.calcsize())
            read_len = len(data)
            if read_len < MessageHeader.calcsize():
                # End of file.
                self.logger.debug('EOF reached.')
                break

            try:
                header = MessageHeader()
                header.unpack(data, warn_on_unrecognized=False)

                # Check if the payload is too big.
                if header.payload_size_bytes > MessageHeader._MAX_EXPECTED_SIZE_BYTES:
                    raise ValueError('Payload size (%d) too large.' % header.payload_size_bytes)

                # Read and validate the payload.
                payload = self.input_file.read(header.payload_size_bytes)
                read_len += len(payload)
                if len(payload) != header.payload_size_bytes:
                    raise ValueError('Not enough data - likely not a valid FusionEngine header.')

                data += payload
                header.validate_crc(data)

                self.logger.trace('Read %s message @ %d (0x%x). [length=%d B, sequence=%d, # messages=%d]' %
                                  (header.get_type_string(), offset_bytes, offset_bytes,
                                   MessageHeader.calcsize() + header.payload_size_bytes, header.sequence_number,
                                   self.valid_count + 1),
                                  depth=1)

                self.valid_count += 1
                self.message_counts.setdefault(header.message_type, 0)
                self.message_counts[header.message_type] += 1

                # Check for sequence number gaps.
                if self.prev_sequence_number is not None and \
                   (header.sequence_number - self.prev_sequence_number) != 1 and \
                   not (header.sequence_number == 0 and self.prev_sequence_number == 0xFFFFFFFF):
                    func = self.logger.warning if self.warn_on_gaps else self.logger.debug
                    func('Data gap detected @ %d (0x%x). [sequence=%d, gap_size=%d, total_messages=%d]' %
                         (offset_bytes, offset_bytes, header.sequence_number,
                          header.sequence_number - self.prev_sequence_number, self.valid_count + 1))
                self.prev_sequence_number = header.sequence_number

                # Deserialize the payload if we need it.
                need_payload = self.return_payload or self.index_builder is not None or self.time_range is not None
                if need_payload:
                    cls = message_type_to_class.get(header.message_type, None)
                    if cls is not None:
                        try:
                            payload = cls()
                        except Exception as e:
                            self.logger.error("Error parsing %s message: %s" % (header.get_type_string(), str(e)))
                            payload = None
                    else:
                        payload = None

                # Add this message to the index file.
                if self.index_builder is not None:
                    p1_time = payload.get_p1_time() if payload is not None else None
                    self.index_builder.append(message_type=header.message_type, offset_bytes=offset_bytes,
                                              p1_time=p1_time)

                # Now, if this message is not in the user-specified filter criteria, skip it.
                if self.message_types is not None and header.message_type not in self.message_types:
                    self.logger.debug("Message type not requested. Skipping.")
                    continue
                elif self.time_range is not None and not self.time_range.is_in_range(payload):
                    if self.time_range.in_range_started() and self.index_builder is None:
                        self.logger.debug("End of time range reached. Finished processing.")
                        break
                    else:
                        self.logger.debug("Message not in time range. Skipping.")
                        continue

                # Construct the result. If we're returning the payload, deserialize the payload.
                result = []
                if self.return_header:
                    result.append(header)
                if self.return_payload:
                    result.append(payload)
                if self.return_bytes:
                    result.append(data)
                if self.return_offset:
                    result.append(offset_bytes)
                return result
            except ValueError as e:
                offset_bytes += 1
                self.logger.trace('%s Rewinding to offset %d (0x%x).' % (str(e), offset_bytes, offset_bytes),
                                  depth=2)
                self.input_file.seek(offset_bytes, os.SEEK_SET)

        # Out of the loop - EOF reached. If we are creating an index file, save it now.
        if self.index_builder is not None:
            index = self.index_builder.to_index()
            index.save(self.index_path)

        # Finished iterating.
        self.input_file.seek(0, os.SEEK_END)
        self.total_bytes_read = self.input_file.tell()
        raise StopIteration()

    def _advance_to_next_sync(self):
        if self.index is None:
            try:
                while True:
                    byte0 = self.input_file.read(1)[0]
                    while True:
                        if byte0 == MessageHeader._SYNC0:
                            byte1 = self.input_file.read(1)[0]
                            if byte1 == MessageHeader._SYNC1:
                                self.input_file.seek(-2, os.SEEK_CUR)
                                return True
                            byte0 = byte1
                        else:
                            break
            except IndexError:
                return False
        else:
            if self.next_index_elem == len(self.index):
                return False
            else:
                offset_bytes = self.index._data['offset'][self.next_index_elem]
                self.next_index_elem += 1
                self.input_file.seek(offset_bytes, os.SEEK_SET)
                return True

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def __getitem__(self, key):
        """!
        @brief Limit the returned messages by type or time.

        @warning
        This operator modifies this class in-place.

        @param key One of the following:
               - An individual @ref MessageType to be returned
               - An iterable listing one or more @ref MessageType%s to be returned
               - A `slice` specifying the start/end of the desired absolute (P1) or relative time range
               - A @ref TimeRange object
        """
        # No key specified (convenience case).
        if key is None:
            pass
        # If we have an index file available, reduce the index to the requested criteria.
        elif self.index is not None:
            self.index = self.index[key]
        # Otherwise, store the criteria and apply them while reading.
        else:
            # Return entries for a specific message type.
            if isinstance(key, MessageType):
                self.message_types = set((key,))
            # Return entries for a list of message types.
            elif isinstance(key, (set, list, tuple)) and len(key) > 0 and isinstance(next(iter(key)), MessageType):
                current_types = set(self.message_types) if self.message_types is not None else set()
                self.message_types = current_types & set(key)
            # Key is a slice in time. Return a subset of the data.
            elif isinstance(key, slice) and (isinstance(key.start, (Timestamp, float)) or
                                             isinstance(key.stop, (Timestamp, float))):
                time_range = TimeRange(start=key.start, end=key.stop, absolute=isinstance(key.start, Timestamp))
                if self.time_range is None:
                    self.time_range = time_range
                else:
                    self.time_range.intersect(time_range, in_place=True)
            # Key is a TimeRange object. Return a subset of the data. All nan elements (messages without P1 time) will
            # be included in the results.
            elif isinstance(key, TimeRange):
                if self.time_range is None:
                    self.time_range = key
                else:
                    self.time_range.intersect(key, in_place=True)
        return self
<commit_msg>Fixed payload decode.<commit_after>import logging
import os

from ..analysis import file_index
from ..messages import MessageType, MessageHeader, Timestamp, message_type_to_class
from ..utils.time_range import TimeRange

class MixedLogReader(object):
    """!
    @brief Generator class for decoding FusionEngine messages contained in a mixed-content binary file.

    For real-time deserialization of an incoming binary data stream, see @ref FusionEngineDecoder.
    """
    logger = logging.getLogger('point_one.fusion_engine.parsers.mixed_log_reader')

    def __init__(self, input_file, warn_on_gaps: bool = False,
                 generate_index: bool = True, ignore_index: bool = False,
                 time_range: TimeRange = None, message_types: set = None,
                 return_header: bool = True, return_payload: bool = True,
                 return_bytes: bool = False, return_offset: bool = False):
        """!
        @brief Construct a new generator instance.

        Each call to @ref next() will return a tuple containing any/all of the message header, payload, serialized
        `bytes`, and byte offset depending on the values of the `return_*` parameters.

        @param input_file The path to an input file (`.p1log` or mixed-content binary file), or an open file-like
               object.
        @param warn_on_gaps If `True`, print warnings if gaps are detected in the FusionEngine message sequence numbers.
        @param generate_index If `True`, generate an index file if one does not exist for faster reading in the future.
               See @ref FileIndex for details.
        @param ignore_index If `True`, ignore the existing index file and read from the binary file directly. If
               `generate_index == True`, this will delete the existing file and create a new one.
        @param return_header If `True`, return the decoded @ref MessageHeader for each message.
        @param return_payload If `True`, parse and return the payload for each message as a subclass of @ref
               MessagePayload. Will return `None` if the payload cannot be parsed.
        @param return_bytes If `True`, return a `bytes` object containing the serialized message header and payload.
        @param return_offset If `True`, return the offset into the file (in bytes) at which the message began.
        """
        self.warn_on_gaps = warn_on_gaps

        self.return_header = return_header
        self.return_payload = return_payload
        self.return_bytes = return_bytes
        self.return_offset = return_offset

        self.time_range = time_range
        self.message_types = message_types

        self.valid_count = 0
        self.message_counts = {}
        self.prev_sequence_number = None
        self.total_bytes_read = 0

        # Open the file to be read.
        if isinstance(input_file, str):
            self.input_file = open(input_file, 'rb')
        else:
            self.input_file = input_file

        # Open the companion index file if one exists.
        input_path = self.input_file.name
        self.index_path = file_index.FileIndex.get_path(input_path)
        self.next_index_elem = 0
        if ignore_index:
            self.index = None
            self.index_builder = file_index.FileIndexBuilder() if generate_index else None
        else:
            if os.path.exists(self.index_path):
                try:
                    self.index = file_index.FileIndex(index_path=self.index_path, data_path=input_path,
                                                      delete_on_error=generate_index)
                    self.index = self.index[self.message_types][self.time_range]
                    self.index_builder = None
                except ValueError as e:
                    self.logger.error("Error loading index file: %s" % str(e))
                    self.index = None
                    self.index_builder = file_index.FileIndexBuilder() if generate_index else None

        if self.index_builder is not None:
            self.logger.debug("Generating index file '%s'." % self.index_path)

    def have_index(self):
        return self.index is not None

    def get_index(self):
        return self.index

    def generating_index(self):
        return self.index_builder is not None

    def get_bytes_read(self):
        return self.total_bytes_read

    def next(self):
        while True:
            if not self._advance_to_next_sync():
                # End of file.
                self.logger.debug('EOF reached.')
                break

            offset_bytes = self.input_file.tell()
            self.total_bytes_read = offset_bytes
            self.logger.trace('Reading candidate message @ %d (0x%x).' % (offset_bytes, offset_bytes), depth=2)

            # Read the next message header.
            data = self.input_file.read(MessageHeader.calcsize())
            read_len = len(data)
            if read_len < MessageHeader.calcsize():
                # End of file.
                self.logger.debug('EOF reached.')
                break

            try:
                header = MessageHeader()
                header.unpack(data, warn_on_unrecognized=False)

                # Check if the payload is too big.
                if header.payload_size_bytes > MessageHeader._MAX_EXPECTED_SIZE_BYTES:
                    raise ValueError('Payload size (%d) too large.' % header.payload_size_bytes)

                # Read and validate the payload.
                payload_bytes = self.input_file.read(header.payload_size_bytes)
                read_len += len(payload_bytes)
                if len(payload_bytes) != header.payload_size_bytes:
                    raise ValueError('Not enough data - likely not a valid FusionEngine header.')

                data += payload_bytes
                header.validate_crc(data)

                self.logger.trace('Read %s message @ %d (0x%x). [length=%d B, sequence=%d, # messages=%d]' %
                                  (header.get_type_string(), offset_bytes, offset_bytes,
                                   MessageHeader.calcsize() + header.payload_size_bytes, header.sequence_number,
                                   self.valid_count + 1),
                                  depth=1)

                self.valid_count += 1
                self.message_counts.setdefault(header.message_type, 0)
                self.message_counts[header.message_type] += 1

                # Check for sequence number gaps.
                if self.prev_sequence_number is not None and \
                   (header.sequence_number - self.prev_sequence_number) != 1 and \
                   not (header.sequence_number == 0 and self.prev_sequence_number == 0xFFFFFFFF):
                    func = self.logger.warning if self.warn_on_gaps else self.logger.debug
                    func('Data gap detected @ %d (0x%x). [sequence=%d, gap_size=%d, total_messages=%d]' %
                         (offset_bytes, offset_bytes, header.sequence_number,
                          header.sequence_number - self.prev_sequence_number, self.valid_count + 1))
                self.prev_sequence_number = header.sequence_number

                # Deserialize the payload if we need it.
                need_payload = self.return_payload or self.index_builder is not None or self.time_range is not None
                if need_payload:
                    cls = message_type_to_class.get(header.message_type, None)
                    if cls is not None:
                        try:
                            payload = cls()
                            payload.unpack(buffer=payload_bytes, offset=0)
                        except Exception as e:
                            self.logger.error("Error parsing %s message: %s" % (header.get_type_string(), str(e)))
                            payload = None
                    else:
                        payload = None

                # Add this message to the index file.
                if self.index_builder is not None:
                    p1_time = payload.get_p1_time() if payload is not None else None
                    self.index_builder.append(message_type=header.message_type, offset_bytes=offset_bytes,
                                              p1_time=p1_time)

                # Now, if this message is not in the user-specified filter criteria, skip it.
                if self.message_types is not None and header.message_type not in self.message_types:
                    self.logger.debug("Message type not requested. Skipping.")
                    continue
                elif self.time_range is not None and not self.time_range.is_in_range(payload):
                    if self.time_range.in_range_started() and self.index_builder is None:
                        self.logger.debug("End of time range reached. Finished processing.")
                        break
                    else:
                        self.logger.debug("Message not in time range. Skipping.")
                        continue

                # Construct the result. If we're returning the payload, deserialize the payload.
                result = []
                if self.return_header:
                    result.append(header)
                if self.return_payload:
                    result.append(payload)
                if self.return_bytes:
                    result.append(data)
                if self.return_offset:
                    result.append(offset_bytes)
                return result
            except ValueError as e:
                offset_bytes += 1
                self.logger.trace('%s Rewinding to offset %d (0x%x).' % (str(e), offset_bytes, offset_bytes),
                                  depth=2)
                self.input_file.seek(offset_bytes, os.SEEK_SET)

        # Out of the loop - EOF reached. If we are creating an index file, save it now.
        if self.index_builder is not None:
            index = self.index_builder.to_index()
            index.save(self.index_path)

        # Finished iterating.
        self.input_file.seek(0, os.SEEK_END)
        self.total_bytes_read = self.input_file.tell()
        raise StopIteration()

    def _advance_to_next_sync(self):
        if self.index is None:
            try:
                while True:
                    byte0 = self.input_file.read(1)[0]
                    while True:
                        if byte0 == MessageHeader._SYNC0:
                            byte1 = self.input_file.read(1)[0]
                            if byte1 == MessageHeader._SYNC1:
                                self.input_file.seek(-2, os.SEEK_CUR)
                                return True
                            byte0 = byte1
                        else:
                            break
            except IndexError:
                return False
        else:
            if self.next_index_elem == len(self.index):
                return False
            else:
                offset_bytes = self.index._data['offset'][self.next_index_elem]
                self.next_index_elem += 1
                self.input_file.seek(offset_bytes, os.SEEK_SET)
                return True

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def __getitem__(self, key):
        """!
        @brief Limit the returned messages by type or time.

        @warning
        This operator modifies this class in-place.

        @param key One of the following:
               - An individual @ref MessageType to be returned
               - An iterable listing one or more @ref MessageType%s to be returned
               - A `slice` specifying the start/end of the desired absolute (P1) or relative time range
               - A @ref TimeRange object
        """
        # No key specified (convenience case).
        if key is None:
            pass
        # If we have an index file available, reduce the index to the requested criteria.
        elif self.index is not None:
            self.index = self.index[key]
        # Otherwise, store the criteria and apply them while reading.
        else:
            # Return entries for a specific message type.
            if isinstance(key, MessageType):
                self.message_types = set((key,))
            # Return entries for a list of message types.
            elif isinstance(key, (set, list, tuple)) and len(key) > 0 and isinstance(next(iter(key)), MessageType):
                current_types = set(self.message_types) if self.message_types is not None else set()
                self.message_types = current_types & set(key)
            # Key is a slice in time. Return a subset of the data.
            elif isinstance(key, slice) and (isinstance(key.start, (Timestamp, float)) or
                                             isinstance(key.stop, (Timestamp, float))):
                time_range = TimeRange(start=key.start, end=key.stop, absolute=isinstance(key.start, Timestamp))
                if self.time_range is None:
                    self.time_range = time_range
                else:
                    self.time_range.intersect(time_range, in_place=True)
            # Key is a TimeRange object. Return a subset of the data. All nan elements (messages without P1 time) will
            # be included in the results.
            elif isinstance(key, TimeRange):
                if self.time_range is None:
                    self.time_range = key
                else:
                    self.time_range.intersect(key, in_place=True)
        return self
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.util.common import JavaValue
from zoo.common.utils import callZooFunc
from bigdl.nn.layer import Layer
from zoo.pipeline.api.keras.engine import KerasNet
import warnings

class InferenceModel(JavaValue):
    """
    Model for thread-safe inference.
    To do inference, you need to first initiate an InferenceModel instance, then call
    load|load_caffe|load_tf|load_openvino to load a pre-trained model, and finally call predict.

    # Arguments
    supported_concurrent_num: Int. How many concurrent threads to invoke. Default is 1.
    """

    def __init__(self, supported_concurrent_num=1, bigdl_type="float"):
        super(InferenceModel, self).__init__(None, bigdl_type, supported_concurrent_num)

    def load_bigdl(self, model_path, weight_path=None):
        """
        Load a pre-trained Analytics Zoo or BigDL model.

        :param model_path: String. The file path to the model.
        :param weight_path: String. The file path to the weights if any. Default is None.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadBigDL",
                    self.value, model_path, weight_path)

    # deprecated in "0.8.0"
    def load(self, model_path, weight_path=None):
        """
        Load a pre-trained Analytics Zoo or BigDL model.

        :param model_path: String. The file path to the model.
        :param weight_path: String. The file path to the weights if any. Default is None.
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type, "inferenceModelLoad",
                    self.value, model_path, weight_path)

    def load_caffe(self, model_path, weight_path):
        """
        Load a pre-trained Caffe model.

        :param model_path: String. The file path to the prototxt file.
        :param weight_path: String. The file path to the Caffe model.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadCaffe",
                    self.value, model_path, weight_path)

    def load_openvino(self, model_path, weight_path, batch_size=0):
        """
        Load an OpenVINI IR.

        :param model_path: String. The file path to the OpenVINO IR xml file.
        :param weight_path: String. The file path to the OpenVINO IR bin file.
        :param batch_size: Int. Set batch Size, default is 0 (use default batch size).
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadOpenVINO",
                    self.value, model_path, weight_path, batch_size)

    def load_tensorflow(self, model_path, model_type="frozenModel", intra_op_parallelism_threads=1,
                        inter_op_parallelism_threads=1, use_per_session_threads=True):
        """
        Load an TensorFlow model using tensorflow.

        :param model_path: String. The file path to the TensorFlow model.
        :param model_type: String. The type of the tensorflow model file. Default is "frozenModel"
        :param intra_op_parallelism_threads: Int. The number of intraOpParallelismThreads.
                                             Default is 1.
        :param inter_op_parallelism_threads: Int. The number of interOpParallelismThreads.
                                             Default is 1.
        :param use_per_session_threads: Boolean. Whether to use perSessionThreads. Default is True.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadTensorFlow",
                    self.value, model_path, model_type, intra_op_parallelism_threads,
                    inter_op_parallelism_threads, use_per_session_threads)

    # deprecated in "0.8.0"
    def load_tf(self, model_path, backend="tensorflow",
                intra_op_parallelism_threads=1, inter_op_parallelism_threads=1,
                use_per_session_threads=True, model_type=None,
                ov_pipeline_config_path=None, ov_extensions_config_path=None):
        """
        Load an TensorFlow model using tensorflow or openvino backend.

        :param model_path: String. The file path to the TensorFlow model.
        :param backend: String. The backend to use for inference. Either 'tensorflow' or 'openvino'.
                        For 'tensorflow' backend, only need to specify arguments
                        intra_op_parallelism_threads, inter_op_parallelism_threads
                        and use_per_session_threads.
                        For 'openvino' backend, only need to specify either model_type or
                        pipeline_config_path together with extensions_config_path.
                        Default is 'tensorflow'.
        :param intra_op_parallelism_threads: For 'tensorflow' backend only. Int.
                                             The number of intraOpParallelismThreads. Default is 1.
        :param inter_op_parallelism_threads: For 'tensorflow' backend only. Int.
                                             The number of interOpParallelismThreads. Default is 1.
        :param use_per_session_threads: For 'tensorflow' backend only. Boolean.
                                        Whether to use perSessionThreads. Default is True.
        :param model_type: For 'openvino' backend only. The type of the TensorFlow model,
                           e.g. faster_rcnn_resnet101_coco, ssd_inception_v2_coco, etc.
        :param ov_pipeline_config_path: For 'openvino' backend only. String.
                                        The file path to the pipeline configure file.
        :param ov_extensions_config_path: For 'openvino' backend only. String.
                                          The file path to the extensions configure file.
                                          Need pipeline_config_path and extensions_config_path
                                          for 'openvino' backend if model_type is not specified.
        """
        warnings.warn("deprecated in 0.8.0")
        backend = backend.lower()
        if backend == "tensorflow" or backend == "tf":
            callZooFunc(self.bigdl_type, "inferenceModelLoadTensorFlow",
                        self.value, model_path, "frozenModel", intra_op_parallelism_threads,
                        inter_op_parallelism_threads, use_per_session_threads)
        elif backend == "openvino" or backend == "ov":
            if model_type:
                if ov_pipeline_config_path:
                    callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                                self.value, model_path, model_type, ov_pipeline_config_path, None)
                else:
                    callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                                self.value, model_path, model_type)
            else:
                if ov_pipeline_config_path is None and ov_extensions_config_path is None:
                    raise Exception("For openvino backend, you must provide either model_type or "
                                    "both pipeline_config_path and extensions_config_path")
                callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                            self.value, model_path, ov_pipeline_config_path,
                            ov_extensions_config_path)
        else:
            raise ValueError("Currently only tensorflow and openvino are supported as backend")

    # deprecated in "0.8.0"
    def load_tf_object_detection_as_openvino(self,
                                             model_path,
                                             object_detection_model_type,
                                             pipeline_config_path,
                                             extensions_config_path
                                             ):
        """
        load object detection TF model as OpenVINO IR
        :param model_path: String, the path of the tensorflow model
        :param object_detection_model_type: String, the type of the tensorflow model
        :param pipeline_config_path: String, the path of the pipeline configure file
        :param extensions_config_path: String, the path of the extensions configure file
        :return:
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type,
                    "inferenceModelOpenVINOLoadTF",
                    self.value,
                    model_path,
                    object_detection_model_type,
                    pipeline_config_path,
                    extensions_config_path)

    # deprecated in "0.8.0"
    def load_tf_image_classification_as_openvino(self,
                                                 model_path,
                                                 image_classification_model_type,
                                                 checkpoint_path,
                                                 input_shape,
                                                 if_reverse_input_channels,
                                                 mean_values,
                                                 scale):
        """
        load image classification TF model as OpenVINO IR
        :param model_path: String, the path of the tensorflow model
        :param image_classification_model_type: String, the type of the tensorflow model
        :param checkpoint_path: String, the path of the tensorflow checkpoint file
        :param input_shape: List of Int,
                input shape that should be fed to an input node(s) of the model
        :param if_reverse_input_channels: Boolean,
                the boolean value of if need reverse input channels.
        :param mean_values: List of Float, all input values coming from original network inputs
                            will be divided by this value.
        :param scale: Float, the scale value, to be used for the input image per channel.
        :return:
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type,
                    "inferenceModelOpenVINOLoadTF",
                    self.value,
                    model_path,
                    image_classification_model_type,
                    checkpoint_path,
                    input_shape,
                    if_reverse_input_channels,
                    [float(value) for value in mean_values],
                    float(scale))

    def predict(self, inputs):
        """
        Do prediction on inputs.

        :param inputs: A numpy array or a list of numpy arrays or JTensor or a list of JTensors.
        """
        jinputs, input_is_table = Layer.check_input(inputs)
        output = callZooFunc(self.bigdl_type,
                             "inferenceModelPredict",
                             self.value,
                             jinputs,
                             input_is_table)
        return KerasNet.convert_output(output)
<commit_msg>python inference model api for saved model (#2355)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from bigdl.util.common import JavaValue
from zoo.common.utils import callZooFunc
from bigdl.nn.layer import Layer
from zoo.pipeline.api.keras.engine import KerasNet
import warnings

class InferenceModel(JavaValue):
    """
    Model for thread-safe inference.
    To do inference, you need to first initiate an InferenceModel instance, then call
    load|load_caffe|load_tf|load_openvino to load a pre-trained model, and finally call predict.

    # Arguments
    supported_concurrent_num: Int. How many concurrent threads to invoke. Default is 1.
    """

    def __init__(self, supported_concurrent_num=1, bigdl_type="float"):
        super(InferenceModel, self).__init__(None, bigdl_type, supported_concurrent_num)

    def load_bigdl(self, model_path, weight_path=None):
        """
        Load a pre-trained Analytics Zoo or BigDL model.

        :param model_path: String. The file path to the model.
        :param weight_path: String. The file path to the weights if any. Default is None.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadBigDL",
                    self.value, model_path, weight_path)

    # deprecated in "0.8.0"
    def load(self, model_path, weight_path=None):
        """
        Load a pre-trained Analytics Zoo or BigDL model.

        :param model_path: String. The file path to the model.
        :param weight_path: String. The file path to the weights if any. Default is None.
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type, "inferenceModelLoad",
                    self.value, model_path, weight_path)

    def load_caffe(self, model_path, weight_path):
        """
        Load a pre-trained Caffe model.

        :param model_path: String. The file path to the prototxt file.
        :param weight_path: String. The file path to the Caffe model.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadCaffe",
                    self.value, model_path, weight_path)

    def load_openvino(self, model_path, weight_path, batch_size=0):
        """
        Load an OpenVINI IR.

        :param model_path: String. The file path to the OpenVINO IR xml file.
        :param weight_path: String. The file path to the OpenVINO IR bin file.
        :param batch_size: Int. Set batch Size, default is 0 (use default batch size).
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadOpenVINO",
                    self.value, model_path, weight_path, batch_size)

    def load_tensorflow(self, model_path, model_type="frozenModel", intra_op_parallelism_threads=1,
                        inter_op_parallelism_threads=1, use_per_session_threads=True):
        """
        Load an TensorFlow model using tensorflow.

        :param model_path: String. The file path to the TensorFlow model.
        :param model_type: String. The type of the tensorflow model file. Default is "frozenModel"
        :param intra_op_parallelism_threads: Int. The number of intraOpParallelismThreads.
                                             Default is 1.
        :param inter_op_parallelism_threads: Int. The number of interOpParallelismThreads.
                                             Default is 1.
        :param use_per_session_threads: Boolean. Whether to use perSessionThreads. Default is True.
        """
        callZooFunc(self.bigdl_type, "inferenceModelLoadTensorFlow",
                    self.value, model_path, model_type, intra_op_parallelism_threads,
                    inter_op_parallelism_threads, use_per_session_threads)

    def load_tensorflow(self, model_path, model_type,
                        inputs, outputs, intra_op_parallelism_threads=1,
                        inter_op_parallelism_threads=1, use_per_session_threads=True):
        """
        Load an TensorFlow model using tensorflow.

        :param model_path: String. The file path to the TensorFlow model.
        :param model_type: String. The type of the tensorflow model file: "frozenModel" or
         "savedModel".
        :param inputs: Array[String]. the inputs of the model.
        inputs outputs: Array[String]. the outputs of the model.
        :param intra_op_parallelism_threads: Int. The number of intraOpParallelismThreads.
                                                Default is 1.
        :param inter_op_parallelism_threads: Int. The number of interOpParallelismThreads.
                                                Default is 1.
        :param use_per_session_threads: Boolean. Whether to use perSessionThreads. Default is True.
           """
        callZooFunc(self.bigdl_type, "inferenceModelLoadTensorFlow",
                    self.value, model_path, model_type,
                    inputs, outputs, intra_op_parallelism_threads,
                    inter_op_parallelism_threads, use_per_session_threads)

    # deprecated in "0.8.0"
    def load_tf(self, model_path, backend="tensorflow",
                intra_op_parallelism_threads=1, inter_op_parallelism_threads=1,
                use_per_session_threads=True, model_type=None,
                ov_pipeline_config_path=None, ov_extensions_config_path=None):
        """
        Load an TensorFlow model using tensorflow or openvino backend.

        :param model_path: String. The file path to the TensorFlow model.
        :param backend: String. The backend to use for inference. Either 'tensorflow' or 'openvino'.
                        For 'tensorflow' backend, only need to specify arguments
                        intra_op_parallelism_threads, inter_op_parallelism_threads
                        and use_per_session_threads.
                        For 'openvino' backend, only need to specify either model_type or
                        pipeline_config_path together with extensions_config_path.
                        Default is 'tensorflow'.
        :param intra_op_parallelism_threads: For 'tensorflow' backend only. Int.
                                             The number of intraOpParallelismThreads. Default is 1.
        :param inter_op_parallelism_threads: For 'tensorflow' backend only. Int.
                                             The number of interOpParallelismThreads. Default is 1.
        :param use_per_session_threads: For 'tensorflow' backend only. Boolean.
                                        Whether to use perSessionThreads. Default is True.
        :param model_type: For 'openvino' backend only. The type of the TensorFlow model,
                           e.g. faster_rcnn_resnet101_coco, ssd_inception_v2_coco, etc.
        :param ov_pipeline_config_path: For 'openvino' backend only. String.
                                        The file path to the pipeline configure file.
        :param ov_extensions_config_path: For 'openvino' backend only. String.
                                          The file path to the extensions configure file.
                                          Need pipeline_config_path and extensions_config_path
                                          for 'openvino' backend if model_type is not specified.
        """
        warnings.warn("deprecated in 0.8.0")
        backend = backend.lower()
        if backend == "tensorflow" or backend == "tf":
            callZooFunc(self.bigdl_type, "inferenceModelLoadTensorFlow",
                        self.value, model_path, "frozenModel", intra_op_parallelism_threads,
                        inter_op_parallelism_threads, use_per_session_threads)
        elif backend == "openvino" or backend == "ov":
            if model_type:
                if ov_pipeline_config_path:
                    callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                                self.value, model_path, model_type, ov_pipeline_config_path, None)
                else:
                    callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                                self.value, model_path, model_type)
            else:
                if ov_pipeline_config_path is None and ov_extensions_config_path is None:
                    raise Exception("For openvino backend, you must provide either model_type or "
                                    "both pipeline_config_path and extensions_config_path")
                callZooFunc(self.bigdl_type, "inferenceModelOpenVINOLoadTF",
                            self.value, model_path, ov_pipeline_config_path,
                            ov_extensions_config_path)
        else:
            raise ValueError("Currently only tensorflow and openvino are supported as backend")

    # deprecated in "0.8.0"
    def load_tf_object_detection_as_openvino(self,
                                             model_path,
                                             object_detection_model_type,
                                             pipeline_config_path,
                                             extensions_config_path
                                             ):
        """
        load object detection TF model as OpenVINO IR
        :param model_path: String, the path of the tensorflow model
        :param object_detection_model_type: String, the type of the tensorflow model
        :param pipeline_config_path: String, the path of the pipeline configure file
        :param extensions_config_path: String, the path of the extensions configure file
        :return:
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type,
                    "inferenceModelOpenVINOLoadTF",
                    self.value,
                    model_path,
                    object_detection_model_type,
                    pipeline_config_path,
                    extensions_config_path)

    # deprecated in "0.8.0"
    def load_tf_image_classification_as_openvino(self,
                                                 model_path,
                                                 image_classification_model_type,
                                                 checkpoint_path,
                                                 input_shape,
                                                 if_reverse_input_channels,
                                                 mean_values,
                                                 scale):
        """
        load image classification TF model as OpenVINO IR
        :param model_path: String, the path of the tensorflow model
        :param image_classification_model_type: String, the type of the tensorflow model
        :param checkpoint_path: String, the path of the tensorflow checkpoint file
        :param input_shape: List of Int,
                input shape that should be fed to an input node(s) of the model
        :param if_reverse_input_channels: Boolean,
                the boolean value of if need reverse input channels.
        :param mean_values: List of Float, all input values coming from original network inputs
                            will be divided by this value.
        :param scale: Float, the scale value, to be used for the input image per channel.
        :return:
        """
        warnings.warn("deprecated in 0.8.0")
        callZooFunc(self.bigdl_type,
                    "inferenceModelOpenVINOLoadTF",
                    self.value,
                    model_path,
                    image_classification_model_type,
                    checkpoint_path,
                    input_shape,
                    if_reverse_input_channels,
                    [float(value) for value in mean_values],
                    float(scale))

    def predict(self, inputs):
        """
        Do prediction on inputs.

        :param inputs: A numpy array or a list of numpy arrays or JTensor or a list of JTensors.
        """
        jinputs, input_is_table = Layer.check_input(inputs)
        output = callZooFunc(self.bigdl_type,
                             "inferenceModelPredict",
                             self.value,
                             jinputs,
                             input_is_table)
        return KerasNet.convert_output(output)
<|endoftext|>