<commit_before>import os
import json
import time
import logging
import gzip
import boto3
import queue
import threading
from boto3.s3.transfer import S3Transfer, TransferConfig
from botocore.exceptions import BotoCoreError

from flumine import BaseStrategy
from flumine.utils import create_short_uuid, file_line_count

logger = logging.getLogger(__name__)

class MarketRecorder(BaseStrategy):

    """
    Simple raw streaming market recorder, context:

        market_expiration: int, Seconds to wait after market closure before removing files
        remove_file: bool, Remove txt file during cleanup
        remove_gz_file: bool, Remove gz file during cleanup
        force_update: bool, Update zip/closure if update received after closure
        load_market_catalogue: bool, Store marketCatalogue as {marketId}.json
        local_dir: str, Dir to store data
        recorder_id: str, Directory name (defaults to random uuid)
    """

    MARKET_ID_LOOKUP = "id"

    def __init__(self, *args, **kwargs):
        BaseStrategy.__init__(self, *args, **kwargs)
        self._market_expiration = self.context.get("market_expiration", 3600)  # seconds
        self._remove_file = self.context.get("remove_file", False)
        self._remove_gz_file = self.context.get("remove_gz_file", False)
        self._force_update = self.context.get("force_update", True)
        self._load_market_catalogue = self.context.get("load_market_catalogue", True)
        self.local_dir = self.context.get("local_dir", "/tmp")
        self.recorder_id = self.context.get("recorder_id", create_short_uuid())
        self._loaded_markets = []  # list of marketIds
        self._queue = queue.Queue()

    def add(self) -> None:
        logger.info("Adding strategy %s with id %s" % (self.name, self.recorder_id))
        # check local dir
        if not os.path.isdir(self.local_dir):
            raise OSError("File dir %s does not exist" % self.local_dir)
        # create sub dir
        directory = os.path.join(self.local_dir, self.recorder_id)
        if not os.path.exists(directory):
            os.makedirs(directory)

    def start(self) -> None:
        # start load processor thread
        threading.Thread(
            name="{0}_load_processor".format(self.name),
            target=self._load_processor,
            daemon=True,
        ).start()

    def process_raw_data(self, clk: str, publish_time: int, data: dict):
        market_id = data.get(self.MARKET_ID_LOOKUP)
        file_directory = os.path.join(self.local_dir, self.recorder_id, market_id)
        with open(file_directory, "a") as f:
            f.write(
                json.dumps(
                    {"op": "mcm", "clk": clk, "pt": publish_time, "mc": [data]},
                    separators=(",", ":"),
                )
                + "\n"
            )

    def process_closed_market(self, market, data: dict) -> None:
        market_id = data.get(self.MARKET_ID_LOOKUP)
        if market_id in self._loaded_markets:
            if self._force_update:
                logger.warning(
                    "File: /{0}/{1}/{2} has already been loaded, updating..".format(
                        self.local_dir, self.recorder_id, market_id
                    )
                )
            else:
                return
        else:
            self._loaded_markets.append(market_id)
        logger.info("Closing market %s" % market_id)

        file_dir = os.path.join(self.local_dir, self.recorder_id, market_id)
        market_definition = data.get("marketDefinition")

        # check that file actually exists
        if not os.path.isfile(file_dir):
            logger.error(
                "File: %s does not exist in /%s/%s/"
                % (self.local_dir, market_id, self.recorder_id)
            )
            return

        # check that file is not empty / 1 line (i.e. the market had already closed on startup)
        line_count = file_line_count(file_dir)
        if line_count == 1:
            logger.warning(
                "File: %s contains one line only and will not be loaded (already closed on startup)"
                % file_dir
            )
            return

        self._queue.put((market, file_dir, market_definition))

    def _load_processor(self):
        # process compression/load in thread
        while True:
            market, file_dir, market_definition = self._queue.get(block=True)
            # check file still exists (potential race condition)
            if not os.path.isfile(file_dir):
                logger.warning(
                    "File: %s does not exist in %s" % (market.market_id, file_dir)
                )
                continue
            # compress file
            compress_file_dir = self._compress_file(file_dir)
            # core load code
            self._load(market, compress_file_dir, market_definition)
            # clean up
            self._clean_up()

    def _compress_file(self, file_dir: str) -> str:
        """compresses txt file into filename.gz"""
        compressed_file_dir = "{0}.gz".format(file_dir)
        with open(file_dir, "rb") as f:
            with gzip.open(compressed_file_dir, "wb") as compressed_file:
                compressed_file.writelines(f)
        return compressed_file_dir

    def _load(self, market, compress_file_dir: str, market_definition: dict) -> None:
        # store marketCatalogue data `{marketId}.json.gz`
        if market and self._load_market_catalogue:
            if market.market_catalogue is None:
                logger.warning(
                    "No marketCatalogue data available for %s" % market.market_id
                )
                return
            market_catalogue_compressed = self._compress_catalogue(
                market.market_catalogue
            )
            # save to file
            file_dir = os.path.join(
                self.local_dir, self.recorder_id, "{0}.json.gz".format(market.market_id)
            )
            with open(file_dir, "wb") as f:
                f.write(market_catalogue_compressed)

    @staticmethod
    def _compress_catalogue(market_catalogue) -> bytes:
        market_catalogue_dumped = market_catalogue.json()
        if isinstance(market_catalogue_dumped, str):
            market_catalogue_dumped = market_catalogue_dumped.encode("utf-8")
        return gzip.compress(market_catalogue_dumped)

    def _clean_up(self) -> None:
        """If gz > market_expiration old remove
        gz and txt file
        """
        directory = os.path.join(self.local_dir, self.recorder_id)
        for file in os.listdir(directory):
            if file.endswith(".gz"):
                gz_path = os.path.join(directory, file)
                file_stats = os.stat(gz_path)
                seconds_since = time.time() - file_stats.st_mtime
                if seconds_since > self._market_expiration:
                    if self._remove_gz_file:
                        logger.info(
                            "Removing: %s, age: %ss"
                            % (gz_path, round(seconds_since, 2))
                        )
                        os.remove(gz_path)
                    txt_path = os.path.join(directory, file.split(".gz")[0])
                    if os.path.exists(txt_path) and self._remove_file:
                        file_stats = os.stat(txt_path)
                        seconds_since = time.time() - file_stats.st_mtime
                        if seconds_since > self._market_expiration:
                            logger.info(
                                "Removing: %s, age: %ss"
                                % (txt_path, round(seconds_since, 2))
                            )
                            os.remove(txt_path)

    @staticmethod
    def _create_metadata(market_definition: dict) -> dict:
        try:
            del market_definition["runners"]
        except KeyError:
            pass
        return dict([a, str(x)] for a, x in market_definition.items())

class S3MarketRecorder(MarketRecorder):
    """
    AWS S3 version of the market recorder
    to automate loading of compressed file
    and marketCatalogue on closure.

        bucket: str, bucket name
        data_type: str, data type
    """

    def __init__(self, *args, **kwargs):
        MarketRecorder.__init__(self, *args, **kwargs)
        self._bucket = self.context["bucket"]
        self._data_type = self.context.get("data_type", "marketdata")
        self._default_event_type_id = self.context.get("default_event_type_id", "7")
        self.s3 = boto3.client("s3")
        transfer_config = TransferConfig(use_threads=False)
        self.transfer = S3Transfer(self.s3, config=transfer_config)

    def add(self) -> None:
        super().add()
        self.s3.head_bucket(Bucket=self._bucket)  # validate bucket/access

    def _load(self, market, compress_file_dir: str, market_definition: dict) -> None:
        # load to s3
        event_type_id = (
            market_definition.get("eventTypeId", 0) if market_definition else self._default_event_type_id
        )
        try:
            self.transfer.upload_file(
                filename=compress_file_dir,
                bucket=self._bucket,
                key=os.path.join(
                    self._data_type,
                    "streaming",
                    event_type_id,
                    os.path.basename(compress_file_dir),
                ),
                extra_args={
                    "Metadata": self._create_metadata(market_definition)
                    if market_definition
                    else {}
                },
            )
            logger.info("%s successfully loaded to s3" % compress_file_dir)
        except (BotoCoreError, Exception) as e:
            logger.error("Error loading to s3: %s" % e)

        # upload marketCatalogue data
        if market and self._load_market_catalogue:
            if market.market_catalogue is None:
                logger.warning(
                    "No marketCatalogue data available for %s" % market.market_id
                )
                return
            market_catalogue_compressed = self._compress_catalogue(
                market.market_catalogue
            )
            try:
                self.s3.put_object(
                    Body=market_catalogue_compressed,
                    Bucket=self._bucket,
                    Key=self._make_market_catalogue_s3_key(market),
                )
                logger.info(
                    "%s successfully loaded marketCatalogue to s3" % market.market_id
                )
            except (BotoCoreError, Exception) as e:
                logger.error("Error loading to s3: %s" % e)

    def _make_market_catalogue_s3_key(self, market) -> str:
        key = os.path.join(
            self._data_type,
            "marketCatalogue",
            "{0}.gz".format(market.market_id),
        )
        return key
<commit_msg>Add S3MarketRecorder._make_prices_file_s3_key<commit_after>import os
import json
import time
import logging
import gzip
import boto3
import queue
import threading
from boto3.s3.transfer import S3Transfer, TransferConfig
from botocore.exceptions import BotoCoreError

from flumine import BaseStrategy
from flumine.utils import create_short_uuid, file_line_count

logger = logging.getLogger(__name__)

class MarketRecorder(BaseStrategy):

    """
    Simple raw streaming market recorder, context:

        market_expiration: int, Seconds to wait after market closure before removing files
        remove_file: bool, Remove txt file during cleanup
        remove_gz_file: bool, Remove gz file during cleanup
        force_update: bool, Update zip/closure if update received after closure
        load_market_catalogue: bool, Store marketCatalogue as {marketId}.json
        local_dir: str, Dir to store data
        recorder_id: str, Directory name (defaults to random uuid)
    """

    MARKET_ID_LOOKUP = "id"

    def __init__(self, *args, **kwargs):
        BaseStrategy.__init__(self, *args, **kwargs)
        self._market_expiration = self.context.get("market_expiration", 3600)  # seconds
        self._remove_file = self.context.get("remove_file", False)
        self._remove_gz_file = self.context.get("remove_gz_file", False)
        self._force_update = self.context.get("force_update", True)
        self._load_market_catalogue = self.context.get("load_market_catalogue", True)
        self.local_dir = self.context.get("local_dir", "/tmp")
        self.recorder_id = self.context.get("recorder_id", create_short_uuid())
        self._loaded_markets = []  # list of marketIds
        self._queue = queue.Queue()

    def add(self) -> None:
        logger.info("Adding strategy %s with id %s" % (self.name, self.recorder_id))
        # check local dir
        if not os.path.isdir(self.local_dir):
            raise OSError("File dir %s does not exist" % self.local_dir)
        # create sub dir
        directory = os.path.join(self.local_dir, self.recorder_id)
        if not os.path.exists(directory):
            os.makedirs(directory)

    def start(self) -> None:
        # start load processor thread
        threading.Thread(
            name="{0}_load_processor".format(self.name),
            target=self._load_processor,
            daemon=True,
        ).start()

    def process_raw_data(self, clk: str, publish_time: int, data: dict):
        market_id = data.get(self.MARKET_ID_LOOKUP)
        file_directory = os.path.join(self.local_dir, self.recorder_id, market_id)
        with open(file_directory, "a") as f:
            f.write(
                json.dumps(
                    {"op": "mcm", "clk": clk, "pt": publish_time, "mc": [data]},
                    separators=(",", ":"),
                )
                + "\n"
            )

    def process_closed_market(self, market, data: dict) -> None:
        market_id = data.get(self.MARKET_ID_LOOKUP)
        if market_id in self._loaded_markets:
            if self._force_update:
                logger.warning(
                    "File: /{0}/{1}/{2} has already been loaded, updating..".format(
                        self.local_dir, self.recorder_id, market_id
                    )
                )
            else:
                return
        else:
            self._loaded_markets.append(market_id)
        logger.info("Closing market %s" % market_id)

        file_dir = os.path.join(self.local_dir, self.recorder_id, market_id)
        market_definition = data.get("marketDefinition")

        # check that file actually exists
        if not os.path.isfile(file_dir):
            logger.error(
                "File: %s does not exist in /%s/%s/"
                % (self.local_dir, market_id, self.recorder_id)
            )
            return

        # check that file is not empty / 1 line (i.e. the market had already closed on startup)
        line_count = file_line_count(file_dir)
        if line_count == 1:
            logger.warning(
                "File: %s contains one line only and will not be loaded (already closed on startup)"
                % file_dir
            )
            return

        self._queue.put((market, file_dir, market_definition))

    def _load_processor(self):
        # process compression/load in thread
        while True:
            market, file_dir, market_definition = self._queue.get(block=True)
            # check file still exists (potential race condition)
            if not os.path.isfile(file_dir):
                logger.warning(
                    "File: %s does not exist in %s" % (market.market_id, file_dir)
                )
                continue
            # compress file
            compress_file_dir = self._compress_file(file_dir)
            # core load code
            self._load(market, compress_file_dir, market_definition)
            # clean up
            self._clean_up()

    def _compress_file(self, file_dir: str) -> str:
        """compresses txt file into filename.gz"""
        compressed_file_dir = "{0}.gz".format(file_dir)
        with open(file_dir, "rb") as f:
            with gzip.open(compressed_file_dir, "wb") as compressed_file:
                compressed_file.writelines(f)
        return compressed_file_dir

    def _load(self, market, compress_file_dir: str, market_definition: dict) -> None:
        # store marketCatalogue data `{marketId}.json.gz`
        if market and self._load_market_catalogue:
            if market.market_catalogue is None:
                logger.warning(
                    "No marketCatalogue data available for %s" % market.market_id
                )
                return
            market_catalogue_compressed = self._compress_catalogue(
                market.market_catalogue
            )
            # save to file
            file_dir = os.path.join(
                self.local_dir, self.recorder_id, "{0}.json.gz".format(market.market_id)
            )
            with open(file_dir, "wb") as f:
                f.write(market_catalogue_compressed)

    @staticmethod
    def _compress_catalogue(market_catalogue) -> bytes:
        market_catalogue_dumped = market_catalogue.json()
        if isinstance(market_catalogue_dumped, str):
            market_catalogue_dumped = market_catalogue_dumped.encode("utf-8")
        return gzip.compress(market_catalogue_dumped)

    def _clean_up(self) -> None:
        """If gz > market_expiration old remove
        gz and txt file
        """
        directory = os.path.join(self.local_dir, self.recorder_id)
        for file in os.listdir(directory):
            if file.endswith(".gz"):
                gz_path = os.path.join(directory, file)
                file_stats = os.stat(gz_path)
                seconds_since = time.time() - file_stats.st_mtime
                if seconds_since > self._market_expiration:
                    if self._remove_gz_file:
                        logger.info(
                            "Removing: %s, age: %ss"
                            % (gz_path, round(seconds_since, 2))
                        )
                        os.remove(gz_path)
                    txt_path = os.path.join(directory, file.split(".gz")[0])
                    if os.path.exists(txt_path) and self._remove_file:
                        file_stats = os.stat(txt_path)
                        seconds_since = time.time() - file_stats.st_mtime
                        if seconds_since > self._market_expiration:
                            logger.info(
                                "Removing: %s, age: %ss"
                                % (txt_path, round(seconds_since, 2))
                            )
                            os.remove(txt_path)

    @staticmethod
    def _create_metadata(market_definition: dict) -> dict:
        try:
            del market_definition["runners"]
        except KeyError:
            pass
        return dict([a, str(x)] for a, x in market_definition.items())

class S3MarketRecorder(MarketRecorder):
    """
    AWS S3 version of the market recorder
    to automate loading of compressed file
    and marketCatalogue on closure.

        bucket: str, bucket name
        data_type: str, data type
    """

    def __init__(self, *args, **kwargs):
        MarketRecorder.__init__(self, *args, **kwargs)
        self._bucket = self.context["bucket"]
        self._data_type = self.context.get("data_type", "marketdata")
        self._default_event_type_id = self.context.get("default_event_type_id", "7")
        self.s3 = boto3.client("s3")
        transfer_config = TransferConfig(use_threads=False)
        self.transfer = S3Transfer(self.s3, config=transfer_config)

    def add(self) -> None:
        super().add()
        self.s3.head_bucket(Bucket=self._bucket)  # validate bucket/access

    def _load(self, market, compress_file_dir: str, market_definition: dict) -> None:
        # load to s3
        try:
            self.transfer.upload_file(
                filename=compress_file_dir,
                bucket=self._bucket,
                key=self._make_prices_file_s3_key(compress_file_dir, market_definition),
                extra_args={
                    "Metadata": self._create_metadata(market_definition)
                    if market_definition
                    else {}
                },
            )
            logger.info("%s successfully loaded to s3" % compress_file_dir)
        except (BotoCoreError, Exception) as e:
            logger.error("Error loading to s3: %s" % e)

        # upload marketCatalogue data
        if market and self._load_market_catalogue:
            if market.market_catalogue is None:
                logger.warning(
                    "No marketCatalogue data available for %s" % market.market_id
                )
                return
            market_catalogue_compressed = self._compress_catalogue(
                market.market_catalogue
            )
            try:
                self.s3.put_object(
                    Body=market_catalogue_compressed,
                    Bucket=self._bucket,
                    Key=self._make_market_catalogue_s3_key(market),
                )
                logger.info(
                    "%s successfully loaded marketCatalogue to s3" % market.market_id
                )
            except (BotoCoreError, Exception) as e:
                logger.error("Error loading to s3: %s" % e)

    def _make_market_catalogue_s3_key(self, market) -> str:
        key = os.path.join(
            self._data_type,
            "marketCatalogue",
            "{0}.gz".format(market.market_id),
        )
        return key

    def _make_prices_file_s3_key(
        self, compress_file_dir: str, market_definition: dict
    ) -> str:
        event_type_id = (
            market_definition.get("eventTypeId", 0)
            if market_definition
            else self._default_event_type_id
        )
        key = os.path.join(
            self._data_type,
            "streaming",
            event_type_id,
            os.path.basename(compress_file_dir),
        )
        return key
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import itertools
import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechTurnGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import RMSprop
from pyannote.audio.embedding.utils import to_condensed
from scipy.spatial.distance import squareform

class TripletLoss(object):
    """

    delta = d(anchor, positive) - d(anchor, negative)

    * with 'positive' clamping:
        loss = max(0, delta + margin x D)
    * with 'sigmoid' clamping:
        loss = sigmoid(10 * delta)

    where d(., .) varies in range [0, D] (e.g. D=2 for euclidean distance).

    Parameters
    ----------
    metric : {'euclidean', 'cosine', 'angular'}, optional
        Defaults to 'cosine'.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    clamp : {'positive', 'sigmoid', 'softmargin'}, optional
        Defaults to 'positive'.
    sampling : {'all', 'hard'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    """

    def __init__(self, duration=3.2,
                 metric='cosine', margin=0.2, clamp='positive',
                 sampling='all', per_label=3, per_fold=None):

        super(TripletLoss, self).__init__()

        self.metric = metric
        self.margin = margin

        if self.metric == 'cosine':
            self.margin_ = self.margin * 2.
        elif self.metric == 'angular':
            self.margin_ = self.margin * np.pi
        elif self.metric == 'euclidean':
            self.margin_ = self.margin * 2.
        else:
            msg = "'metric' must be one of {'euclidean', 'cosine', 'angular'}."
            raise ValueError(msg)

        if clamp not in {'positive', 'sigmoid', 'softmargin'}:
            msg = "'clamp' must be one of {'positive', 'sigmoid', 'softmargin'}."
            raise ValueError(msg)
        self.clamp = clamp

        if sampling not in {'all', 'hard'}:
            msg = "'sampling' must be one of {'all', 'hard'}."
            raise ValueError(msg)
        self.sampling = sampling

        self.per_fold = per_fold
        self.per_label = per_label
        self.duration = duration

    def pdist(self, fX):
        """Compute pdist à-la scipy.spatial.distance.pdist

        Parameters
        ----------
        fX : (n, d) torch.autograd.Variable
            Embeddings.

        Returns
        -------
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix
        """

        n_sequences, _ = fX.size()
        distances = []

        for i in range(1, n_sequences):

            if self.metric in ('cosine', 'angular'):

                d = 1. - F.cosine_similarity(
                    fX[i, :].expand(i, -1), fX[:i, :],
                    dim=1, eps=1e-8)

                if self.metric == 'angular':
                    d = torch.acos(1. - d)

            elif self.metric == 'euclidean':
                d = F.pairwise_distance(
                    fX[i, :].expand(i, -1), fX[:i, :],
                    p=2, eps=1e-06)

            distances.append(d)

        return torch.cat(distances)

    def batch_hard(self, y, distances):
        """

        Parameters
        ----------
        y : list
            Sequence labels.
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix

        Returns
        -------
        anchors, positives, negatives : list of int
            Triplets indices.
        """

        anchors, positives, negatives = [], [], []

        if distances.is_cuda:
            distances = squareform(distances.data.cpu().numpy())
        else:
            distances = squareform(distances.data.numpy())
        y = np.array(y)

        for anchor, y_anchor in enumerate(y):

            d = distances[anchor]
            pos = np.where(y == y_anchor)[0]
            pos = [p for p in pos if p != anchor]

            # hardest positive
            positive = int(pos[np.argmax(d[pos])])

            # hardest negative
            neg = np.where(y != y_anchor)[0]
            negative = int(neg[np.argmin(d[neg])])

            anchors.append(anchor)
            positives.append(positive)
            negatives.append(negative)

        return anchors, positives, negatives

    def batch_all(self, y, distances):
        """

        Parameters
        ----------
        y : list
            Sequence labels.
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix

        Returns
        -------
        anchors, positives, negatives : list of int
            Triplets indices.
        """

        anchors, positives, negatives = [], [], []

        for anchor, y_anchor in enumerate(y):
            for positive, y_positive in enumerate(y):

                # if same embedding or different labels, skip
                if (anchor == positive) or (y_anchor != y_positive):
                    continue

                for negative, y_negative in enumerate(y):

                    if y_negative == y_anchor:
                        continue

                    anchors.append(anchor)
                    positives.append(positive)
                    negatives.append(negative)

        return anchors, positives, negatives

    def triplet_loss(self, distances, anchors, positives, negatives):
        """Compute average triplet loss

        Parameters
        ----------
        distances : torch.autograd.Variable
            Condensed matrix of pairwise distances.
        anchors, positives, negatives : list of int
            Triplets indices.

        Returns
        -------
        loss : torch.autograd.Variable
            Average triplet loss.
        """

        # convert indices from squared matrix
        # to condensed matrix referential
        p = list(map(to_condensed, anchors, positives))
        n = list(map(to_condensed, anchors, negatives))

        # compute raw triplet loss (no margin, no clamping)
        # the lower, the better
        delta = distances[p] - distances[n]

        # clamp triplet loss
        if self.clamp == 'positive':
            loss = torch.clamp(delta + self.margin_, min=0)

        elif self.clamp == 'softmargin':
            loss = torch.log1p(torch.exp(delta))

        elif self.clamp == 'sigmoid':
            loss = F.sigmoid(10 * delta)

        # return average triplet loss
        return loss

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            n_epochs=1000, gpu=False):

        logging_callback = LoggingCallbackPytorch(log_dir=log_dir)

        try:
            batch_generator = SpeechTurnGenerator(
                feature_extraction,
                per_label=self.per_label, per_fold=self.per_fold,
                duration=self.duration)
            batches = batch_generator(protocol, subset=subset)
            batch = next(batches)
        except OSError as e:
            del batch_generator.data_
            batch_generator = SpeechTurnGenerator(
                feature_extraction,
                per_label=self.per_label, per_fold=self.per_fold,
                duration=self.duration, fast=False)
            batches = batch_generator(protocol, subset=subset)
            batch = next(batches)

        # one minute per speaker
        duration_per_epoch = 60. * batch_generator.n_labels
        duration_per_batch = self.duration * batch_generator.n_sequences_per_batch
        batches_per_epoch = int(np.ceil(duration_per_epoch / duration_per_batch))

        if gpu:
            model = model.cuda()

        optimizer = RMSprop(model.parameters())
        model.internal = False

        while True:

            for epoch in range(n_epochs):

                loss_min = np.inf
                loss_max = -np.inf
                loss_avg = 0.

                nonzero_min = 1.
                nonzero_max = 0.
                nonzero_avg = 0.

                desc = 'Epoch #{0}'.format(epoch)
                for _ in tqdm(range(batches_per_epoch), desc=desc):

                    model.zero_grad()

                    batch = next(batches)

                    X = Variable(torch.from_numpy(
                        np.array(np.rollaxis(batch['X'], 0, 2),
                                 dtype=np.float32)))

                    if gpu:
                        X = X.cuda()

                    fX = model(X)

                    # pre-compute pairwise distances
                    distances = self.pdist(fX)

                    # sample triplets
                    if self.sampling == 'all':
                        anchors, positives, negatives = self.batch_all(
                            batch['y'], distances)

                    elif self.sampling == 'hard':
                        anchors, positives, negatives = self.batch_hard(
                            batch['y'], distances)

                    # compute triplet loss
                    losses = self.triplet_loss(
                        distances, anchors, positives, negatives)

                    # log ratio of non-zero triplets
                    if gpu:
                        nonzero_ = np.mean(losses.data.cpu().numpy() > 0)
                    else:
                        nonzero_ = np.mean(losses.data.numpy() > 0)
                    nonzero_avg += nonzero_
                    nonzero_min = min(nonzero_min, nonzero_)
                    nonzero_max = max(nonzero_max, nonzero_)

                    loss = torch.mean(losses)

                    # log batch loss
                    if gpu:
                        loss_ = float(loss.data.cpu().numpy())
                    else:
                        loss_ = float(loss.data.numpy())
                    loss_avg += loss_
                    loss_min = min(loss_min, loss_)
                    loss_max = max(loss_max, loss_)

                    loss.backward()
                    optimizer.step()

                loss_avg /= batches_per_epoch
                nonzero_avg /= batches_per_epoch

                logs = {'loss_avg': loss_avg,
                        'loss_min': loss_min,
                        'loss_max': loss_max,
                        'nonzero_avg': nonzero_avg,
                        'nonzero_max': nonzero_max,
                        'nonzero_min': nonzero_min}

                logging_callback.model = model
                logging_callback.optimizer = optimizer
                logging_callback.on_epoch_end(epoch, logs=logs)
<commit_msg>feat: log norms and distances<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import itertools
import numpy as np
from tqdm import tqdm
import torch
from torch.autograd import Variable
import torch.nn.functional as F
from pyannote.audio.generators.speaker import SpeechTurnGenerator
from pyannote.audio.callback import LoggingCallbackPytorch
from torch.optim import RMSprop
from pyannote.audio.embedding.utils import to_condensed
from scipy.spatial.distance import squareform

class TripletLoss(object):
    """

    delta = d(anchor, positive) - d(anchor, negative)

    * with 'positive' clamping:
        loss = max(0, delta + margin x D)
    * with 'sigmoid' clamping:
        loss = sigmoid(10 * delta)

    where d(., .) varies in range [0, D] (e.g. D=2 for euclidean distance).

    Parameters
    ----------
    metric : {'euclidean', 'cosine', 'angular'}, optional
        Defaults to 'cosine'.
    margin: float, optional
        Margin factor. Defaults to 0.2.
    clamp : {'positive', 'sigmoid', 'softmargin'}, optional
        Defaults to 'positive'.
    sampling : {'all', 'hard'}, optional
        Triplet sampling strategy.
    per_label : int, optional
        Number of sequences per speaker in each batch. Defaults to 3.
    per_fold : int, optional
        If provided, sample triplets from groups of `per_fold` speakers at a
        time. Defaults to sample triplets from the whole speaker set.
    """

    def __init__(self, duration=3.2,
                 metric='cosine', margin=0.2, clamp='positive',
                 sampling='all', per_label=3, per_fold=None):

        super(TripletLoss, self).__init__()

        self.metric = metric
        self.margin = margin

        if self.metric == 'cosine':
            self.margin_ = self.margin * 2.
        elif self.metric == 'angular':
            self.margin_ = self.margin * np.pi
        elif self.metric == 'euclidean':
            self.margin_ = self.margin * 2.
        else:
            msg = "'metric' must be one of {'euclidean', 'cosine', 'angular'}."
            raise ValueError(msg)

        if clamp not in {'positive', 'sigmoid', 'softmargin'}:
            msg = "'clamp' must be one of {'positive', 'sigmoid', 'softmargin'}."
            raise ValueError(msg)
        self.clamp = clamp

        if sampling not in {'all', 'hard'}:
            msg = "'sampling' must be one of {'all', 'hard'}."
            raise ValueError(msg)
        self.sampling = sampling

        self.per_fold = per_fold
        self.per_label = per_label
        self.duration = duration

    def pdist(self, fX):
        """Compute pdist à-la scipy.spatial.distance.pdist

        Parameters
        ----------
        fX : (n, d) torch.autograd.Variable
            Embeddings.

        Returns
        -------
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix
        """

        n_sequences, _ = fX.size()
        distances = []

        for i in range(1, n_sequences):

            if self.metric in ('cosine', 'angular'):

                d = 1. - F.cosine_similarity(
                    fX[i, :].expand(i, -1), fX[:i, :],
                    dim=1, eps=1e-8)

                if self.metric == 'angular':
                    d = torch.acos(1. - d)

            elif self.metric == 'euclidean':
                d = F.pairwise_distance(
                    fX[i, :].expand(i, -1), fX[:i, :],
                    p=2, eps=1e-06)

            distances.append(d)

        return torch.cat(distances)

    def batch_hard(self, y, distances):
        """

        Parameters
        ----------
        y : list
            Sequence labels.
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix

        Returns
        -------
        anchors, positives, negatives : list of int
            Triplets indices.
        """

        anchors, positives, negatives = [], [], []

        if distances.is_cuda:
            distances = squareform(distances.data.cpu().numpy())
        else:
            distances = squareform(distances.data.numpy())
        y = np.array(y)

        for anchor, y_anchor in enumerate(y):

            d = distances[anchor]
            pos = np.where(y == y_anchor)[0]
            pos = [p for p in pos if p != anchor]

            # hardest positive
            positive = int(pos[np.argmax(d[pos])])

            # hardest negative
            neg = np.where(y != y_anchor)[0]
            negative = int(neg[np.argmin(d[neg])])

            anchors.append(anchor)
            positives.append(positive)
            negatives.append(negative)

        return anchors, positives, negatives

    def batch_all(self, y, distances):
        """

        Parameters
        ----------
        y : list
            Sequence labels.
        distances : (n * (n-1) / 2,) torch.autograd.Variable
            Condensed pairwise distance matrix

        Returns
        -------
        anchors, positives, negatives : list of int
            Triplets indices.
        """

        anchors, positives, negatives = [], [], []

        for anchor, y_anchor in enumerate(y):
            for positive, y_positive in enumerate(y):

                # if same embedding or different labels, skip
                if (anchor == positive) or (y_anchor != y_positive):
                    continue

                for negative, y_negative in enumerate(y):

                    if y_negative == y_anchor:
                        continue

                    anchors.append(anchor)
                    positives.append(positive)
                    negatives.append(negative)

        return anchors, positives, negatives

    def triplet_loss(self, distances, anchors, positives, negatives):
        """Compute average triplet loss

        Parameters
        ----------
        distances : torch.autograd.Variable
            Condensed matrix of pairwise distances.
        anchors, positives, negatives : list of int
            Triplets indices.

        Returns
        -------
        loss : torch.autograd.Variable
            Average triplet loss.
        """

        # convert indices from squared matrix
        # to condensed matrix referential
        p = list(map(to_condensed, anchors, positives))
        n = list(map(to_condensed, anchors, negatives))

        # compute raw triplet loss (no margin, no clamping)
        # the lower, the better
        delta = distances[p] - distances[n]

        # clamp triplet loss
        if self.clamp == 'positive':
            loss = torch.clamp(delta + self.margin_, min=0)

        elif self.clamp == 'softmargin':
            loss = torch.log1p(torch.exp(delta))

        elif self.clamp == 'sigmoid':
            loss = F.sigmoid(10 * delta)

        # return average triplet loss
        return loss

    def fit(self, model, feature_extraction, protocol, log_dir, subset='train',
            n_epochs=1000, gpu=False):

        logging_callback = LoggingCallbackPytorch(log_dir=log_dir)

        try:
            batch_generator = SpeechTurnGenerator(
                feature_extraction,
                per_label=self.per_label, per_fold=self.per_fold,
                duration=self.duration)
            batches = batch_generator(protocol, subset=subset)
            batch = next(batches)
        except OSError as e:
            del batch_generator.data_
            batch_generator = SpeechTurnGenerator(
                feature_extraction,
                per_label=self.per_label, per_fold=self.per_fold,
                duration=self.duration, fast=False)
            batches = batch_generator(protocol, subset=subset)
            batch = next(batches)

        # one minute per speaker
        duration_per_epoch = 60. * batch_generator.n_labels
        duration_per_batch = self.duration * batch_generator.n_sequences_per_batch
        batches_per_epoch = int(np.ceil(duration_per_epoch / duration_per_batch))

        if gpu:
            model = model.cuda()

        optimizer = RMSprop(model.parameters())
        model.internal = False

        while True:

            for epoch in range(n_epochs):

                loss_min = np.inf
                loss_max = -np.inf
                loss_avg = 0.

                nonzero_min = 1.
                nonzero_max = 0.
                nonzero_avg = 0.

                distances_avg = 0.
                norm_avg = 0.

                desc = 'Epoch #{0}'.format(epoch)
                for _ in tqdm(range(batches_per_epoch), desc=desc):

                    model.zero_grad()

                    batch = next(batches)

                    X = Variable(torch.from_numpy(
                        np.array(np.rollaxis(batch['X'], 0, 2),
                                 dtype=np.float32)))

                    if gpu:
                        X = X.cuda()

                    fX = model(X)
                    if gpu:
                        fX_ = fX.data.cpu().numpy()
                    else:
                        fX_ = fX.data.numpy()
                    norm_avg += np.mean(np.linalg.norm(fX_, axis=0))
                    # TODO. percentile

                    # pre-compute pairwise distances
                    distances = self.pdist(fX)

                    if gpu:
                        distances_ = distances.data.cpu().numpy()
                    else:
                        distances_ = distances.data.numpy()
                    distances_avg += np.mean(distances_)
                    # TODO. percentile

                    # sample triplets
                    if self.sampling == 'all':
                        anchors, positives, negatives = self.batch_all(
                            batch['y'], distances)

                    elif self.sampling == 'hard':
                        anchors, positives, negatives = self.batch_hard(
                            batch['y'], distances)

                    # compute triplet loss
                    losses = self.triplet_loss(
                        distances, anchors, positives, negatives)

                    # log ratio of non-zero triplets
                    if gpu:
                        nonzero_ = np.mean(losses.data.cpu().numpy() > 0)
                    else:
                        nonzero_ = np.mean(losses.data.numpy() > 0)
                    nonzero_avg += nonzero_
                    nonzero_min = min(nonzero_min, nonzero_)
                    nonzero_max = max(nonzero_max, nonzero_)

                    loss = torch.mean(losses)

                    # log batch loss
                    if gpu:
                        loss_ = float(loss.data.cpu().numpy())
                    else:
                        loss_ = float(loss.data.numpy())
                    loss_avg += loss_
                    loss_min = min(loss_min, loss_)
                    loss_max = max(loss_max, loss_)

                    loss.backward()
                    optimizer.step()

                loss_avg /= batches_per_epoch
                nonzero_avg /= batches_per_epoch
                distances_avg /= batches_per_epoch
                norm_avg /= batches_per_epoch

                logs = {'loss_avg': loss_avg,
                        'loss_min': loss_min,
                        'loss_max': loss_max,
                        'nonzero_avg': nonzero_avg,
                        'nonzero_max': nonzero_max,
                        'nonzero_min': nonzero_min,
                        'distances_avg': distances_avg,
                        'norm_avg': norm_avg}

                logging_callback.model = model
                logging_callback.optimizer = optimizer
                logging_callback.on_epoch_end(epoch, logs=logs)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

from celery import shared_task, group
from celery.utils.log import get_task_logger
from django.utils.translation import ugettext_lazy as _

logger = get_task_logger(__name__)

@shared_task
def case_export(export_id):
    from .models import CaseExport

    logger.info("Starting case export #%d..." % export_id)

    CaseExport.objects.get(pk=export_id).do_export()

@shared_task(ignore_result=True)
def get_all_cases_passed_response_time():
    from .models import Case
    cases = Case.get_all_passed_response_time()
    tasks = []
    for case in cases:
        tasks.append(reassign_case.s(case.pk))

    if tasks:
        group_task = group(tasks)
        group_task.apply_async()

@shared_task(ignore_result=True)
def reassign_case(case_id):
    from .models import Case
    try:
        case = Case.objects.get(pk=case_id)
    except Case.DoesNotExist:
        logger.warn("Could not load case #%d for re-assignment" % case_id)
        return

    if not case.has_passed_response_time:
        # the case has been reassigned more recently that our expected window, do nothing
        return

    note = _("This case's required response time as passed and therefor has been re-assigned")
    case.reassign(None, case.last_assignee, note=note, user_assignee=case.last_user_assignee)
<commit_msg>Added check if the last case reassignment was by the system<commit_after>from __future__ import unicode_literals

from celery import shared_task, group
from celery.utils.log import get_task_logger
from django.utils.translation import ugettext_lazy as _

logger = get_task_logger(__name__)

@shared_task
def case_export(export_id):
    from .models import CaseExport

    logger.info("Starting case export #%d..." % export_id)

    CaseExport.objects.get(pk=export_id).do_export()

@shared_task(ignore_result=True)
def get_all_cases_passed_response_time():
    from .models import Case
    cases = Case.get_all_passed_response_time()
    tasks = []
    for case in cases:
        tasks.append(reassign_case.s(case.pk))

    if tasks:
        group_task = group(tasks)
        group_task.apply_async()

@shared_task(ignore_result=True)
def reassign_case(case_id):
    from .models import Case, CaseAction
    try:
        case = Case.objects.get(pk=case_id)
    except Case.DoesNotExist:
        logger.warn("Could not load case #%d for re-assignment" % case_id)
        return

    if not case.has_passed_response_time:
        # the case has been reassigned more recently that our expected window, do nothing
        return

    action = case.actions.filter(action=CaseAction.REASSIGN).latest('created_on')
    if action.created_by is None:
        # The last time this case was reassigned was by the system, we should do nothing now, otherwise the
        # assignment will just keep flipping back and forth
        return

    note = _("This case's required response time as passed and therefor has been re-assigned")
    case.reassign(None, case.last_assignee, note=note, user_assignee=case.last_user_assignee)
<|endoftext|>


<commit_before># Orca
#
# Copyright 2005-2009 Sun Microsystems Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

"""Custom formatting for OpenOffice and StarOffice."""

__id__ = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2005-2009 Sun Microsystems Inc."
__license__   = "LGPL"

# pylint: disable-msg=C0301

import copy

import pyatspi

import orca.formatting
import orca.settings

formatting = {
    'speech': {
        # Get rid of unselectedCell because we don't run into that in OOo
        # and we'd end up always speaking "not selected" for all table cells.
        #
        'suffix': {
            'focused': '[]',
            'unfocused': 'newNodeLevel + ' + orca.formatting.TUTORIAL,
            'basicWhereAmI': orca.formatting.TUTORIAL + ' + description',
            'detailedWhereAmI' : '[]'
            },
        pyatspi.ROLE_COMBO_BOX: {
            'focused': 'name + positionInList + availability',
            'unfocused': 'labelAndName + roleName + positionInList + availability'
            },
        pyatspi.ROLE_LABEL: {
            'focused': 'expandableState + availability',
            'unfocused': 'name + allTextSelection + expandableState + availability + positionInList',
            'basicWhereAmI': 'roleName + name + positionInList + expandableState + (nodeLevel or nestingLevel)'
            },
        pyatspi.ROLE_PUSH_BUTTON: {
            'unfocused': 'labelAndName + roleName + toggleState + availability',
            'focused': 'labelAndName + toggleState'
            },
        pyatspi.ROLE_TOGGLE_BUTTON: {
            'focused': 'labelAndName + toggleState'
            },
        pyatspi.ROLE_TABLE_CELL: {
            'focused': '(tableCell2ChildLabel + tableCell2ChildToggle) or cellCheckedState + (expandableState and (expandableState + numberOfChildren))',
            'unfocused': 'endOfTableIndicator + tableCellRow',
            'basicWhereAmI': 'parentRoleName + columnHeader + rowHeader + roleName + cellCheckedState + (realActiveDescendantDisplayedText or imageDescription + image) + columnAndRow + expandableState + nodeLevel',
            'detailedWhereAmI': 'parentRoleName + columnHeader + rowHeader + roleName + cellCheckedState + (realActiveDescendantDisplayedText or imageDescription + image) + columnAndRow + tableCellRow + expandableState + nodeLevel'
            },
        'ROLE_SPREADSHEET_CELL': {
            # We treat spreadsheet cells differently from other table cells in
            # whereAmI.
            #
            'basicWhereAmI': 'roleName + column + columnHeader + row + rowHeader + (textContent or spreadSheetCell) + anyTextSelection'
            },
    },
    'braille': {
        pyatspi.ROLE_LABEL: {
            'unfocused': '[Text(obj,asString((label or name)))]',
            'focused': '[Text(obj,asString((label or name)))]'
            },
        pyatspi.ROLE_LIST: {
            'unfocused': '[Component(obj,\
                                     asString(labelOrName + roleName + required))]'
        },
        pyatspi.ROLE_SCROLL_PANE: {
            'unfocused': 'asPageTabOrScrollPane\
                          + (childTab\
                             and ([Region(" ")] + childTab) or [])'
        }
    }
}

if orca.settings.useExperimentalSpeechProsody:
    formatting['speech']['ROLE_SPREADSHEET_CELL']['basicWhereAmI'] = \
        'roleName + pause + column + pause + columnHeader + pause + row + pause + rowHeader + pause + (textContent or spreadSheetCell) + pause + anyTextSelection + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['focused'] = \
        '((tableCell2ChildLabel + tableCell2ChildToggle) or cellCheckedState) + pause + (expandableState and (expandableState + pause + numberOfChildren + pause))'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['unfocused'] = \
        'endOfTableIndicator + pause + tableCellRow + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['basicWhereAmI'] = \
        'parentRoleName + pause + columnHeader + pause + rowHeader + pause + roleName + pause + cellCheckedState + pause + (realActiveDescendantDisplayedText or imageDescription + image) + pause + columnAndRow + pause + expandableState + pause + nodeLevel + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['detailedWhereAmI'] = \
        'parentRoleName + pause + columnHeader + pause + rowHeader + pause + roleName + pause + cellCheckedState + pause + (realActiveDescendantDisplayedText or imageDescription + image) + pause + columnAndRow + pause + tableCellRow + pause + expandableState + pause + nodeLevel + pause'

class Formatting(orca.formatting.Formatting):
    def __init__(self, script):
        orca.formatting.Formatting.__init__(self, script)
        self.update(copy.deepcopy(formatting))
        self._defaultFormatting = orca.formatting.Formatting(script)

    def getFormat(self, **args):
        if args.get('useDefaultFormatting', False):
            return self._defaultFormatting.getFormat(**args)
        else:
            return orca.formatting.Formatting.getFormat(self, **args)
<commit_msg>Fix for bgo#624628 - Orca does not announce the state of formatting toggle buttons in OOo 3.3 dev<commit_after># Orca
#
# Copyright 2005-2009 Sun Microsystems Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., Franklin Street, Fifth Floor,
# Boston MA  02110-1301 USA.

"""Custom formatting for OpenOffice and StarOffice."""

__id__ = "$Id$"
__version__   = "$Revision$"
__date__      = "$Date$"
__copyright__ = "Copyright (c) 2005-2009 Sun Microsystems Inc."
__license__   = "LGPL"

# pylint: disable-msg=C0301

import copy

import pyatspi

import orca.formatting
import orca.settings

formatting = {
    'speech': {
        # Get rid of unselectedCell because we don't run into that in OOo
        # and we'd end up always speaking "not selected" for all table cells.
        #
        'suffix': {
            'focused': '[]',
            'unfocused': 'newNodeLevel + ' + orca.formatting.TUTORIAL,
            'basicWhereAmI': orca.formatting.TUTORIAL + ' + description',
            'detailedWhereAmI' : '[]'
            },
        pyatspi.ROLE_COMBO_BOX: {
            'focused': 'name + positionInList + availability',
            'unfocused': 'labelAndName + roleName + positionInList + availability'
            },
        pyatspi.ROLE_LABEL: {
            'focused': 'expandableState + availability',
            'unfocused': 'name + allTextSelection + expandableState + availability + positionInList',
            'basicWhereAmI': 'roleName + name + positionInList + expandableState + (nodeLevel or nestingLevel)'
            },
        pyatspi.ROLE_PUSH_BUTTON: {
            'unfocused': 'labelAndName + roleName + toggleState + availability',
            'focused': 'labelAndName + toggleState'
            },
        pyatspi.ROLE_TOGGLE_BUTTON: {
            'unfocused': 'labelAndName + toggleState',
            'focused': 'labelAndName + toggleState'
            },
        pyatspi.ROLE_TABLE_CELL: {
            'focused': '(tableCell2ChildLabel + tableCell2ChildToggle) or cellCheckedState + (expandableState and (expandableState + numberOfChildren))',
            'unfocused': 'endOfTableIndicator + tableCellRow',
            'basicWhereAmI': 'parentRoleName + columnHeader + rowHeader + roleName + cellCheckedState + (realActiveDescendantDisplayedText or imageDescription + image) + columnAndRow + expandableState + nodeLevel',
            'detailedWhereAmI': 'parentRoleName + columnHeader + rowHeader + roleName + cellCheckedState + (realActiveDescendantDisplayedText or imageDescription + image) + columnAndRow + tableCellRow + expandableState + nodeLevel'
            },
        'ROLE_SPREADSHEET_CELL': {
            # We treat spreadsheet cells differently from other table cells in
            # whereAmI.
            #
            'basicWhereAmI': 'roleName + column + columnHeader + row + rowHeader + (textContent or spreadSheetCell) + anyTextSelection'
            },
    },
    'braille': {
        pyatspi.ROLE_LABEL: {
            'unfocused': '[Text(obj,asString((label or name)))]',
            'focused': '[Text(obj,asString((label or name)))]'
            },
        pyatspi.ROLE_LIST: {
            'unfocused': '[Component(obj,\
                                     asString(labelOrName + roleName + required))]'
        },
        pyatspi.ROLE_SCROLL_PANE: {
            'unfocused': 'asPageTabOrScrollPane\
                          + (childTab\
                             and ([Region(" ")] + childTab) or [])'
        }
    }
}

if orca.settings.useExperimentalSpeechProsody:
    formatting['speech']['ROLE_SPREADSHEET_CELL']['basicWhereAmI'] = \
        'roleName + pause + column + pause + columnHeader + pause + row + pause + rowHeader + pause + (textContent or spreadSheetCell) + pause + anyTextSelection + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['focused'] = \
        '((tableCell2ChildLabel + tableCell2ChildToggle) or cellCheckedState) + pause + (expandableState and (expandableState + pause + numberOfChildren + pause))'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['unfocused'] = \
        'endOfTableIndicator + pause + tableCellRow + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['basicWhereAmI'] = \
        'parentRoleName + pause + columnHeader + pause + rowHeader + pause + roleName + pause + cellCheckedState + pause + (realActiveDescendantDisplayedText or imageDescription + image) + pause + columnAndRow + pause + expandableState + pause + nodeLevel + pause'
    formatting['speech'][pyatspi.ROLE_TABLE_CELL]['detailedWhereAmI'] = \
        'parentRoleName + pause + columnHeader + pause + rowHeader + pause + roleName + pause + cellCheckedState + pause + (realActiveDescendantDisplayedText or imageDescription + image) + pause + columnAndRow + pause + tableCellRow + pause + expandableState + pause + nodeLevel + pause'

class Formatting(orca.formatting.Formatting):
    def __init__(self, script):
        orca.formatting.Formatting.__init__(self, script)
        self.update(copy.deepcopy(formatting))
        self._defaultFormatting = orca.formatting.Formatting(script)

    def getFormat(self, **args):
        if args.get('useDefaultFormatting', False):
            return self._defaultFormatting.getFormat(**args)
        else:
            return orca.formatting.Formatting.getFormat(self, **args)
<|endoftext|>


<commit_before>from basefuncs import *
from functional import compose

class FunctionalFunction(object):
	def __init__(self, func):
		self.func = func
	def __call__(self, *args, **kwargs):
		return self.func(*args, **kwargs)
	def __add__(self, other):
		return FunctionalFunction(compose(other, self))

class ParseRunner:
	def __init__(self, numArgs):
		self.numArgs = numArgs

	def __call__(self, func):
		def wrapped(tks):
			p = 0
			while p < len(tks):# - self.numArgs:
				try:
					r = func(tks, p)
				except IndexError:
					r = None
				if r is not None:
					del tks[p:p+self.numArgs]
					tks.insert(p,r)
				p += 1
			return tks
		return FunctionalFunction(wrapped)

@ParseRunner(3)
def biopParser(tkens, pos):
	if (tkens[pos].tag in EXPRESSION and
		tkens[pos+1].tag==BIOP and
		tkens[pos+2].tag in EXPRESSION):
		#parses expression
		return Token(value=tkens[pos+1].value, tag='BIOPEXP', args=[tkens[pos],tkens[pos+2]])
	else: return None

@ParseRunner(3)
def parenParser(tkens, pos):
	if (tkens[pos].tag==LPAREN and
		tkens[pos+1].tag in EXPRESSION and
		tkens[pos+2].tag==RPAREN):
		#parses expression
		return tkens[pos+1]
	else: return None

@ParseRunner(3)
def asopParser(tkens, pos):
	if (tkens[pos].tag== ID and
		tkens[pos+1].tag==ASOP and
		tkens[pos+2].tag in EXPRESSION):
		#parses expression
		return Token(value=tkens[pos].value, tag='ASOPS', args=tkens[pos+2])
	else: return None

@ParseRunner(2)
def uniopParser(tkens, pos):
	if (tkens[pos].tag==UNIOP and
		tkens[pos+1].tag in EXPRESSION):
		return Token(value=tkens[pos].value, tag='UNIOPEXP', args=tkens[pos+1])
	else: return None

@ParseRunner(2)
def ioStateParser(tkens, pos):
	if (tkens[pos].tag==IOSTATE and
		(tkens[pos+1].tag==ID or
			(tkens[pos].value[-1]=='<' and tkens[pos+1].tag in EXPRESSION))):
		return Token(value=tkens[pos].value, tag='IOSTATES', args=tkens[pos+1])
	else: return None

runParser = uniopParser + ioStateParser + biopParser + parenParser

def Parse(tokenlist):
	origr = ''
	while origr != repr(tokenlist):
		origr = repr(tokenlist)
		tokenlist = runParser(tokenlist)
	tokenlist = asopParser(tokenlist)
	return tokenlist<commit_msg>Parse complete<commit_after>from basefuncs import *
from functional import compose

class FunctionalFunction(object):
	def __init__(self, func):
		self.func = func
	def __call__(self, *args, **kwargs):
		return self.func(*args, **kwargs)
	def __add__(self, other):
		return FunctionalFunction(compose(other, self))

class ParserError(Exception):
    def __init__(self):
        message = "Parse Error"
        super(ParserError, self).__init__(message)

class ParseRunner:
	def __init__(self, numArgs):
		self.numArgs = numArgs

	def __call__(self, func):
		def wrapped(tks):
			p = 0
			while p < len(tks):# - self.numArgs:
				try:
					r = func(tks, p)
				except IndexError:
					r = None
				if r is not None:
					del tks[p:p+self.numArgs]
					tks.insert(p,r)
				else:
					p += 1
			return tks
		return FunctionalFunction(wrapped)

def condParser(ts):
	p = 1
	stops = []
	while p <= len(ts):
		if ts[-p].tag == ENDCON:
			stops.append(p)
			#print(ts[-p], p)
		elif ts[-p].tag == CONDSTATE:
			try:
				end = -stops.pop()
			except IndexError:
				raise ParserError
			states = ts[1-p:end]
			if all(t.tag in STATEMENT for t in states[1:]) and states[0].tag in EXPRESSION:
				op = ts[-p].value
				tok = Token(value=op,tag='CONDSTATES',args=states)
				del ts[-p:end+1]
				ts.insert(end+1,tok)
				p = -end
			else:
				raise ParserError
		p += 1
	return ts

@ParseRunner(3)
def biopParser(tkens, pos):
	if (tkens[pos].tag in EXPRESSION and
		tkens[pos+1].tag==BIOP and
		tkens[pos+2].tag in EXPRESSION):
		#parses expression
		return Token(value=tkens[pos+1].value, tag='BIOPEXP', args=[tkens[pos],tkens[pos+2]])
	else: return None

@ParseRunner(3)
def parenParser(tkens, pos):
	if (tkens[pos].tag==LPAREN and
		tkens[pos+1].tag in EXPRESSION and
		tkens[pos+2].tag==RPAREN):
		#parses expression
		return tkens[pos+1]
	else: return None

@ParseRunner(3)
def asopParser(tkens, pos):
	if (tkens[pos].tag== ID and
		tkens[pos+1].tag==ASOP and
		tkens[pos+2].tag in EXPRESSION):
		#parses expression
		return Token(value=tkens[pos].value, tag='ASOPS', args=tkens[pos+2])
	else: return None

@ParseRunner(2)
def uniopParser(tkens, pos):
	if (tkens[pos].tag==UNIOP and
		tkens[pos+1].tag in EXPRESSION):
		return Token(value=tkens[pos].value, tag='UNIOPEXP', args=tkens[pos+1])
	else: return None

@ParseRunner(2)
def ioStateParser(tkens, pos):
	if (tkens[pos].tag==IOSTATE and
		(tkens[pos+1].tag==ID or
			(tkens[pos].value[-1]=='<' and tkens[pos+1].tag in EXPRESSION))):
		return Token(value=tkens[pos].value, tag='IOSTATES', args=tkens[pos+1])
	else: return None

runParser = uniopParser + ioStateParser + biopParser + parenParser

def Parse(tokenlist):
	origr = ''
	while origr != repr(tokenlist):
		origr = repr(tokenlist)
		tokenlist = runParser(tokenlist)
	tokenlist = condParser(asopParser(tokenlist))
	return tokenlist<|endoftext|>


<commit_before>TITLE    = 'Channel Title'
PREFIX   = '/video/channelname'
ART      = 'art-default.jpg'
ICON     = 'icon-default.png'
STREAM_URL = 'http://smooth.ruv.cache.is/'

def Start(): # Initialize the plug-in

  Plugin.AddViewGroup("Details", viewMode="InfoList", mediaType="items")
  Plugin.AddViewGroup("List", viewMode="List", mediaType="items")

# Setup the default attributes for the ObjectContainer
ObjectContainer.title1 = TITLE
ObjectContainer.view_group = 'List'
ObjectContainer.art = R(ART)

# Setup the default attributes for the other objects
DirectoryObject.thumb = R(ICON)
DirectoryObject.art = R(ART)
VideoClipObject.thumb = R(ICON)
VideoClipObject.art = R(ART)

#####################################################################
@handler(PREFIX, TITLE)
def MainMenu():

  oc = ObjectContainer()

  oc.add(VideoClipObject(
  url = STREAM_URL + "lokad/4897620R12.mp4",
  title = "Live",
  summary = "Skemmtilegt",
  thumb = Callback(Thumb, url=thumb),
#  duration = "",
#  originally_available_at = date
  ))

  return oc

def Thumb(url):

  try:
    data = HTTP.Request(url, cacheTime = CACHE_1MONTH).content
    return DataObject(data, 'image/jpeg')
  except:
    return Redirect(R(ICON)) <commit_msg>direct play test fix<commit_after>TITLE    = 'Sarpurinn'
PREFIX   = '/video/sarpurinn'
ART      = 'art-default.jpg'
ICON     = 'icon-default.png'
STREAM_URL = 'http://smooth.ruv.cache.is/'

def Start(): # Initialize the plug-in

  Plugin.AddViewGroup("Details", viewMode="InfoList", mediaType="items")
  Plugin.AddViewGroup("List", viewMode="List", mediaType="items")

# Setup the default attributes for the ObjectContainer
ObjectContainer.title1 = TITLE
ObjectContainer.view_group = 'List'
ObjectContainer.art = R(ART)

# Setup the default attributes for the other objects
DirectoryObject.thumb = R(ICON)
DirectoryObject.art = R(ART)
VideoClipObject.thumb = R(ICON)
VideoClipObject.art = R(ART)

#####################################################################
@handler(PREFIX, TITLE)
def MainMenu():

  oc = ObjectContainer()

  oc.add(VideoClipObject(
  url = STREAM_URL + "lokad/4897620R12.mp4",
  title = "Live",
  summary = "Skemmtilegt",
  thumb = Callback(Thumb, url=thumb),
#  duration = "",
#  originally_available_at = date
  ))

  return oc

def Thumb(url):

  try:
    data = HTTP.Request(url, cacheTime = CACHE_1MONTH).content
    return DataObject(data, 'image/jpeg')
  except:
    return Redirect(R(ICON)) <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

# standard library dependencies
try:
    # prefer lxml as it supports XPath
    from lxml import etree
except ImportError:
    import xml.etree.ElementTree as etree

from operator import attrgetter
import itertools
from petl.compat import string_types, text_type

# internal dependencies
from petl.util.base import Table
from petl.io.sources import read_source_from_arg

def fromxml(source, *args, **kwargs):
    """
    Extract data from an XML file. E.g.::

        >>> import petl as etl
        >>> # setup a file to demonstrate with
        ... d = '''

        ...

        ...         foo	bar
        ...

        ...

        ...         a	1
        ...

        ...

        ...         b	2
        ...

        ...

        ...         c	2
        ...

        ...
'''
        >>> with open('example1.xml', 'w') as f:
        ...     f.write(d)
        ...
        212
        >>> table1 = etl.fromxml('example1.xml', 'tr', 'td')
        >>> table1
        +-----+-----+
        | foo | bar |
        +=====+=====+
        | 'a' | '1' |
        +-----+-----+
        | 'b' | '2' |
        +-----+-----+
        | 'c' | '2' |
        +-----+-----+

    If the data values are stored in an attribute, provide the attribute
    name as an extra positional argument::

        >>> d = '''

        ...

        ...
        ...

        ...

        ...
        ...

        ...

        ...
        ...

        ...

        ...
        ...

        ...
'''
        >>> with open('example2.xml', 'w') as f:
        ...     f.write(d)
        ...
        220
        >>> table2 = etl.fromxml('example2.xml', 'tr', 'td', 'v')
        >>> table2
        +-----+-----+
        | foo | bar |
        +=====+=====+
        | 'a' | '1' |
        +-----+-----+
        | 'b' | '2' |
        +-----+-----+
        | 'c' | '2' |
        +-----+-----+

    Data values can also be extracted by providing a mapping of field
    names to element paths::

        >>> d = '''

        ...     <row>
        ...         <foo>a</foo><baz><bar v='1'/><bar v='3'/></baz>
        ...     </row>
        ...     <row>
        ...         <foo>b</foo><baz><bar v='2'/></baz>
        ...     </row>
        ...     <row>
        ...         <foo>c</foo><baz><bar v='2'/></baz>
        ...     </row>
        ...
'''
        >>> with open('example3.xml', 'w') as f:
        ...     f.write(d)
        ...
        223
        >>> table3 = etl.fromxml('example3.xml', 'row',
        ...                      {'foo': 'foo', 'bar': ('baz/bar', 'v')})
        >>> table3
        +------------+-----+
        | bar        | foo |
        +============+=====+
        | ('1', '3') | 'a' |
        +------------+-----+
        | '2'        | 'b' |
        +------------+-----+
        | '2'        | 'c' |
        +------------+-----+

    If `lxml <http://lxml.de/>`_ is installed, full XPath expressions can be
    used.

    Note that the implementation is currently **not** streaming, i.e.,
    the whole document is loaded into memory.

    If multiple elements match a given field, all values are reported as a
    tuple.

    If there is more than one element name used for row values, a tuple
    or list of paths can be provided, e.g.,
    ``fromxml('example.html', './/tr', ('th', 'td'))``.

    """

    source = read_source_from_arg(source)
    return XmlView(source, *args, **kwargs)

class XmlView(Table):

    def __init__(self, source, *args, **kwargs):
        self.source = source
        self.args = args
        if len(args) == 2 and isinstance(args[1], (string_types, tuple, list)):
            self.rmatch = args[0]
            self.vmatch = args[1]
            self.vdict = None
            self.attr = None
        elif len(args) == 2 and isinstance(args[1], dict):
            self.rmatch = args[0]
            self.vmatch = None
            self.vdict = args[1]
            self.attr = None
        elif len(args) == 3:
            self.rmatch = args[0]
            self.vmatch = args[1]
            self.vdict = None
            self.attr = args[2]
        else:
            assert False, 'bad parameters'
        self.missing = kwargs.get('missing', None)

    def __iter__(self):
        vmatch = self.vmatch
        vdict = self.vdict

        with self.source.open('rb') as xmlf:

            tree = etree.parse(xmlf)
            if not hasattr(tree, 'iterfind'):
                # Python 2.6 compatibility
                tree.iterfind = tree.findall

            if vmatch is not None:
                # simple case, all value paths are the same
                for rowelm in tree.iterfind(self.rmatch):
                    if self.attr is None:
                        getv = attrgetter('text')
                    else:
                        getv = lambda e: e.get(self.attr)
                    if isinstance(vmatch, string_types):
                        # match only one path
                        velms = rowelm.findall(vmatch)
                    else:
                        # match multiple paths
                        velms = itertools.chain(*[rowelm.findall(enm)
                                                  for enm in vmatch])
                    yield tuple(getv(velm)
                                for velm in velms)

            else:
                # difficult case, deal with different paths for each field

                # determine output header
                flds = tuple(sorted(map(text_type, vdict.keys())))
                yield flds

                # setup value getters
                vmatches = dict()
                vgetters = dict()
                for f in flds:
                    vmatch = self.vdict[f]
                    if isinstance(vmatch, string_types):
                        # match element path
                        vmatches[f] = vmatch
                        vgetters[f] = element_text_getter(self.missing)
                    else:
                        # match element path and attribute name
                        vmatches[f] = vmatch[0]
                        attr = vmatch[1]
                        vgetters[f] = attribute_text_getter(attr, self.missing)

                # determine data rows
                for rowelm in tree.iterfind(self.rmatch):
                    yield tuple(vgetters[f](rowelm.findall(vmatches[f]))
                                for f in flds)

def element_text_getter(missing):
    def _get(v):
        if len(v) > 1:
            return tuple(e.text for e in v)
        elif len(v) == 1:
            return v[0].text
        else:
            return missing
    return _get

def attribute_text_getter(attr, missing):
    def _get(v):
        if len(v) > 1:
            return tuple(e.get(attr) for e in v)
        elif len(v) == 1:
            return v[0].get(attr)
        else:
            return missing
    return _get
<commit_msg>allow using a custom/restricted xml parser<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

# standard library dependencies
try:
    # prefer lxml as it supports XPath
    from lxml import etree
except ImportError:
    import xml.etree.ElementTree as etree

from operator import attrgetter
import itertools
from petl.compat import string_types, text_type

# internal dependencies
from petl.util.base import Table
from petl.io.sources import read_source_from_arg

def fromxml(source, *args, **kwargs):
    """
    Extract data from an XML file. E.g.::

        >>> import petl as etl
        >>> # setup a file to demonstrate with
        ... d = '''

        ...

        ...         foo	bar
        ...

        ...

        ...         a	1
        ...

        ...

        ...         b	2
        ...

        ...

        ...         c	2
        ...

        ...
'''
        >>> with open('example1.xml', 'w') as f:
        ...     f.write(d)
        ...
        212
        >>> table1 = etl.fromxml('example1.xml', 'tr', 'td')
        >>> table1
        +-----+-----+
        | foo | bar |
        +=====+=====+
        | 'a' | '1' |
        +-----+-----+
        | 'b' | '2' |
        +-----+-----+
        | 'c' | '2' |
        +-----+-----+

    If the data values are stored in an attribute, provide the attribute
    name as an extra positional argument::

        >>> d = '''

        ...

        ...
        ...

        ...

        ...
        ...

        ...

        ...
        ...

        ...

        ...
        ...

        ...
'''
        >>> with open('example2.xml', 'w') as f:
        ...     f.write(d)
        ...
        220
        >>> table2 = etl.fromxml('example2.xml', 'tr', 'td', 'v')
        >>> table2
        +-----+-----+
        | foo | bar |
        +=====+=====+
        | 'a' | '1' |
        +-----+-----+
        | 'b' | '2' |
        +-----+-----+
        | 'c' | '2' |
        +-----+-----+

    Data values can also be extracted by providing a mapping of field
    names to element paths::

        >>> d = '''

        ...     <row>
        ...         <foo>a</foo><baz><bar v='1'/><bar v='3'/></baz>
        ...     </row>
        ...     <row>
        ...         <foo>b</foo><baz><bar v='2'/></baz>
        ...     </row>
        ...     <row>
        ...         <foo>c</foo><baz><bar v='2'/></baz>
        ...     </row>
        ...
'''
        >>> with open('example3.xml', 'w') as f:
        ...     f.write(d)
        ...
        223
        >>> table3 = etl.fromxml('example3.xml', 'row',
        ...                      {'foo': 'foo', 'bar': ('baz/bar', 'v')})
        >>> table3
        +------------+-----+
        | bar        | foo |
        +============+=====+
        | ('1', '3') | 'a' |
        +------------+-----+
        | '2'        | 'b' |
        +------------+-----+
        | '2'        | 'c' |
        +------------+-----+

    If `lxml <http://lxml.de/>`_ is installed, full XPath expressions can be
    used.

    Note that the implementation is currently **not** streaming, i.e.,
    the whole document is loaded into memory.

    If multiple elements match a given field, all values are reported as a
    tuple.

    If there is more than one element name used for row values, a tuple
    or list of paths can be provided, e.g.,
    ``fromxml('example.html', './/tr', ('th', 'td'))``.

    Optionally a custom parser can be provided, e.g.,
    ``etl.fromxml('example1.xml', 'tr', 'td', parser=my_parser)``.

    """

    source = read_source_from_arg(source)
    return XmlView(source, *args, **kwargs)

class XmlView(Table):

    def __init__(self, source, *args, **kwargs):
        self.source = source
        self.args = args
        if len(args) == 2 and isinstance(args[1], (string_types, tuple, list)):
            self.rmatch = args[0]
            self.vmatch = args[1]
            self.vdict = None
            self.attr = None
        elif len(args) == 2 and isinstance(args[1], dict):
            self.rmatch = args[0]
            self.vmatch = None
            self.vdict = args[1]
            self.attr = None
        elif len(args) == 3:
            self.rmatch = args[0]
            self.vmatch = args[1]
            self.vdict = None
            self.attr = args[2]
        else:
            assert False, 'bad parameters'
        self.missing = kwargs.get('missing', None)
        self.user_parser = kwargs.get('parser', None)

    def __iter__(self):
        vmatch = self.vmatch
        vdict = self.vdict

        with self.source.open('rb') as xmlf:
            parser2 = _create_xml_parser(self.user_parser)
            tree = etree.parse(xmlf, parser=parser2)
            if not hasattr(tree, 'iterfind'):
                # Python 2.6 compatibility
                tree.iterfind = tree.findall

            if vmatch is not None:
                # simple case, all value paths are the same
                for rowelm in tree.iterfind(self.rmatch):
                    if self.attr is None:
                        getv = attrgetter('text')
                    else:
                        getv = lambda e: e.get(self.attr)
                    if isinstance(vmatch, string_types):
                        # match only one path
                        velms = rowelm.findall(vmatch)
                    else:
                        # match multiple paths
                        velms = itertools.chain(*[rowelm.findall(enm)
                                                  for enm in vmatch])
                    yield tuple(getv(velm)
                                for velm in velms)

            else:
                # difficult case, deal with different paths for each field

                # determine output header
                flds = tuple(sorted(map(text_type, vdict.keys())))
                yield flds

                # setup value getters
                vmatches = dict()
                vgetters = dict()
                for f in flds:
                    vmatch = self.vdict[f]
                    if isinstance(vmatch, string_types):
                        # match element path
                        vmatches[f] = vmatch
                        vgetters[f] = element_text_getter(self.missing)
                    else:
                        # match element path and attribute name
                        vmatches[f] = vmatch[0]
                        attr = vmatch[1]
                        vgetters[f] = attribute_text_getter(attr, self.missing)

                # determine data rows
                for rowelm in tree.iterfind(self.rmatch):
                    yield tuple(vgetters[f](rowelm.findall(vmatches[f]))
                                for f in flds)

def _create_xml_parser(user_parser):
    if user_parser is not None:
        return user_parser
    try:
        # Default lxml parser.
        # This will throw an error if parser is not set and lxml could not be imported
        # because Python's built XML parser doesn't like the `resolve_entities` kwarg.
        # return etree.XMLParser(resolve_entities=False)
        return etree.XMLParser(resolve_entities=False)
    except TypeError:
        # lxml not available
        return None

def element_text_getter(missing):
    def _get(v):
        if len(v) > 1:
            return tuple(e.text for e in v)
        elif len(v) == 1:
            return v[0].text
        else:
            return missing
    return _get

def attribute_text_getter(attr, missing):
    def _get(v):
        if len(v) > 1:
            return tuple(e.get(attr) for e in v)
        elif len(v) == 1:
            return v[0].get(attr)
        else:
            return missing
    return _get
<|endoftext|>