<commit_before>from .utils import config, checks, formats
import discord
from discord.ext import commands
import discord.utils
import random, json, asyncio
from urllib.parse import unquote

class Radio:
    """The radio-bot related commands."""

    def __init__(self, bot):
        self.bot = bot
        if not discord.opus.is_loaded():
            discord.opus.load_opus('/usr/local/lib/libopus.so') #FreeBSD path

        self.player = None
        self.stopped = True
        self.break_loop = asyncio.Event()
        #self.break_loop.clear()
        self.q = asyncio.Queue()
        self.play_next_song = asyncio.Event()
        self.current_song = None
        self.songs_dir = 'radio/'
        self.songs = []
        self.playlists = {}
        self.update_song_list()

    @property
    def is_playing(self):
        return self.player is not None and self.player.is_playing() and not self.stopped

    def toggle_next_song(self):
        if not self.stopped:
            self.bot.loop.call_soon_threadsafe(self.play_next_song.set)

    def update_song_list(self):
        self.songs = self.bot.yadisk.list_files(self.songs_dir)
    #    self.update_playlists()

    #def update_playlists(self):
    #    pl_files = self.bot.yadisk.list_files(self.songs_dir + 'playlists/')
    #    for pl in pl_files:
    #        pl_url = self.bot.yadisk.direct_link(self.songs_dir + 'playlists' + pl)
    #        pl_data = self.bot.yadisk._get(pl_url).json()['songs']
    #        self.playlists[pl] = []
    #        for song in pl_data:
    #            if song in self.songs:
    #                self.playlists[pl].append(song)

    @commands.command(pass_context=True)
    async def join(self, ctx, *, channel_name : str):
        """Зайти на указанный голосовой канал."""
        if self.bot.is_voice_connected():
            await ctx.invoke(self.leave)

        check = lambda c: c.name == channel_name and c.type == discord.ChannelType.voice
        channel = discord.utils.find(check, ctx.message.server.channels)
        if channel is None:
            await self.bot.say('Нет такого голосового канала.')
        await self.bot.join_voice_channel(channel)

    @commands.command(pass_context=True)
    async def leave(self, ctx):
        """Покинуть текущий голосовой канал."""
        await ctx.invoke(self.stop)
        await self.bot.voice.disconnect()

    @commands.command()
    async def pause(self):
        """Приостановить воспроизведение."""
        if self.player is not None:
            self.player.pause()

    @commands.command()
    async def resume(self):
        """Продолжить воспроизведение."""
        if self.player is not None and not self.is_playing:
            self.player.resume()

    @commands.command()
    async def skip(self):
        """Перейти к следующей песне в очереди."""
        if self.player is not None and self.is_playing:
            self.player.stop()
            self.toggle_next_song()

    @commands.command()
    async def stop(self):
        """Остановить воспроизведение."""
        if self.is_playing:
            self.break_loop.set()
            self.play_next_song.set()
            self.stopped = True
            self.player.stop()

    @commands.command(pass_context=True)
    async def play(self, ctx):
        """Начать воспроизведение песен из очереди."""
        if self.player is not None and not self.stopped:
            if not self.is_playing:
                await ctx.invoke(self.resume)
                return
            else:
                await self.bot.say('Уже играю песенку.')
                return

        while True:
            if not self.bot.is_voice_connected():
                author_channel = ctx.message.author.voice_channel
                if author_channel is not None:
                    print('Joining {}.'.format(author_channel.name))
                    await ctx.invoke(self.join, channel_name=author_channel.name)
                else:
                    await self.bot.say('Не выбран голосовой канал.')
                    return

            if self.q.empty():
                await self.q.put(random.choice(self.songs))
            self.play_next_song.clear()
            self.current = await self.q.get()
            self.player = self.bot.voice.create_ffmpeg_player(
                self.bot.yadisk.direct_link(self.songs_dir + self.current),
                after=self.toggle_next_song)
                #options="-loglevel debug -report",
                #headers = dict(self.bot.pycopy.session.headers)
            self.stopped = False
            self.player.start()
            song_name = unquote(self.current.split('/')[-1])
            await self.bot.change_status(discord.Game(name=song_name))

            await self.play_next_song.wait()
            if self.break_loop.is_set():
                self.break_loop.clear()
                return
        await self.bot.say('Leaving play loop')

    @commands.command(aliases=['c'])
    async def current(self):
        """Что там играет?"""
        if self.is_playing:
            song_name = unquote(self.current.split('/')[-1])
            await self.bot.say(song_name)

    @commands.command()
    async def update(self):
        """Обновить список песен."""
        self.update_song_list()
        await self.bot.say("Найдено {} песенок".format(len(self.songs)))

    @commands.command()
    async def list(self):
        """Вывести список всех доступных песен."""
        song_list = ""
        id = 1
        for song in self.songs:
            song_list += "{}. {}\n".format(id, song)
            id += 1
            if len(song_list) > 1800:
                await self.bot.say(song_list)
                song_list = ''
        await self.bot.say(song_list)

    @commands.command(aliases=['ss'])
    async def searchsong(self, search_word : str):
        """Искать песню по названию."""
        search_result = ""
        id = 1
        found = False
        for song in self.songs:
            if search_word.lower() in song.lower():
                found = True
                search_result += "{}. {}\n".format(id, song)
                if len(search_result) > 1800:
                    await self.bot.say(search_result)
                    search_result = ''
            id += 1
        if not found:
            search_result = "Нет такой песни."
        await self.bot.say(search_result)

    @commands.command()
    async def add(self, song_num : int):
        """Добавить в конец очереди песню с данным номером."""
        await self.q.put(self.songs[song_num-1])
        await self.bot.say("{} будет следующей песенкой".format(self.songs[song_num-1]))

    #@commands.group(pass_context=True, aliases=['pl'])
    #async def playlist(self):
    #    return

    #@playlist.command()
    #async def add(self, playlist : str, song : str):
    #    if not playlist in self.playlists:
    #        await ctx.invoke(self.new, playlist=playlist)
    #    self.playlists[playlist].append(song)
    #    self.bot.yadisk.upload(self.songs_dir + 'playlists' + playlist, self.playlists[playlist])

def setup(bot):
    bot.add_cog(Radio(bot))
<commit_msg>Update radio.py<commit_after>from .utils import config, checks, formats
import discord
from discord.ext import commands
import discord.utils
import random, json, asyncio
from urllib.parse import unquote

class Radio:
    """The radio-bot related commands."""

    def __init__(self, bot):
        self.bot = bot
        if not discord.opus.is_loaded():
            discord.opus.load_opus('/usr/local/lib/libopus.so') #FreeBSD path

        self.player = None
        self.stopped = True
        self.break_loop = asyncio.Event()
        #self.break_loop.clear()
        self.q = asyncio.Queue()
        self.play_next_song = asyncio.Event()
        self.current_song = None
        self.songs_dir = 'radio/'
        self.songs = []
        self.playlists = {}
        self.update_song_list()

    @property
    def is_playing(self):
        return self.player is not None and self.player.is_playing() and not self.stopped

    def toggle_next_song(self):
        if not self.stopped:
            self.bot.loop.call_soon_threadsafe(self.play_next_song.set)

    def update_song_list(self):
        self.songs = self.bot.yadisk.list_files(self.songs_dir)
        self.update_playlists()

    def update_playlists(self):
        pl_files = self.bot.yadisk.list_files(self.songs_dir + 'playlists/')
        for pl in pl_files:
            pl_url = self.bot.yadisk.direct_link(self.songs_dir + 'playlists' + pl)
            pl_data = self.bot.yadisk._get(pl_url).json()['songs']
            self.playlists[pl] = []
            for song in pl_data:
                if song in self.songs:
                    self.playlists[pl].append(song)

    @commands.command(pass_context=True)
    async def join(self, ctx, *, channel_name : str):
        """Зайти на указанный голосовой канал."""
        if self.bot.is_voice_connected():
            await ctx.invoke(self.leave)

        check = lambda c: c.name == channel_name and c.type == discord.ChannelType.voice
        channel = discord.utils.find(check, ctx.message.server.channels)
        if channel is None:
            await self.bot.say('Нет такого голосового канала.')
        await self.bot.join_voice_channel(channel)

    @commands.command(pass_context=True)
    async def leave(self, ctx):
        """Покинуть текущий голосовой канал."""
        await ctx.invoke(self.stop)
        await self.bot.voice.disconnect()

    @commands.command()
    async def pause(self):
        """Приостановить воспроизведение."""
        if self.player is not None:
            self.player.pause()

    @commands.command()
    async def resume(self):
        """Продолжить воспроизведение."""
        if self.player is not None and not self.is_playing:
            self.player.resume()

    @commands.command()
    async def skip(self):
        """Перейти к следующей песне в очереди."""
        if self.player is not None and self.is_playing:
            self.player.stop()
            self.toggle_next_song()

    @commands.command()
    async def stop(self):
        """Остановить воспроизведение."""
        if self.is_playing:
            self.break_loop.set()
            self.play_next_song.set()
            self.stopped = True
            self.player.stop()

    @commands.command(pass_context=True)
    async def play(self, ctx):
        """Начать воспроизведение песен из очереди."""
        if self.player is not None and not self.stopped:
            if not self.is_playing:
                await ctx.invoke(self.resume)
                return
            else:
                await self.bot.say('Уже играю песенку.')
                return

        while True:
            if not self.bot.is_voice_connected():
                author_channel = ctx.message.author.voice_channel
                if author_channel is not None:
                    print('Joining {}.'.format(author_channel.name))
                    await ctx.invoke(self.join, channel_name=author_channel.name)
                else:
                    await self.bot.say('Не выбран голосовой канал.')
                    return

            if self.q.empty():
                await self.q.put(random.choice(self.songs))
            self.play_next_song.clear()
            self.current = await self.q.get()
            self.player = self.bot.voice.create_ffmpeg_player(
                self.bot.yadisk.direct_link(self.songs_dir + self.current),
                after=self.toggle_next_song)
                #options="-loglevel debug -report",
                #headers = dict(self.bot.pycopy.session.headers)
            self.stopped = False
            self.player.start()
            song_name = unquote(self.current.split('/')[-1])
            await self.bot.change_status(discord.Game(name=song_name))

            await self.play_next_song.wait()
            if self.break_loop.is_set():
                self.break_loop.clear()
                return
        await self.bot.say('Leaving play loop')

    @commands.command(aliases=['c'])
    async def current(self):
        """Что там играет?"""
        if self.is_playing:
            song_name = unquote(self.current.split('/')[-1])
            await self.bot.say(song_name)

    @commands.command()
    async def update(self):
        """Обновить список песен."""
        self.update_song_list()
        await self.bot.say("Найдено {} песенок".format(len(self.songs)))

    @commands.command()
    async def list(self):
        """Вывести список всех доступных песен."""
        song_list = ""
        id = 1
        for song in self.songs:
            song_list += "{}. {}\n".format(id, song)
            id += 1
            if len(song_list) > 1800:
                await self.bot.say(song_list)
                song_list = ''
        await self.bot.say(song_list)

    @commands.command(aliases=['ss'])
    async def searchsong(self, search_word : str):
        """Искать песню по названию."""
        search_result = ""
        id = 1
        found = False
        for song in self.songs:
            if search_word.lower() in song.lower():
                found = True
                search_result += "{}. {}\n".format(id, song)
                if len(search_result) > 1800:
                    await self.bot.say(search_result)
                    search_result = ''
            id += 1
        if not found:
            search_result = "Нет такой песни."
        await self.bot.say(search_result)

    @commands.command()
    async def add(self, song_num : int):
        """Добавить в конец очереди песню с данным номером."""
        await self.q.put(self.songs[song_num-1])
        await self.bot.say("{} будет следующей песенкой".format(self.songs[song_num-1]))

    @commands.group(pass_context=True, aliases=['pl'])
    async def playlist(self):
        return

    @playlist.command()
    async def add(self, playlist : str, song : str):
        if not playlist in self.playlists:
            await ctx.invoke(self.new, playlist=playlist)
        self.playlists[playlist].append(song)
        pl_json = {'name': playlist, 'songs': self.playlists[playlist]}
        self.bot.yadisk.upload(self.songs_dir + 'playlists' + playlist, json.dumps(pl_json))

def setup(bot):
    bot.add_cog(Radio(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
import argparse
import os.path
import threading
import numpy as np
from pygiftgrab import (VideoSourceFactory,
                        ColourSpace, IObservableObserver,
                        VideoTargetFactory, Codec)

"""
This script provides a multi-threading reliability check.
The background is issue #16. It looks like in applications
where multiple Python threads are involved, occasionally
the acquisition of the Global Interpreter Lock leads to a
deadlocks, which crashes the whole application with a
non-specific segmentation fault.

In this script we run a number of multi-threaded GIFT-Grab
pipelines, which should serve as a validation that this
problem is fixed.
"""

buffer_red = None
lock = threading.Lock()

class BuffererRed(IObservableObserver):

    def __init__(self):
        super(BuffererRed, self).__init__()

    def update(self, frame):
        global buffer_red
        with lock:
            data = frame.data(True)
            if buffer_red is None:
                buffer_red = np.copy(data)
            else:
                buffer_red[:, :, :] = data[:, :, :]

class HistogrammerRed(threading.Thread):

    def __init__(self):
        super(HistogrammerRed, self).__init__()
        self.running = False

    def run(self):
        if self.running:
            return

        global buffer_red
        histogram, num_bins = None, 10
        redness_scale = np.array([i for i in range(1, num_bins + 1)], np.float)
        self.running = True
        while self.running:
            with lock:
                if buffer_red is not None:
                    histogram, _ = np.histogram(
                        buffer_red[:, :, 2], bins=num_bins, range=(0, 256)
                    )
            if histogram is not None:
                redness = np.sum(histogram * redness_scale)
                redness /= np.sum(histogram)
                redness /= num_bins
                redness *= 100
                print('Redness: {} %'.format(int(round(redness))))
            sleep(0.100)

    def stop(self):
        self.running = False

class Dyer(IObservableObserver):

    def __init__(self, channel, increment):
        super(Dyer, self).__init__()
        assert 0 <= channel < 3
        assert 0 <= increment < 256
        self.channel = channel
        self.increment = increment

    def update(self, frame):
        with lock:
            data = frame.data(True)
            channel_data = data[:, :, self.channel]
            channel_data[channel_data < 255 - self.increment] += self.increment

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True,
                        metavar='VIDEO_FILE',
                        help='Input video file (HEVC-encoded MP4)')
    args = parser.parse_args()
    in_file = args.input

    filename = os.path.basename(in_file)
    filename, ext = os.path.splitext(filename)
    assert filename
    assert ext == '.mp4'

    sfac = VideoSourceFactory.get_instance()
    reader = sfac.create_file_reader(in_file, ColourSpace.BGRA)

    tfac = VideoTargetFactory.get_instance()
    frame_rate = reader.get_frame_rate()

    red_dyer = Dyer(2, 64)
    green_dyer = Dyer(1, 64)

    bufferer_red = BuffererRed()

    hist = HistogrammerRed()
    hist.start()

    red_file = os.path.join('.', ''.join([filename, '-red', ext]))
    red_writer = tfac.create_file_writer(Codec.HEVC, red_file, frame_rate)
    yellow_file = os.path.join('.', ''.join([filename, '-yellow', ext]))
    yellow_writer = tfac.create_file_writer(Codec.HEVC, yellow_file, frame_rate)

    reader.attach(red_dyer)
    red_dyer.attach(red_writer)
    red_dyer.attach(bufferer_red)
    red_dyer.attach(green_dyer)
    green_dyer.attach(yellow_writer)

    sleep(20)  # operate pipeline for 20 sec
    hist.stop()

    reader.detach(red_dyer)
    red_dyer.detach(red_writer)
    red_dyer.detach(bufferer_red)
    red_dyer.detach(green_dyer)
    green_dyer.detach(yellow_writer)
<commit_msg>Issue #16: added a BuffererOrig class, a virtual replica of BuffererRed<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
import argparse
import os.path
import threading
import numpy as np
from pygiftgrab import (VideoSourceFactory,
                        ColourSpace, IObservableObserver,
                        VideoTargetFactory, Codec)

"""
This script provides a multi-threading reliability check.
The background is issue #16. It looks like in applications
where multiple Python threads are involved, occasionally
the acquisition of the Global Interpreter Lock leads to a
deadlocks, which crashes the whole application with a
non-specific segmentation fault.

In this script we run a number of multi-threaded GIFT-Grab
pipelines, which should serve as a validation that this
problem is fixed.
"""

buffer_red, buffer_orig = None, None
lock = threading.Lock()

class BuffererRed(IObservableObserver):

    def __init__(self):
        super(BuffererRed, self).__init__()

    def update(self, frame):
        global buffer_red
        with lock:
            data = frame.data(True)
            if buffer_red is None:
                buffer_red = np.copy(data)
            else:
                buffer_red[:, :, :] = data[:, :, :]

class BuffererOrig(IObservableObserver):

    def __init__(self):
        super(BuffererOrig, self).__init__()

    def update(self, frame):
        global buffer_orig
        with lock:
            data = frame.data(True)
            if buffer_orig is None:
                buffer_orig = np.copy(data)
            else:
                buffer_red[:, :, :] = data[:, :, :]

class HistogrammerRed(threading.Thread):

    def __init__(self):
        super(HistogrammerRed, self).__init__()
        self.running = False

    def run(self):
        if self.running:
            return

        global buffer_red
        histogram, num_bins = None, 10
        redness_scale = np.array([i for i in range(1, num_bins + 1)], np.float)
        self.running = True
        while self.running:
            with lock:
                if buffer_red is not None:
                    histogram, _ = np.histogram(
                        buffer_red[:, :, 2], bins=num_bins, range=(0, 256)
                    )
            if histogram is not None:
                redness = np.sum(histogram * redness_scale)
                redness /= np.sum(histogram)
                redness /= num_bins
                redness *= 100
                print('Redness: {} %'.format(int(round(redness))))
            sleep(0.100)

    def stop(self):
        self.running = False

class Dyer(IObservableObserver):

    def __init__(self, channel, increment):
        super(Dyer, self).__init__()
        assert 0 <= channel < 3
        assert 0 <= increment < 256
        self.channel = channel
        self.increment = increment

    def update(self, frame):
        with lock:
            data = frame.data(True)
            channel_data = data[:, :, self.channel]
            channel_data[channel_data < 255 - self.increment] += self.increment

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True,
                        metavar='VIDEO_FILE',
                        help='Input video file (HEVC-encoded MP4)')
    args = parser.parse_args()
    in_file = args.input

    filename = os.path.basename(in_file)
    filename, ext = os.path.splitext(filename)
    assert filename
    assert ext == '.mp4'

    sfac = VideoSourceFactory.get_instance()
    reader = sfac.create_file_reader(in_file, ColourSpace.BGRA)

    tfac = VideoTargetFactory.get_instance()
    frame_rate = reader.get_frame_rate()

    red_dyer = Dyer(2, 64)
    green_dyer = Dyer(1, 64)

    bufferer_red = BuffererRed()

    hist = HistogrammerRed()
    hist.start()

    red_file = os.path.join('.', ''.join([filename, '-red', ext]))
    red_writer = tfac.create_file_writer(Codec.HEVC, red_file, frame_rate)
    yellow_file = os.path.join('.', ''.join([filename, '-yellow', ext]))
    yellow_writer = tfac.create_file_writer(Codec.HEVC, yellow_file, frame_rate)

    reader.attach(red_dyer)
    red_dyer.attach(red_writer)
    red_dyer.attach(bufferer_red)
    red_dyer.attach(green_dyer)
    green_dyer.attach(yellow_writer)

    sleep(20)  # operate pipeline for 20 sec
    hist.stop()

    reader.detach(red_dyer)
    red_dyer.detach(red_writer)
    red_dyer.detach(bufferer_red)
    red_dyer.detach(green_dyer)
    green_dyer.detach(yellow_writer)
<|endoftext|>


<commit_before>import unittest
from unittest import mock
import logging
logging.basicConfig()

from external import SCORE_URL
from external.lfp_api import get_score, get_calendar

from tools.web import get_url_content as _url_download

class TestLFPApi(unittest.TestCase):

    @mock.patch('external.lfp_api.get_url_content')
    def test_get_scores(self, mock_url_content):
        """
        This test ensures LFP website does not change its html structure over time
        by checking some known results
        """

        mock_url_content.return_value = _url_download(SCORE_URL % (101, 1))

        self.assertEqual(get_score(1, 'Monaco', 'Toulouse'), (3, 2))
        self.assertEqual(get_score(1, 'Troyes', 'Rennes'), (1, 1))
        self.assertEqual(get_score(1, 'Marseille', 'Dijon'), (3, 0))

    def test_download_calendar(self):
        """
        This test is to ensure LFP calendar is still available for download
        and the ICAL parsing is not broken over time
        """
        calendar = get_calendar()
        self.assertEqual(len(calendar), 380)

        for key in ['date', 'time', 'team_a', 'team_b', 'week']:
            self.assertTrue(key in calendar[0])
<commit_msg>use fixed list of lfp teams for tests<commit_after>import unittest
from unittest import mock
import logging
logging.basicConfig()

from external import SCORE_URL
from external.lfp_api import get_score, get_calendar
from tools.web import get_url_content as _url_download
from . import TEST_LFP_TEAM_MAP

class TestLFPApi(unittest.TestCase):

    @mock.patch.dict('external.lfp_results.LFP_TEAM_MAP', TEST_LFP_TEAM_MAP)
    @mock.patch('external.lfp_api.get_url_content')
    def test_get_scores(self, mock_url_content):
        """
        This test ensures LFP website does not change its html structure over time
        by checking some known results
        """

        mock_url_content.return_value = _url_download(SCORE_URL % (101, 1))

        self.assertEqual(get_score(1, 'Monaco', 'Toulouse'), (3, 2))
        self.assertEqual(get_score(1, 'Troyes', 'Rennes'), (1, 1))
        self.assertEqual(get_score(1, 'Marseille', 'Dijon'), (3, 0))

    def test_download_calendar(self):
        """
        This test is to ensure LFP calendar is still available for download
        and the ICAL parsing is not broken over time
        """
        calendar = get_calendar()
        self.assertEqual(len(calendar), 380)

        for key in ['date', 'time', 'team_a', 'team_b', 'week']:
            self.assertTrue(key in calendar[0])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import yaml
import shutil
import tempfile

from lektor.pluginsystem import Plugin
from lektor.publisher import Publisher, Command

class GaePublisher(Publisher):
    def __init__(self, env, output_path):
        super(GaePublisher, self).__init__(env, output_path)
        self.temp_path = tempfile.mkdtemp()

    def __del__(self):
        shutil.rmtree(self.temp_path)

    def write_301(self):
        # 301 redirect to canonical URLs with trailing slash
        target = os.path.join(self.temp_path, "301.php")
        with open(target, "w") as file:
            file.write("<?php header('Location: ' + strtok($_SERVER['REQUEST_URI'],'?') + '/', TRUE, 301); ?>")
        return True

    def write_404(self):
        # 404 response code with custom error page
        if not os.path.isfile(os.path.join(self.output_path, "404.html")):
            return False
        target = os.path.join(self.temp_path, "404.php")
        with open(target, "w") as file:
            file.write("<?php http_response_code(404); require('404.html'); ?>")
        return True

    def get_files(self):
        # Get list of output files
        all_files = []
        for root, dirs, files in os.walk(self.output_path):
            files   = [f for f in files if not f[0] == "."]
            dirs[:] = [d for d in dirs  if not d[0] == "."]
            for f in files:
                fullpath = os.path.join(root, f)
                relpath = os.path.relpath(fullpath, self.output_path)
                if relpath != "404.html":
                    all_files.append(relpath)
        return all_files

    def gen_handler(self, remote_path, local_path):
        # Generate a URL handler
        return {
            "url": remote_path.replace(".", "\\."),
            "static_files": local_path,
            "upload": local_path.replace(".", "\\."),
            "http_headers": {
                "Vary": "Accept-Encoding",
            },
        }

    def publish(self, target_url, credentials=None, **extra):
        # Initialize the app
        (empty, application, version) = os.path.split(str(target_url.path)) + (1,)
        app = {
            "application": application,
            "version": version,
            "runtime": 'php55',
            "api_version": 1,
            "handlers" : [],
        }

        # Add a handler for each static page
        for path in self.get_files():
            local_path = os.path.join(self.temp_path, path)
            remote_path = "/".join([""] + path.split(os.path.sep))
            if os.path.basename(local_path) == "index.html":
                # Strip off the index.html
                remote_path = os.path.dirname(remote_path)
                if remote_path != "/":
                    # Add 301 redirects to canonical URLs
                    remote_path = remote_path + "/"
                    app["handlers"].append({
                        "url": os.path.dirname(remote_path).replace(".", "\\."),
                        "script": "301.php",
                    })
            app["handlers"].append(self.gen_handler(remote_path, local_path))

        # Create PHP handlers
        self.write_301()
        if self.write_404():
            app["handlers"].append({
                "url": "/.*",
                "script": "404.php",
            })

        # Generate the YAML
        target = os.path.join(self.temp_path, "app.yaml")
        with open(target, "w") as file:
            file.write(yaml.dump(app))

        # TODO: Find the Google App Engine SDK
        for line in Command(['python', r'C:\Program Files (x86)\Google\google_appengine\appcfg.py', 'update', self.temp_path]):
            yield line

class GaePlugin(Plugin):
    name = u"Google App Engine"
    description = u"Publishes your Lektor site to Google App Engine."

    def on_setup_env(self, **extra):
        scheme = "gae";
        if hasattr(self.env, "publishers"):
            self.env.add_publisher(scheme, GaePublisher)
        else:
            from lektor.publisher import publishers
            publishers[scheme] = GaePublisher
<commit_msg>A working version which successfully deploys<commit_after># -*- coding: utf-8 -*-
import os
import re
import sys
import yaml

from lektor.pluginsystem import Plugin
from lektor.publisher import Publisher, Command

class GaePublisher(Publisher):
    def write_301(self):
        # 301 redirect to canonical URLs with trailing slash
        target = os.path.join(self.output_path, "301.php")
        with open(target, "w") as file:
            file.write("<?php header('Location: http://'.$_SERVER['HTTP_HOST'].strtok($_SERVER['REQUEST_URI'],'?').'/', TRUE, 301); ?>")
        return True

    def write_404(self):
        # 404 response code with custom error page
        if not os.path.isfile(os.path.join(self.output_path, "404.html")):
            return False
        target = os.path.join(self.output_path, "404.php")
        with open(target, "w") as file:
            file.write("<?php http_response_code(404); require('404.html'); ?>")
        return True

    def get_files(self):
        # Get list of output files
        all_files = []
        for root, dirs, files in os.walk(self.output_path):
            files   = [f for f in files if not f[0] == "."]
            dirs[:] = [d for d in dirs  if not d[0] == "."]
            for f in files:
                fullpath = os.path.join(root, f)
                relpath = os.path.relpath(fullpath, self.output_path)
                if relpath != "404.html":
                    all_files.append(relpath)
        return all_files

    def gen_handler(self, remote_path, local_path):
        # Generate a URL handler
        return {
            "url": "/" + re.escape(remote_path),
            "static_files": local_path.replace("\\", "\\\\"),
            "upload": re.escape(local_path),
            "http_headers": {
                "Vary": "Accept-Encoding",
            },
        }

    def find_sdk(self):
        for path in os.environ['PATH'].split(os.pathsep):
            cmd = os.path.join(path, 'appcfg.py')
            if os.path.isfile(cmd):
                return cmd
        raise RuntimeError('Could not locate Google App Engine SDK.')

    def publish(self, target_url, credentials=None, **extra):
        # Initialize the app; default to version 1 if not specified
        (empty, application, version) = os.path.split(str(target_url.path)) + (1,)
        app = {
            "application": application,
            "version": version,
            "runtime": 'php55',
            "api_version": 1,
            "handlers" : [],
        }

        # Add a handler for each static page
        for path in self.get_files():
            # Force use of forward slashes for app engine
            local_path = "/".join(path.split(os.path.sep))
            remote_path = local_path
            if os.path.basename(local_path) == "index.html":
                # Strip off the index.html
                remote_path = os.path.dirname(remote_path)
                if remote_path != "":
                    # Add 301 redirects to canonical URLs
                    remote_path = remote_path + "/"
                    app["handlers"].append({
                        "url": "/" + re.escape(os.path.dirname(remote_path)),
                        "script": "301.php",
                    })
            app["handlers"].append(self.gen_handler(remote_path, local_path))

        # Create PHP handlers
        self.write_301()
        if self.write_404():
            app["handlers"].append({
                "url": "/.*",
                "script": "404.php",
            })

        # Generate the YAML
        target = os.path.join(self.output_path, "app.yaml")
        with open(target, "w") as file:
            file.write(yaml.dump(app))

        cmd = self.find_sdk()
        for line in Command([sys.executable, cmd, 'update', self.output_path]):
            yield line

class GaePlugin(Plugin):
    name = u"Google App Engine"
    description = u"Publishes your Lektor site to Google App Engine."

    def on_setup_env(self, **extra):
        scheme = "gae";
        if hasattr(self.env, "publishers"):
            self.env.add_publisher(scheme, GaePublisher)
        else:
            from lektor.publisher import publishers
            publishers[scheme] = GaePublisher
<|endoftext|>


<commit_before>import math

def distance(x1, y1, x2, y2):
	"""calc distance between (x1,y1) and (x2, y2)"""
	l = (x2 - x1) * (x2 - x1 ) + (y2 - y1) * (y2 - y1)
	return math.sqrt(l)

<commit_msg>added distance comparison<commit_after>import math

def distance(x1, y1, x2, y2):
	"""calc distance between (x1,y1) and (x2, y2)"""
	l = (x2 - x1) * (x2 - x1 ) + (y2 - y1) * (y2 - y1)
	return math.sqrt(l)

def distance2(x1, y1, x2, y2):
	"""calc distance between (x1,y1) and (x2, y2)
	without sqrt of result - faster for length comparisons"""
	l = (x2 - x1) * (x2 - x1 ) + (y2 - y1) * (y2 - y1)
	return l

<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import random
import numpy as np

from pyannote.core import Segment
from pyannote.core import Timeline
from pyannote.core import Annotation
from pyannote.core import SlidingWindow
from pyannote.core import PYANNOTE_SEGMENT
from pyannote.core import PYANNOTE_TRACK
from pyannote.core import PYANNOTE_LABEL

def random_segment(segments, weighted=False):
    """Generate segment with probability proportional to its duration"""

    p = None
    if weighted:
        total = float(sum(s.duration for s in segments))
        p = [s.duration / total for s in segments]

    n_segments = len(segments)
    while True:
        i = np.random.choice(n_segments, p=p)
        yield segments[i]

def random_subsegment(segment, duration):
    """Pick a subsegment at random"""
    if segment.duration < duration:
        raise ValueError('segment is too short')
    while True:
        t = segment.start + random.random() * (segment.duration - duration)
        yield Segment(t, t + duration)

class SlidingSegments(object):
    """Fixed-duration running segment generator

    Parameters
    ----------
    duration: float, optional
        Segment duration. Defaults to 3.2 seconds.
    step: float, optional
        Step duration. Defaults to 0.8 seconds.
    """

    def __init__(self, duration=3.2, step=0.8):
        super(SlidingSegments, self).__init__()

        if not duration > 0:
            raise ValueError('Duration must be strictly positive.')
        self.duration = duration

        if not step > 0:
            raise ValueError('Step must be strictly positive.')
        self.step = step

    def signature(self):
        return {'type': PYANNOTE_SEGMENT, 'duration': self.duration}

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for segment in self.iter_segments(reference):
            yield segment

    def iter_segments(self, source):
        """
        Parameters
        ----------
        source : float, Segment, Timeline or Annotation
            If `float`, yield running segments within [0, source).
            If `Segment`, yield running segments within this segment.
            If `Timeline`, yield running segments within this timeline.
            If `Annotation`, yield running segments within its timeline.
        """

        if isinstance(source, Annotation):
            segments = source.get_timeline()

        elif isinstance(source, Timeline):
            segments = source

        elif isinstance(source, Segment):
            segments = [source]

        elif isinstance(source, (int, float)):
            if not self.duration > 0:
                raise ValueError('Duration must be strictly positive.')
            segments = [Segment(0, duration)]

        else:
            raise TypeError('source must be float, Segment, Timeline or Annotation')

        segments = [segment for segment in segments
                            if segment.duration > self.duration]
        if not segments:
            raise ValueError('Source must contain at least one segment longer than requested duration.')

        for segment in segments:
            window = SlidingWindow(duration=self.duration,
                                   step=self.step,
                                   start=segment.start,
                                   end=segment.end)
            for s in window:
                # this is needed because window may go beyond segment.end
                if s in segment:
                    yield s

class RandomSegments(object):
    """Random segment generator

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    weighted: boolean, optional
        When True, probability of generating a segment is proportional to its
        duration.
    """
    def __init__(self, duration=0., weighted=False):
        super(RandomSegments, self).__init__()
        self.duration = duration
        self.weighted = weighted

    def signature(self):
        return {'type': PYANNOTE_SEGMENT, 'duration': self.duration}

    def pick(self, segment):
        """Pick a subsegment at random"""
        t = segment.start + random.random() * (segment.duration - self.duration)
        return Segment(t, t + self.duration)

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for segment in self.iter_segments(reference):
            yield segment

    def iter_segments(self, source):
        """
        Parameters
        ----------
        source : float, Segment, Timeline or Annotation
            If `float`, yield random segments within [0, source).
            If `Segment`, yield random segments within this segment.
            If `Timeline`, yield random segments within this timeline.
            If `Annotation`, yield random segments within its timeline.
        """

        if isinstance(source, Annotation):
            segments = source.get_timeline()

        elif isinstance(source, Timeline):
            segments = source

        elif isinstance(source, Segment):
            segments = [source]

        elif isinstance(source, (int, float)):
            if not self.duration > 0:
                raise ValueError('Duration must be strictly positive.')
            segments = [Segment(0, duration)]

        else:
            raise TypeError('source must be float, Segment, Timeline or Annotation')

        segments = [segment for segment in segments
                            if segment.duration > self.duration]
        if not segments:
            raise ValueError('Source must contain at least one segment longer than requested duration.')

        for segment in random_segment(segments, weighted=self.weighted):
            if self.duration:
                if segment.duration < self.duration:
                    continue
                yield next(random_subsegment(segment, self.duration))
            else:
                yield segment

class RandomTracks(object):
    """(segment, track) tuple generator

    Parameters
    ----------
    yield_label: boolean, optional
        When True, yield (segment, track, label) tuples.
        Defaults to yielding (segment, track) tuples.
    """

    def __init__(self, yield_label=False):
        super(RandomTracks, self).__init__()
        self.yield_label = yield_label

    def signature(self):
        signature = [
            {'type': PYANNOTE_SEGMENT, 'duration': 0.0},
            {'type': PYANNOTE_TRACK}
        ]
        if self.yield_label:
            signature.append({'type': PYANNOTE_LABEL})
        return signature

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for track in self.iter_tracks(reference):
            yield track

    def iter_tracks(self, from_annotation):
        """Yield (segment, track) tuples

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which tracks are obtained.
        """
        segments = from_annotation.get_timeline()
        n_segments = len(segments)
        while True:
            index = random.randrange(n_segments)
            segment = segments[index]
            track = random.choice(list(from_annotation.get_tracks(segment)))
            if self.yield_label:
                label = from_annotation[segment, track]
                yield segment, track, label
            else:
                yield segment, track

class RandomTrackTriplets(object):
    """(anchor, positive, negative) track triplets generator

    Parameters
    ----------
    per_label: int, optional
        Number of consecutive triplets yielded with the same anchor label
        before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, track, label) tuples.
        Defaults to yielding triplets of (segment, track) tuples.
        Useful for logging which labels are more difficult to discriminate.
    """

    def __init__(self, per_label=40, yield_label=False):
        super(RandomTrackTriplets, self).__init__()
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        return [RandomTracks(yield_label=self.yield_label).signature()] * 3

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for triplet in self.iter_triplets(reference):
            yield triplet

    def iter_triplets(self, from_annotation):
        """Yield (anchor, positive, negative) triplets of tracks

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """
        for label in from_annotation.labels():

            p = RandomTracks(yield_label=self.yield_label)
            positives = p.iter_tracks(from_annotation.subset([label]))

            n = RandomTracks(yield_label=self.yield_label)
            negatives = n.iter_tracks(from_annotation.subset([label], invert=True))

            for _ in range(self.per_label):
                try:
                    anchor = next(positives)
                    positive = next(positives)
                    negative = next(negatives)
                except StopIteration as e:
                    break
                yield anchor, positive, negative

class RandomSegmentTriplets(object):
    """(anchor, positive, negative) segment triplets generator

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    per_label: int, optional
        Number of consecutive triplets yielded with the same anchor label
        before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, label) tuples.
        Default to yielding segment triplets.
        Useful for logging which labels are more difficult to discriminate.
    """

    def __init__(self, duration=0., per_label=40, yield_label=False):
        super(RandomSegmentTriplets, self).__init__()
        self.duration = duration
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        if self.yield_label:
            return 3 * [{'type': PYANNOTE_SEGMENT, 'duration': self.duration},
                        {'type': 'label'}]
        else:
            return 3 * [{'type': PYANNOTE_SEGMENT, 'duration': self.duration}]

    def pick(self, segment):
        """Pick a subsegment at random"""
        t = segment.start + random.random() * (segment.duration - self.duration)
        return Segment(t, t + self.duration)

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for triplet in self.iter_triplets(reference):
            yield triplet

    def iter_triplets(self, from_annotation):
        """Yield (anchor, positive, negative) segment triplets

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """

        t = RandomTrackTriplets(per_label=self.per_label,
                                yield_label=self.yield_label)

        annotation = Annotation(uri=from_annotation.uri,
                                modality=from_annotation.modality)
        for segment, track, label in from_annotation.itertracks(label=True):
            if segment.duration < self.duration:
                continue
            annotation[segment, track] = label

        if len(annotation.labels()) < 2:
            return

        triplets = t.iter_triplets(annotation)

        for triplet in triplets:

            a, p, n = [item[0] for item in triplet]

            if self.duration:
                a, p, n = [self.pick(s) for s in (a, p, n)]

            if self.yield_label:
                a_, p_, n_ = [item[2] for item in triplet]
                yield (a, a_), (p, p_), (n, n_)
            else:
                yield a, p, n

class RandomSegmentPairs(object):
    """((query, returned), relevance) generator

    where `query` and `returned` are segments and `relevance` is boolean
    indicates whether `returned` has the same label as `query`.

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    per_label: int, optional
        Number of consecutive relevant and irrelevant pairs yielded with the
        same query label before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, label) tuples.
        Default to yielding segment triplets.
        Useful for logging which labels are more difficult to discriminate.

    """
    def __init__(self, duration=0., per_label=40, yield_label=False):
        super(RandomSegmentPairs, self).__init__()
        self.duration = duration
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        t = RandomSegmentTriplets(duration=self.duration,
                                  per_label=self.per_label,
                                  yield_label=self.yield_label)
        signature = t.signature()
        return [(signature[0], signature[0]), {'type': 'boolean'}]

    def __call__(self, protocol_item):
        _, _, reference = protocol_item
        for pair in self.iter_pairs(reference):
            yield pair

    def iter_pairs(self, from_annotation):
        """Yield ((query, returned), relevance)

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """

        t = RandomSegmentTriplets(duration=self.duration,
                                  per_label=self.per_label,
                                  yield_label=self.yield_label)
        triplets = t.iter_triplets(from_annotation)

        for query, positive, negative in triplets:
            yield [(query, positive), True]
            yield [(query, negative), False]
<commit_msg>chore: rename __call__ to from_protocol_item<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

import random
import numpy as np

from pyannote.core import Segment
from pyannote.core import Timeline
from pyannote.core import Annotation
from pyannote.core import SlidingWindow
from pyannote.core import PYANNOTE_SEGMENT
from pyannote.core import PYANNOTE_TRACK
from pyannote.core import PYANNOTE_LABEL

def random_segment(segments, weighted=False):
    """Generate segment with probability proportional to its duration"""

    p = None
    if weighted:
        total = float(sum(s.duration for s in segments))
        p = [s.duration / total for s in segments]

    n_segments = len(segments)
    while True:
        i = np.random.choice(n_segments, p=p)
        yield segments[i]

def random_subsegment(segment, duration):
    """Pick a subsegment at random"""
    if segment.duration < duration:
        raise ValueError('segment is too short')
    while True:
        t = segment.start + random.random() * (segment.duration - duration)
        yield Segment(t, t + duration)

class SlidingSegments(object):
    """Fixed-duration running segment generator

    Parameters
    ----------
    duration: float, optional
        Segment duration. Defaults to 3.2 seconds.
    step: float, optional
        Step duration. Defaults to 0.8 seconds.
    """

    def __init__(self, duration=3.2, step=0.8):
        super(SlidingSegments, self).__init__()

        if not duration > 0:
            raise ValueError('Duration must be strictly positive.')
        self.duration = duration

        if not step > 0:
            raise ValueError('Step must be strictly positive.')
        self.step = step

    def signature(self):
        return {'type': PYANNOTE_SEGMENT, 'duration': self.duration}

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for segment in self.iter_segments(reference):
            yield segment

    def iter_segments(self, source):
        """
        Parameters
        ----------
        source : float, Segment, Timeline or Annotation
            If `float`, yield running segments within [0, source).
            If `Segment`, yield running segments within this segment.
            If `Timeline`, yield running segments within this timeline.
            If `Annotation`, yield running segments within its timeline.
        """

        if isinstance(source, Annotation):
            segments = source.get_timeline()

        elif isinstance(source, Timeline):
            segments = source

        elif isinstance(source, Segment):
            segments = [source]

        elif isinstance(source, (int, float)):
            if not self.duration > 0:
                raise ValueError('Duration must be strictly positive.')
            segments = [Segment(0, duration)]

        else:
            raise TypeError('source must be float, Segment, Timeline or Annotation')

        segments = [segment for segment in segments
                            if segment.duration > self.duration]
        if not segments:
            raise ValueError('Source must contain at least one segment longer than requested duration.')

        for segment in segments:
            window = SlidingWindow(duration=self.duration,
                                   step=self.step,
                                   start=segment.start,
                                   end=segment.end)
            for s in window:
                # this is needed because window may go beyond segment.end
                if s in segment:
                    yield s

class RandomSegments(object):
    """Random segment generator

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    weighted: boolean, optional
        When True, probability of generating a segment is proportional to its
        duration.
    """
    def __init__(self, duration=0., weighted=False):
        super(RandomSegments, self).__init__()
        self.duration = duration
        self.weighted = weighted

    def signature(self):
        return {'type': PYANNOTE_SEGMENT, 'duration': self.duration}

    def pick(self, segment):
        """Pick a subsegment at random"""
        t = segment.start + random.random() * (segment.duration - self.duration)
        return Segment(t, t + self.duration)

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for segment in self.iter_segments(reference):
            yield segment

    def iter_segments(self, source):
        """
        Parameters
        ----------
        source : float, Segment, Timeline or Annotation
            If `float`, yield random segments within [0, source).
            If `Segment`, yield random segments within this segment.
            If `Timeline`, yield random segments within this timeline.
            If `Annotation`, yield random segments within its timeline.
        """

        if isinstance(source, Annotation):
            segments = source.get_timeline()

        elif isinstance(source, Timeline):
            segments = source

        elif isinstance(source, Segment):
            segments = [source]

        elif isinstance(source, (int, float)):
            if not self.duration > 0:
                raise ValueError('Duration must be strictly positive.')
            segments = [Segment(0, duration)]

        else:
            raise TypeError('source must be float, Segment, Timeline or Annotation')

        segments = [segment for segment in segments
                            if segment.duration > self.duration]
        if not segments:
            raise ValueError('Source must contain at least one segment longer than requested duration.')

        for segment in random_segment(segments, weighted=self.weighted):
            if self.duration:
                if segment.duration < self.duration:
                    continue
                yield next(random_subsegment(segment, self.duration))
            else:
                yield segment

class RandomTracks(object):
    """(segment, track) tuple generator

    Parameters
    ----------
    yield_label: boolean, optional
        When True, yield (segment, track, label) tuples.
        Defaults to yielding (segment, track) tuples.
    """

    def __init__(self, yield_label=False):
        super(RandomTracks, self).__init__()
        self.yield_label = yield_label

    def signature(self):
        signature = [
            {'type': PYANNOTE_SEGMENT, 'duration': 0.0},
            {'type': PYANNOTE_TRACK}
        ]
        if self.yield_label:
            signature.append({'type': PYANNOTE_LABEL})
        return signature

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for track in self.iter_tracks(reference):
            yield track

    def iter_tracks(self, from_annotation):
        """Yield (segment, track) tuples

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which tracks are obtained.
        """
        segments = from_annotation.get_timeline()
        n_segments = len(segments)
        while True:
            index = random.randrange(n_segments)
            segment = segments[index]
            track = random.choice(list(from_annotation.get_tracks(segment)))
            if self.yield_label:
                label = from_annotation[segment, track]
                yield segment, track, label
            else:
                yield segment, track

class RandomTrackTriplets(object):
    """(anchor, positive, negative) track triplets generator

    Parameters
    ----------
    per_label: int, optional
        Number of consecutive triplets yielded with the same anchor label
        before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, track, label) tuples.
        Defaults to yielding triplets of (segment, track) tuples.
        Useful for logging which labels are more difficult to discriminate.
    """

    def __init__(self, per_label=40, yield_label=False):
        super(RandomTrackTriplets, self).__init__()
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        return [RandomTracks(yield_label=self.yield_label).signature()] * 3

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for triplet in self.iter_triplets(reference):
            yield triplet

    def iter_triplets(self, from_annotation):
        """Yield (anchor, positive, negative) triplets of tracks

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """
        for label in from_annotation.labels():

            p = RandomTracks(yield_label=self.yield_label)
            positives = p.iter_tracks(from_annotation.subset([label]))

            n = RandomTracks(yield_label=self.yield_label)
            negatives = n.iter_tracks(from_annotation.subset([label], invert=True))

            for _ in range(self.per_label):
                try:
                    anchor = next(positives)
                    positive = next(positives)
                    negative = next(negatives)
                except StopIteration as e:
                    break
                yield anchor, positive, negative

class RandomSegmentTriplets(object):
    """(anchor, positive, negative) segment triplets generator

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    per_label: int, optional
        Number of consecutive triplets yielded with the same anchor label
        before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, label) tuples.
        Default to yielding segment triplets.
        Useful for logging which labels are more difficult to discriminate.
    """

    def __init__(self, duration=0., per_label=40, yield_label=False):
        super(RandomSegmentTriplets, self).__init__()
        self.duration = duration
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        if self.yield_label:
            return 3 * [{'type': PYANNOTE_SEGMENT, 'duration': self.duration},
                        {'type': 'label'}]
        else:
            return 3 * [{'type': PYANNOTE_SEGMENT, 'duration': self.duration}]

    def pick(self, segment):
        """Pick a subsegment at random"""
        t = segment.start + random.random() * (segment.duration - self.duration)
        return Segment(t, t + self.duration)

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for triplet in self.iter_triplets(reference):
            yield triplet

    def iter_triplets(self, from_annotation):
        """Yield (anchor, positive, negative) segment triplets

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """

        t = RandomTrackTriplets(per_label=self.per_label,
                                yield_label=self.yield_label)

        annotation = Annotation(uri=from_annotation.uri,
                                modality=from_annotation.modality)
        for segment, track, label in from_annotation.itertracks(label=True):
            if segment.duration < self.duration:
                continue
            annotation[segment, track] = label

        if len(annotation.labels()) < 2:
            return

        triplets = t.iter_triplets(annotation)

        for triplet in triplets:

            a, p, n = [item[0] for item in triplet]

            if self.duration:
                a, p, n = [self.pick(s) for s in (a, p, n)]

            if self.yield_label:
                a_, p_, n_ = [item[2] for item in triplet]
                yield (a, a_), (p, p_), (n, n_)
            else:
                yield a, p, n

class RandomSegmentPairs(object):
    """((query, returned), relevance) generator

    where `query` and `returned` are segments and `relevance` is boolean
    indicates whether `returned` has the same label as `query`.

    Parameters
    ----------
    duration: float, optional
        When provided, yield (random) subsegments with this `duration`.
        Defaults to yielding full segments.
    per_label: int, optional
        Number of consecutive relevant and irrelevant pairs yielded with the
        same query label before switching to another label.
    yield_label: boolean, optional
        When True, yield triplets of (segment, label) tuples.
        Default to yielding segment triplets.
        Useful for logging which labels are more difficult to discriminate.

    """
    def __init__(self, duration=0., per_label=40, yield_label=False):
        super(RandomSegmentPairs, self).__init__()
        self.duration = duration
        self.per_label = per_label
        self.yield_label = yield_label

    def signature(self):
        t = RandomSegmentTriplets(duration=self.duration,
                                  per_label=self.per_label,
                                  yield_label=self.yield_label)
        signature = t.signature()
        return [(signature[0], signature[0]), {'type': 'boolean'}]

    def from_protocol_item(self, protocol_item):
        _, _, reference = protocol_item
        for pair in self.iter_pairs(reference):
            yield pair

    def iter_pairs(self, from_annotation):
        """Yield ((query, returned), relevance)

        Parameters
        ----------
        from_annotation : Annotation
            Annotation from which triplets are obtained.
        """

        t = RandomSegmentTriplets(duration=self.duration,
                                  per_label=self.per_label,
                                  yield_label=self.yield_label)
        triplets = t.iter_triplets(from_annotation)

        for query, positive, negative in triplets:
            yield [(query, positive), True]
            yield [(query, negative), False]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
import argparse
import os.path
import threading
import numpy as np
from pygiftgrab import (VideoSourceFactory,
                        ColourSpace, IObservableObserver,
                        VideoTargetFactory, Codec)

"""
This script provides a multi-threading reliability check.
The background is issue #16. It looks like in applications
where multiple Python threads are involved, occasionally
the acquisition of the Global Interpreter Lock leads to a
deadlocks, which crashes the whole application with a
non-specific segmentation fault.

In this script we run a number of multi-threaded GIFT-Grab
pipelines, which should serve as a validation that this
problem is fixed.
"""

buffer_red = None
lock = threading.Lock()

class BuffererRed(IObservableObserver):

    def __init__(self):
        super(BuffererRed, self).__init__()

    def update(self, frame):
        global buffer_red
        with lock:
            data = frame.data(True)
            if buffer_red is None:
                buffer_red = np.copy(data)
            else:
                buffer_red[:, :, :] = data[:, :, :]

class HistogrammerRed(threading.Thread):

    def __init__(self):
        super(HistogrammerRed, self).__init__()
        self.running = False

    def run(self):
        if self.running:
            return

        # TODO
        self.running = True
        while self.running:
            print('Running')
            # TODO
            sleep(0.100)

    def stop(self):
        self.running = False

class Dyer(IObservableObserver):

    def __init__(self, channel, value):
        super(Dyer, self).__init__()
        assert 0 <= channel < 3
        assert 0 <= value < 256
        self.channel = channel
        self.value = value

    def update(self, frame):
        data = frame.data(True)
        data[:, :, self.channel] = self.value

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True,
                        metavar='VIDEO_FILE',
                        help='Input video file (HEVC-encoded MP4)')
    args = parser.parse_args()
    in_file = args.input

    filename = os.path.basename(in_file)
    filename, ext = os.path.splitext(filename)
    assert filename
    assert ext == '.mp4'

    sfac = VideoSourceFactory.get_instance()
    reader = sfac.create_file_reader(in_file, ColourSpace.BGRA)

    tfac = VideoTargetFactory.get_instance()
    frame_rate = reader.get_frame_rate()

    red = Dyer(2, 127)
    green = Dyer(1, 191)
    hist = HistogrammerRed()
    hist.start()

    red_file = os.path.join('.', ''.join([filename, '-red', ext]))
    red_writer = tfac.create_file_writer(Codec.HEVC, red_file, frame_rate)
    yellow_file = os.path.join('.', ''.join([filename, '-yellow', ext]))
    yellow_writer = tfac.create_file_writer(Codec.HEVC, yellow_file, frame_rate)

    reader.attach(red)
    red.attach(red_writer)
    red.attach(green)
    green.attach(yellow_writer)

    sleep(20)  # operate pipeline for 20 sec
    hist.stop()

    reader.detach(red)
    red.detach(red_writer)
    red.detach(green)
    green.detach(yellow_writer)
<commit_msg>Issue #16: implemented run of HistogrammerRed<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from time import sleep
import argparse
import os.path
import threading
import numpy as np
from pygiftgrab import (VideoSourceFactory,
                        ColourSpace, IObservableObserver,
                        VideoTargetFactory, Codec)

"""
This script provides a multi-threading reliability check.
The background is issue #16. It looks like in applications
where multiple Python threads are involved, occasionally
the acquisition of the Global Interpreter Lock leads to a
deadlocks, which crashes the whole application with a
non-specific segmentation fault.

In this script we run a number of multi-threaded GIFT-Grab
pipelines, which should serve as a validation that this
problem is fixed.
"""

buffer_red = None
lock = threading.Lock()

class BuffererRed(IObservableObserver):

    def __init__(self):
        super(BuffererRed, self).__init__()

    def update(self, frame):
        global buffer_red
        with lock:
            data = frame.data(True)
            if buffer_red is None:
                buffer_red = np.copy(data)
            else:
                buffer_red[:, :, :] = data[:, :, :]

class HistogrammerRed(threading.Thread):

    def __init__(self):
        super(HistogrammerRed, self).__init__()
        self.running = False

    def run(self):
        if self.running:
            return

        global buffer_red
        histogram = None
        self.running = True
        while self.running:
            with lock:
                if buffer_red is not None:
                    histogram = np.histogram(buffer_red[:, :, 2],
                                             bins=8, range=(0, 256))
            print(histogram)
            sleep(0.100)

    def stop(self):
        self.running = False

class Dyer(IObservableObserver):

    def __init__(self, channel, value):
        super(Dyer, self).__init__()
        assert 0 <= channel < 3
        assert 0 <= value < 256
        self.channel = channel
        self.value = value

    def update(self, frame):
        data = frame.data(True)
        data[:, :, self.channel] = self.value

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True,
                        metavar='VIDEO_FILE',
                        help='Input video file (HEVC-encoded MP4)')
    args = parser.parse_args()
    in_file = args.input

    filename = os.path.basename(in_file)
    filename, ext = os.path.splitext(filename)
    assert filename
    assert ext == '.mp4'

    sfac = VideoSourceFactory.get_instance()
    reader = sfac.create_file_reader(in_file, ColourSpace.BGRA)

    tfac = VideoTargetFactory.get_instance()
    frame_rate = reader.get_frame_rate()

    red = Dyer(2, 127)
    green = Dyer(1, 191)
    hist = HistogrammerRed()
    hist.start()

    red_file = os.path.join('.', ''.join([filename, '-red', ext]))
    red_writer = tfac.create_file_writer(Codec.HEVC, red_file, frame_rate)
    yellow_file = os.path.join('.', ''.join([filename, '-yellow', ext]))
    yellow_writer = tfac.create_file_writer(Codec.HEVC, yellow_file, frame_rate)

    reader.attach(red)
    red.attach(red_writer)
    red.attach(green)
    green.attach(yellow_writer)

    sleep(20)  # operate pipeline for 20 sec
    hist.stop()

    reader.detach(red)
    red.detach(red_writer)
    red.detach(green)
    green.detach(yellow_writer)
<|endoftext|>


<commit_before>import re

from django.contrib.auth import logout as djlogout
from django.contrib.auth.decorators import login_required
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext

from ublogging import api
from ublogging import uapi
from ublogging.models import Post, User, Profile
from ublogging.models import RegisterUserForm
from ublogging.profile import profile, renewapikey
from ublogging.register import join, validate
from ublogging.uapi import paginate_list
import flash
import ublogging

def sweet(request, sweetid):
    sweetid=int(sweetid)
    try:
        sweet = Post.objects.get(id=sweetid)
        user = sweet.user
        profile = Profile.objects.get(user=user)
    except:
        sweet = None
    return render_to_response('status/index.html', {
            'latest_post_list': None,
            'sweet': sweet,
            'viewing_user': user,
            'viewing_profile': profile,
        }, context_instance=RequestContext(request))

def public_timeline(request):
    latest_post_list = uapi.public_timeline(request)

    return render_to_response('status/index.html', {
            'latest_post_list': latest_post_list,
            'feedurl': reverse('django.contrib.syndication.views.feed', kwargs={'url': 'public'}),
            'viewing': "public",
        }, context_instance=RequestContext(request))

def index(request):
    try:
        latest_post_list = uapi.own_timeline(request)
    except:
        return public_timeline(request)

    return show_statuses(request, latest_post_list)

def user(request, user_name):
    latest_post_list = uapi.user_timeline(user_name, request)
    u = User.objects.get(username=user_name)
    return show_statuses(request, latest_post_list, user=u)

def show_statuses(request, latest_post_list, user=None):
    if user:
        profile = Profile.objects.get(user=user)
        feedurl = reverse('django.contrib.syndication.views.feed', kwargs={'url': 'user/%s' % user.username})
    else:
        feedurl = reverse('django.contrib.syndication.views.feed', kwargs={'url': 'user/%s' % request.user.username})
        profile = None

    return render_to_response('status/index.html', {
            'latest_post_list': latest_post_list,
            'viewing_user': user,
            'viewing_profile': profile,
            'feedurl': feedurl,
        }, context_instance=RequestContext(request))

@login_required
def new(request):
    text = request.POST['text']
    uapi.new_post(request.user, text, request)
    return HttpResponseRedirect(request.META['HTTP_REFERER'])
    #return HttpResponseRedirect(reverse('ublogging.views.index'))

@login_required
def logout(request):
    djlogout(request)
    return HttpResponseRedirect(reverse('ublogging.views.index'))

def refresh_index(request, lastid, pagenumber):
    url = request.META['HTTP_REFERER']
    usere = r'(.*)/user/(?P<username>[^\?\/]*)'
    publice = r'(.*)/public_timeline'
    lastid=int(lastid)

    user_timeline = re.match(usere, url)
    public_timeline = re.match(publice, url)
    if user_timeline:
        latest_post_list = uapi.user_timeline(user_timeline.group('username'), request, paginated=False)
    elif public_timeline:
        latest_post_list = uapi.public_timeline(request, paginated=False)
    else:
        try:
            latest_post_list = uapi.own_timeline(request, paginated=False)
        except:
            latest_post_list = uapi.public_timeline(request, paginated=False)

    lastpost = Post.objects.filter(pk=lastid)[0]
    latest_post_list = latest_post_list.filter(pub_date__gt=lastpost.pub_date)

    if latest_post_list:
        latest_post_list = paginate_list(request, latest_post_list, int(pagenumber))
        return render_to_response('status/refresh.html',
                                {'latest_post_list': latest_post_list },
                                context_instance=RequestContext(request))
    else:
        return HttpResponse("")
<commit_msg>Added 404 for sweets and user views<commit_after>import re

from django.contrib.auth import logout as djlogout
from django.contrib.auth.decorators import login_required
from django.core import serializers
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.http import HttpResponse
from django.http import HttpResponseRedirect
from django.shortcuts import get_object_or_404
from django.shortcuts import render_to_response
from django.template import RequestContext

from ublogging import api
from ublogging import uapi
from ublogging.models import Post, User, Profile
from ublogging.models import RegisterUserForm
from ublogging.profile import profile, renewapikey
from ublogging.register import join, validate
from ublogging.uapi import paginate_list
import flash
import ublogging

def sweet(request, sweetid):
    sweetid=int(sweetid)
    sweet = get_object_or_404(Post, id=sweetid)
    user = sweet.user
    profile = Profile.objects.get(user=user)
    return render_to_response('status/index.html', {
            'latest_post_list': None,
            'sweet': sweet,
            'viewing_user': user,
            'viewing_profile': profile,
        }, context_instance=RequestContext(request))

def public_timeline(request):
    latest_post_list = uapi.public_timeline(request)

    return render_to_response('status/index.html', {
            'latest_post_list': latest_post_list,
            'feedurl': reverse('django.contrib.syndication.views.feed', kwargs={'url': 'public'}),
            'viewing': "public",
        }, context_instance=RequestContext(request))

def index(request):
    try:
        latest_post_list = uapi.own_timeline(request)
    except:
        return public_timeline(request)

    return show_statuses(request, latest_post_list)

def user(request, user_name):
    u = get_object_or_404(User, username=user_name)
    latest_post_list = uapi.user_timeline(user_name, request)
    return show_statuses(request, latest_post_list, user=u)

def show_statuses(request, latest_post_list, user=None):
    if user:
        profile = Profile.objects.get(user=user)
        feedurl = reverse('django.contrib.syndication.views.feed', kwargs={'url': 'user/%s' % user.username})
    else:
        feedurl = reverse('django.contrib.syndication.views.feed', kwargs={'url': 'user/%s' % request.user.username})
        profile = None

    return render_to_response('status/index.html', {
            'latest_post_list': latest_post_list,
            'viewing_user': user,
            'viewing_profile': profile,
            'feedurl': feedurl,
        }, context_instance=RequestContext(request))

@login_required
def new(request):
    text = request.POST['text']
    uapi.new_post(request.user, text, request)
    return HttpResponseRedirect(request.META['HTTP_REFERER'])
    #return HttpResponseRedirect(reverse('ublogging.views.index'))

@login_required
def logout(request):
    djlogout(request)
    return HttpResponseRedirect(reverse('ublogging.views.index'))

def refresh_index(request, lastid, pagenumber):
    url = request.META['HTTP_REFERER']
    usere = r'(.*)/user/(?P<username>[^\?\/]*)'
    publice = r'(.*)/public_timeline'
    lastid=int(lastid)

    user_timeline = re.match(usere, url)
    public_timeline = re.match(publice, url)
    if user_timeline:
        latest_post_list = uapi.user_timeline(user_timeline.group('username'), request, paginated=False)
    elif public_timeline:
        latest_post_list = uapi.public_timeline(request, paginated=False)
    else:
        try:
            latest_post_list = uapi.own_timeline(request, paginated=False)
        except:
            latest_post_list = uapi.public_timeline(request, paginated=False)

    lastpost = Post.objects.filter(pk=lastid)[0]
    latest_post_list = latest_post_list.filter(pub_date__gt=lastpost.pub_date)

    if latest_post_list:
        latest_post_list = paginate_list(request, latest_post_list, int(pagenumber))
        return render_to_response('status/refresh.html',
                                {'latest_post_list': latest_post_list },
                                context_instance=RequestContext(request))
    else:
        return HttpResponse("")
<|endoftext|>


<commit_before># Copyright 2006 Andrey Golovizin
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with rdiff-backup; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

"""BibTeX parser"""

import codecs, locale
from pyparsing import *
from pybtex.core import Entry, Person

file_extension = 'bib'

class BibData:
    def __init__(self):
        self.records = {}
        self.strings = {}

    def addRecord(self, s, loc, toks):
        for i in toks:
            entry = Entry(i[0])
            fields = {}
            for field in i[2]:
                value = field[1][0] % tuple([self.strings[arg] for arg in field[1][1]])
                if field[0] in Entry.valid_roles:
                    for name in value.split(' and '):
                        entry.add_person(Person(name), field[0])
                else:
                    entry.fields[field[0]] = value
            #fields['TYPE'] = i[0]
            self.records[i[1]] = entry

    def addString(self, s, loc, toks):
        for i in toks:
            s = i[1][0] % tuple([self.strings[arg] for arg in i[1][1]])
            self.strings[i[0]] = s

class Parser:
    def __init__(self):
        self.set_encoding(locale.getpreferredencoding())
        self.data = BibData()

        lparenth = Literal('(').suppress()
        rparenth = Literal(')').suppress()
        lbrace = Literal('{').suppress()
        rbrace = Literal('}').suppress()
        def bibtexGroup(s):
            return (lparenth + s + rparenth) | (lbrace + s + rbrace)

        equal = Literal('=').suppress()
        at = Literal('@').suppress()
        comma = Literal(',').suppress()
        quotedString = Combine( '"' + ZeroOrMore( CharsNotIn('\"\n\r') ) + '"' )
        bracedString = Forward()
        bracedString << '{' + ZeroOrMore(CharsNotIn('{}\n\r') | bracedString) + '}'
        bracedString.setParseAction(self.processBracedString)
        bibTeXString = quotedString | bracedString
        bibTeXString.setParseAction(self.decode)

        name = Word(alphanums + '!$&*+-./:;<>?[]^_`|').setParseAction(downcaseTokens)
        value = Group(delimitedList(bibTeXString | Word(alphanums).setParseAction(downcaseTokens) | Word(nums), delim='#'))

        #fields
        field = Group(name + equal + value)
        field.setParseAction(self.processField)
        fields = Dict(delimitedList(field))

        #String
        string_body = bibtexGroup(fields)#Word(alphanums).setParseAction(upcaseTokens) + equal + value))
        string = at + CaselessLiteral('STRING').suppress() + string_body
        string.setParseAction(self.data.addString)

        #Meta
        meta_body = bibtexGroup(fields)#Group(bibtexGroup(Word(alphanums).setParseAction(upcaseTokens) + equal + field_value))
        meta_body.setParseAction(self.processMeta)
        meta = at + CaselessLiteral('META') + meta_body

        #Record
        record_header = at + Word(alphas).setParseAction(upcaseTokens)
        record_name = Word(printables.replace(',', ''))
        record_body = bibtexGroup(record_name + comma + Group(fields))
        #group(record_name + comma + fields)
        #lbrace + record_name + comma + fields + rbrace
        record = Group(record_header + record_body)
        record.setParseAction(self.data.addRecord)

        #Comment
        comment_header = Word(alphas) | (at + CaselessLiteral('COMMENT'))
        comment = Suppress(comment_header + lbrace + ZeroOrMore(CharsNotIn('}')) + rbrace)

        #Raw text
        raw_text = CharsNotIn('@').suppress()

        self.BibTeX_BNF = ZeroOrMore(meta | comment | raw_text | Suppress(string) | Suppress(record)) + StringEnd()

    def set_encoding(self, s):
        self._decode = codecs.getdecoder(s)

    def decode(self, s, loc, toks):
        return map(lambda x:self._decode(x)[0], toks)

    def processBracedString(self, s, loc, toks):
        return "".join(toks)

    def processMeta(self, s, loc, toks):
        for i in toks:
            if i[0] == 'ENCODING':
                self.setEncoding(i[1][0])

    def processField(self, s, loc, toks):
        result = []
        for token in toks:
            strings = []
            args = []
            for s in token[1]:
                s = s.replace('%', '%%')
                if (s.startswith('"') and s.endswith('"')) or \
                   (s.startswith('{') and s.endswith('}')):
                    strings.append(s[1:-1])
                elif s.isdigit():
                    strings.append(str(s))
                else:
                    strings.append('%s')
                    args.append(s)
            result.append((token[0], ("".join(strings), args)))
        return result

    def parse_file(self, filename):
        """parse BibTeX file and return a tree"""
        f = open(filename)
        s = f.read()
        f.close()
        try:
            self.BibTeX_BNF.parseString(s)
            #print self.data.records
            return self.data.records
        except ParseException, e:
            print "%s: syntax error:" % filename
            print e
            sys.exit(1)
<commit_msg>added macros for month names<commit_after># Copyright 2006 Andrey Golovizin
#
# This file is part of pybtex.
#
# pybtex is free software; you can redistribute it and/or modify
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.
#
# pybtex is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with rdiff-backup; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
# USA

"""BibTeX parser"""

import codecs, locale
from pyparsing import *
from pybtex.core import Entry, Person

file_extension = 'bib'

class BibData:
    def __init__(self):
        self.records = {}
        self.strings = {
            'jan': 'January',
            'feb': 'February',
            'mar': 'March',
            'apr': 'April',
            'may': 'May',
            'jun': 'June',
            'jul': 'July',
            'aug': 'August',
            'sep': 'September',
            'oct': 'October',
            'nov': 'November',
            'dec': 'December'
        }

    def addRecord(self, s, loc, toks):
        for i in toks:
            entry = Entry(i[0])
            fields = {}
            for field in i[2]:
                value = field[1][0] % tuple([self.strings[arg] for arg in field[1][1]])
                if field[0] in Entry.valid_roles:
                    for name in value.split(' and '):
                        entry.add_person(Person(name), field[0])
                else:
                    entry.fields[field[0]] = value
            #fields['TYPE'] = i[0]
            self.records[i[1]] = entry

    def addString(self, s, loc, toks):
        for i in toks:
            s = i[1][0] % tuple([self.strings[arg] for arg in i[1][1]])
            self.strings[i[0]] = s

class Parser:
    def __init__(self):
        self.set_encoding(locale.getpreferredencoding())
        self.data = BibData()

        lparenth = Literal('(').suppress()
        rparenth = Literal(')').suppress()
        lbrace = Literal('{').suppress()
        rbrace = Literal('}').suppress()
        def bibtexGroup(s):
            return (lparenth + s + rparenth) | (lbrace + s + rbrace)

        equal = Literal('=').suppress()
        at = Literal('@').suppress()
        comma = Literal(',').suppress()
        quotedString = Combine( '"' + ZeroOrMore( CharsNotIn('\"\n\r') ) + '"' )
        bracedString = Forward()
        bracedString << '{' + ZeroOrMore(CharsNotIn('{}\n\r') | bracedString) + '}'
        bracedString.setParseAction(self.processBracedString)
        bibTeXString = quotedString | bracedString
        bibTeXString.setParseAction(self.decode)

        name = Word(alphanums + '!$&*+-./:;<>?[]^_`|').setParseAction(downcaseTokens)
        value = Group(delimitedList(bibTeXString | Word(alphanums).setParseAction(downcaseTokens) | Word(nums), delim='#'))

        #fields
        field = Group(name + equal + value)
        field.setParseAction(self.processField)
        fields = Dict(delimitedList(field))

        #String
        string_body = bibtexGroup(fields)#Word(alphanums).setParseAction(upcaseTokens) + equal + value))
        string = at + CaselessLiteral('STRING').suppress() + string_body
        string.setParseAction(self.data.addString)

        #Meta
        meta_body = bibtexGroup(fields)#Group(bibtexGroup(Word(alphanums).setParseAction(upcaseTokens) + equal + field_value))
        meta_body.setParseAction(self.processMeta)
        meta = at + CaselessLiteral('META') + meta_body

        #Record
        record_header = at + Word(alphas).setParseAction(upcaseTokens)
        record_name = Word(printables.replace(',', ''))
        record_body = bibtexGroup(record_name + comma + Group(fields))
        #group(record_name + comma + fields)
        #lbrace + record_name + comma + fields + rbrace
        record = Group(record_header + record_body)
        record.setParseAction(self.data.addRecord)

        #Comment
        comment_header = Word(alphas) | (at + CaselessLiteral('COMMENT'))
        comment = Suppress(comment_header + lbrace + ZeroOrMore(CharsNotIn('}')) + rbrace)

        #Raw text
        raw_text = CharsNotIn('@').suppress()

        self.BibTeX_BNF = ZeroOrMore(meta | comment | raw_text | Suppress(string) | Suppress(record)) + StringEnd()

    def set_encoding(self, s):
        self._decode = codecs.getdecoder(s)

    def decode(self, s, loc, toks):
        return map(lambda x:self._decode(x)[0], toks)

    def processBracedString(self, s, loc, toks):
        return "".join(toks)

    def processMeta(self, s, loc, toks):
        for i in toks:
            if i[0] == 'ENCODING':
                self.setEncoding(i[1][0])

    def processField(self, s, loc, toks):
        result = []
        for token in toks:
            strings = []
            args = []
            for s in token[1]:
                s = s.replace('%', '%%')
                if (s.startswith('"') and s.endswith('"')) or \
                   (s.startswith('{') and s.endswith('}')):
                    strings.append(s[1:-1])
                elif s.isdigit():
                    strings.append(str(s))
                else:
                    strings.append('%s')
                    args.append(s)
            result.append((token[0], ("".join(strings), args)))
        return result

    def parse_file(self, filename):
        """parse BibTeX file and return a tree"""
        f = open(filename)
        s = f.read()
        f.close()
        try:
            self.BibTeX_BNF.parseString(s)
            #print self.data.records
            return self.data.records
        except ParseException, e:
            print "%s: syntax error:" % filename
            print e
            sys.exit(1)
<|endoftext|>


<commit_before>"""A functions module, includes all the standard functions.

Combinatorial - factorial, fibonacci, harmonic, bernoulli...
Elementary - hyperbolic, trigonometric, exponential, floor and ceiling, sqrt...
Special - gamma, zeta,spherical harmonics...
"""
from sympy.core.basic import Basic

import combinatorial
import elementary
import special

from special.polynomials import (legendre, assoc_legendre, hermite, chebyshevt,
        chebyshevu, chebyshevu_root, chebyshevt_root, laguerre_l)

# see #391
from combinatorial.factorials import factorial, factorial2, rf, ff, binomial
from combinatorial.factorials import factorial, RisingFactorial, FallingFactorial
from combinatorial.factorials import binomial, factorial2
from combinatorial.numbers import fibonacci, lucas, harmonic, bernoulli, bell, euler, catalan

from elementary.miscellaneous import sqrt, root, Min, Max, Id, real_root
from elementary.complexes import (re, im, sign, Abs, conjugate, arg,
                      polar_lift, periodic_argument, unbranched_argument,
                      principal_branch)
from elementary.trigonometric import acot, cot, tan, cos, sin, asin, acos, atan, atan2
from elementary.exponential import exp_polar, exp, log, LambertW
from elementary.hyperbolic import sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth
from elementary.integers import floor, ceiling
from elementary.piecewise import Piecewise, piecewise_fold

from special.error_functions import erf, Ei, expint, E1, Si, Ci, Shi, Chi
from special.gamma_functions import gamma, lowergamma, uppergamma, polygamma, \
         loggamma, digamma, trigamma, beta
from special.zeta_functions import dirichlet_eta, zeta, lerchphi, polylog
from special.spherical_harmonics import Ylm, Zlm
from special.tensor_functions import Eijk, LeviCivita, KroneckerDelta
from special.delta_functions import DiracDelta, Heaviside
from special.bsplines import bspline_basis, bspline_basis_set
from special.bessel import besselj, bessely, besseli, besselk, hankel1, \
                           hankel2, jn, yn, jn_zeros
from special.hyper import hyper, meijerg

ln = log
<commit_msg>Export the erfs(z) function.<commit_after>"""A functions module, includes all the standard functions.

Combinatorial - factorial, fibonacci, harmonic, bernoulli...
Elementary - hyperbolic, trigonometric, exponential, floor and ceiling, sqrt...
Special - gamma, zeta,spherical harmonics...
"""
from sympy.core.basic import Basic

import combinatorial
import elementary
import special

from special.polynomials import (legendre, assoc_legendre, hermite, chebyshevt,
        chebyshevu, chebyshevu_root, chebyshevt_root, laguerre_l)

# see #391
from combinatorial.factorials import factorial, factorial2, rf, ff, binomial
from combinatorial.factorials import factorial, RisingFactorial, FallingFactorial
from combinatorial.factorials import binomial, factorial2
from combinatorial.numbers import fibonacci, lucas, harmonic, bernoulli, bell, euler, catalan

from elementary.miscellaneous import sqrt, root, Min, Max, Id, real_root
from elementary.complexes import (re, im, sign, Abs, conjugate, arg,
                      polar_lift, periodic_argument, unbranched_argument,
                      principal_branch)
from elementary.trigonometric import acot, cot, tan, cos, sin, asin, acos, atan, atan2
from elementary.exponential import exp_polar, exp, log, LambertW
from elementary.hyperbolic import sinh, cosh, tanh, coth, asinh, acosh, atanh, acoth
from elementary.integers import floor, ceiling
from elementary.piecewise import Piecewise, piecewise_fold

from special.error_functions import erf, Ei, expint, E1, Si, Ci, Shi, Chi, erfs
from special.gamma_functions import gamma, lowergamma, uppergamma, polygamma, \
         loggamma, digamma, trigamma, beta
from special.zeta_functions import dirichlet_eta, zeta, lerchphi, polylog
from special.spherical_harmonics import Ylm, Zlm
from special.tensor_functions import Eijk, LeviCivita, KroneckerDelta
from special.delta_functions import DiracDelta, Heaviside
from special.bsplines import bspline_basis, bspline_basis_set
from special.bessel import besselj, bessely, besseli, besselk, hankel1, \
                           hankel2, jn, yn, jn_zeros
from special.hyper import hyper, meijerg

ln = log
<|endoftext|>