<commit_before>from redbot.core import commands, checks, data_manager, Config
import tempfile
import gtts
import discord
import asyncio
import os
import random
import lavalink
import pydub
import aiohttp

class SFX(commands.Cog):
    """Plays uploaded sounds or text-to-speech using gTTS."""

    def __init__(self):
        self.tts_languages = list(gtts.lang.tts_langs().keys())
        self.last_track_info = None
        self.current_sfx = None
        self.config = Config.get_conf(self, identifier=134621854878007296)
        self.sound_base = (data_manager.cog_data_path(self) / 'sounds').as_posix()
        self.session = aiohttp.ClientSession()
        default_config = {
            'padding': 700,
            'tts_lang': 'en',
            'sounds': {}
        }
        self.config.register_guild(**default_config)
        lavalink.register_event_listener(self.ll_check)
        if not os.path.exists(self.sound_base):
            os.makedirs(self.sound_base)

    def __unload(self):
        lavalink.unregister_event_listener(self.ll_check)

    @commands.command(usage='[language code] <text>')
    @commands.cooldown(rate=1, per=1, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def tts(self, ctx, *, text):
        """
        Plays the given text as TTS in your current voice channel.

        Turns a string of text into audio using the server's default language, if none is specified.
        Use `[p]ttslangs` for a list of valid language codes.
        """

        if not ctx.author.voice or not ctx.author.voice.channel:
            await ctx.send('You are not connected to a voice channel.')
            return

        cfg_tts_lang = await self.config.guild(ctx.guild).tts_lang()
        cfg_padding = await self.config.guild(ctx.guild).padding()
        lang = cfg_tts_lang
        try:
            lang, text = text.split(' ', maxsplit=1)
            if lang not in self.tts_languages:
                text = f'{lang} {text}'
                lang = cfg_tts_lang
        except ValueError:
            pass

        tts_audio = gtts.gTTS(text, lang=lang)
        audio_file = os.path.join(tempfile.gettempdir(), ''.join(random.choice('0123456789ABCDEF') for i in range(12)) + '.mp3')
        tts_audio.save(audio_file)
        audio_data = pydub.AudioSegment.from_mp3(audio_file)
        silence = pydub.AudioSegment.silent(duration=cfg_padding)
        padded_audio = silence + audio_data + silence
        padded_audio.export(audio_file, format='mp3')
        await self._play_sfx(ctx.author.voice.channel, audio_file, True)

    @commands.group()
    async def sfxconfig(self, ctx):
        """Configures the SFX cog."""
        pass

    @sfxconfig.command(usage='[language code]')
    @checks.mod()
    async def tts_lang(self, ctx, lang: str = None):
        """
        Configures the default TTS language.

        Gets/sets the default language for the `[p]tts` command.
        Use `[p]ttslangs` for a list of language codes.
        """

        cfg_tts_lang = await self.config.guild(ctx.guild).tts_lang()
        if lang is None:
            await ctx.send(f"Current value of `tts_lang`: {cfg_tts_lang}")
            return

        if lang not in self.tts_languages:
            await ctx.send('Invalid langauge. Use [p]ttsconfig langlist for a list of languages.')
            return

        cfg_tts_lang = lang
        await self.config.guild(ctx.guild).tts_lang.set(cfg_tts_lang)
        await ctx.send(f'`tts_lang` set to {lang}.')

    @sfxconfig.command(usage='<duration>')
    @checks.mod()
    async def padding(self, ctx, padding: int = None):
        """
        Configures the default padding.

        Gets/sets the default duration of padding (in ms) for the `[p]tts` and `[p]addsfx` commands.
        Adjust if the sound gets cut off at the beginning or the end.

        Warning: Sounds do not get affected immediately. You have to re-add them for this to have an effect on them.
        TTS gets affected immediately.
        """

        cfg_padding = await self.config.guild(ctx.guild).padding()
        if padding is None:
            await ctx.send(f"Current value of `padding`: {cfg_padding}")
            return

        cfg_padding = padding
        await self.config.guild(ctx.guild).padding.set(cfg_padding)
        await ctx.send(f'`padding` set to {padding}.')

    @commands.command()
    @commands.cooldown(rate=1, per=2, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def ttslangs(self, ctx):
        """
        Lists available TTS Languages.

        Prints the list of valid languages for use with `[p]tts`.
        """

        await ctx.send(f"List of valid languages: {', '.join(self.tts_languages)}")

    @commands.command()
    @commands.cooldown(rate=1, per=1, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def sfx(self, ctx, soundname: str):
        """
        Plays an existing sound in your current voice channel.
        """

        if not ctx.author.voice or not ctx.author.voice.channel:
            await ctx.send('You are not connected to a voice channel.')
            return

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()
        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])
        if not os.path.exists(filepath):
            del cfg_sounds[soundname]
            await self.config.guild(ctx.guild).sounds.set(cfg_sounds)
            await ctx.send('Looks like this sound\'s file has gone missing! I\'ve removed it from the list of sounds.')
            return

        await self._play_sfx(ctx.author.voice.channel, filepath)

    @commands.command()
    @checks.mod()
    async def addsfx(self, ctx, name: str, link: str=None):
        """Adds a new sound.

        Either upload the file as a Discord attachment and make your comment
        `[p]addsfx <name>`, or use `[p]addsfx <name> <direct-URL-to-file>`.
        """
        cfg_sounds = await self.config.guild(ctx.guild).sounds()
        cfg_padding = await self.config.guild(ctx.guild).padding()

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        attach = ctx.message.attachments
        if len(attach) > 1 or (attach and link):
            await ctx.send('Please only add one sound at a time.')
            return

        url = ''
        filename = ''
        if attach:
            a = attach[0]
            url = a.url
            filename = a.filename
        elif link:
            url = ''.join(link)
            filename = os.path.basename(
                '_'.join(url.split()).replace('%20', '_'))
        else:
            await ctx.send('You must provide either a Discord attachment or a direct link to a sound.')
            return

        _, file_extension = os.path.splitext(filename)
        if file_extension != '.wav' and file_extension != '.mp3':
            await ctx.send('Only .wav and .mp3 sounds are currently supported.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), filename)

        if name in cfg_sounds.keys():
            await ctx.send('A sound with that name already exists. Please choose another name and try again.')
            return

        if os.path.exists(filepath):
            await ctx.send('A sound with that filename already exists. Please change the filename and try again.')
            return

        async with self.session.get(url) as new_sound:
            f = open(filepath, 'wb')
            f.write(await new_sound.read())
            f.close()

        audio_data = pydub.AudioSegment.from_file(filepath, format=file_extension[1:])
        silence = pydub.AudioSegment.silent(duration=cfg_padding)
        padded_audio = silence + audio_data + silence
        padded_audio.export(filepath, format=file_extension[1:])

        cfg_sounds[name] = filename
        await self.config.guild(ctx.guild).sounds.set(cfg_sounds)

        await ctx.send(f'Sound {name} added.')

    @commands.command()
    @checks.mod()
    async def delsfx(self, ctx, soundname: str):
        """Deletes an existing sound."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])

        if os.path.exists(filepath):
            os.remove(filepath)

        del cfg_sounds[soundname]
        await self.config.guild(ctx.guild).sounds.set(cfg_sounds)

        await ctx.send(f'Sound {soundname} deleted.')

    @commands.command()
    @commands.cooldown(rate=1, per=3, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def allsfx(self, ctx):
        """Prints all available sounds for this server."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if len(cfg_sounds.items()) == 0:
            await ctx.send(f'No sounds found. Use `{ctx.prefix}addsfx` to add one.')
            return

        paginator = discord.ext.commands.help.Paginator()
        for soundname, filepath in cfg_sounds.items():
            paginator.add_line(soundname)

        await ctx.send('Sounds for this server:')
        for page in paginator.pages:
            await ctx.send(page)

    @commands.command(no_pm=True, pass_context=True, aliases=['getsound'])
    @commands.cooldown(rate=1, per=2, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def getsfx(self, ctx, soundname: str):
        """Uploads the given sound."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])
        if not os.path.exists(filepath):
            del cfg_sounds[soundname]
            await self.config.guild(ctx.guild).sounds.set(cfg_sounds)
            await ctx.send('Looks like this sound\'s file has gone missing! I\'ve removed it from the list of sounds.')
            return

        await ctx.send(file=discord.File(filepath))

    async def _play_sfx(self, vc, filepath, is_tts=False):
        player = await lavalink.connect(vc)
        track = (await player.get_tracks(query=filepath))[0]

        if player.current is None:
            player.queue.append(track)
            self.current_sfx = (track, is_tts)
            await player.play()
            return

        if self.current_sfx is not None:
            player.queue.insert(0, track)
            await player.skip()
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = (track, is_tts)
            return

        self.last_track_info = (player.current, player.position)
        self.current_sfx = (track, is_tts)
        player.queue.insert(0, track)
        player.queue.insert(1, player.current)
        await player.skip()

    async def ll_check(self, player, event, reason):
        if self.current_sfx is None and self.last_track_info is None:
            return

        if event == lavalink.LavalinkEvents.TRACK_EXCEPTION and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            return

        if event == lavalink.LavalinkEvents.TRACK_STUCK and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            await player.skip()
            return

        if event == lavalink.LavalinkEvents.TRACK_END and player.current is None and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            return

        if event == lavalink.LavalinkEvents.TRACK_END and player.current.track_identifier == self.last_track_info[0].track_identifier:
            print(str(self.last_track_info[0].uri))
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            await player.pause()
            await player.seek(self.last_track_info[1])
            await player.pause(False)
            self.last_track_info = None<commit_msg>[SFX] Fixed exceptions when playing a sound when nothing else is currently playing<commit_after>from redbot.core import commands, checks, data_manager, Config
import tempfile
import gtts
import discord
import asyncio
import os
import random
import lavalink
import pydub
import aiohttp

class SFX(commands.Cog):
    """Plays uploaded sounds or text-to-speech using gTTS."""

    def __init__(self):
        self.tts_languages = list(gtts.lang.tts_langs().keys())
        self.last_track_info = None
        self.current_sfx = None
        self.config = Config.get_conf(self, identifier=134621854878007296)
        self.sound_base = (data_manager.cog_data_path(self) / 'sounds').as_posix()
        self.session = aiohttp.ClientSession()
        default_config = {
            'padding': 700,
            'tts_lang': 'en',
            'sounds': {}
        }
        self.config.register_guild(**default_config)
        lavalink.register_event_listener(self.ll_check)
        if not os.path.exists(self.sound_base):
            os.makedirs(self.sound_base)

    def __unload(self):
        lavalink.unregister_event_listener(self.ll_check)

    @commands.command(usage='[language code] <text>')
    @commands.cooldown(rate=1, per=1, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def tts(self, ctx, *, text):
        """
        Plays the given text as TTS in your current voice channel.

        Turns a string of text into audio using the server's default language, if none is specified.
        Use `[p]ttslangs` for a list of valid language codes.
        """

        if not ctx.author.voice or not ctx.author.voice.channel:
            await ctx.send('You are not connected to a voice channel.')
            return

        cfg_tts_lang = await self.config.guild(ctx.guild).tts_lang()
        cfg_padding = await self.config.guild(ctx.guild).padding()
        lang = cfg_tts_lang
        try:
            lang, text = text.split(' ', maxsplit=1)
            if lang not in self.tts_languages:
                text = f'{lang} {text}'
                lang = cfg_tts_lang
        except ValueError:
            pass

        tts_audio = gtts.gTTS(text, lang=lang)
        audio_file = os.path.join(tempfile.gettempdir(), ''.join(random.choice('0123456789ABCDEF') for i in range(12)) + '.mp3')
        tts_audio.save(audio_file)
        audio_data = pydub.AudioSegment.from_mp3(audio_file)
        silence = pydub.AudioSegment.silent(duration=cfg_padding)
        padded_audio = silence + audio_data + silence
        padded_audio.export(audio_file, format='mp3')
        await self._play_sfx(ctx.author.voice.channel, audio_file, True)

    @commands.group()
    async def sfxconfig(self, ctx):
        """Configures the SFX cog."""
        pass

    @sfxconfig.command(usage='[language code]')
    @checks.mod()
    async def tts_lang(self, ctx, lang: str = None):
        """
        Configures the default TTS language.

        Gets/sets the default language for the `[p]tts` command.
        Use `[p]ttslangs` for a list of language codes.
        """

        cfg_tts_lang = await self.config.guild(ctx.guild).tts_lang()
        if lang is None:
            await ctx.send(f"Current value of `tts_lang`: {cfg_tts_lang}")
            return

        if lang not in self.tts_languages:
            await ctx.send('Invalid langauge. Use [p]ttsconfig langlist for a list of languages.')
            return

        cfg_tts_lang = lang
        await self.config.guild(ctx.guild).tts_lang.set(cfg_tts_lang)
        await ctx.send(f'`tts_lang` set to {lang}.')

    @sfxconfig.command(usage='<duration>')
    @checks.mod()
    async def padding(self, ctx, padding: int = None):
        """
        Configures the default padding.

        Gets/sets the default duration of padding (in ms) for the `[p]tts` and `[p]addsfx` commands.
        Adjust if the sound gets cut off at the beginning or the end.

        Warning: Sounds do not get affected immediately. You have to re-add them for this to have an effect on them.
        TTS gets affected immediately.
        """

        cfg_padding = await self.config.guild(ctx.guild).padding()
        if padding is None:
            await ctx.send(f"Current value of `padding`: {cfg_padding}")
            return

        cfg_padding = padding
        await self.config.guild(ctx.guild).padding.set(cfg_padding)
        await ctx.send(f'`padding` set to {padding}.')

    @commands.command()
    @commands.cooldown(rate=1, per=2, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def ttslangs(self, ctx):
        """
        Lists available TTS Languages.

        Prints the list of valid languages for use with `[p]tts`.
        """

        await ctx.send(f"List of valid languages: {', '.join(self.tts_languages)}")

    @commands.command()
    @commands.cooldown(rate=1, per=1, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def sfx(self, ctx, soundname: str):
        """
        Plays an existing sound in your current voice channel.
        """

        if not ctx.author.voice or not ctx.author.voice.channel:
            await ctx.send('You are not connected to a voice channel.')
            return

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()
        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])
        if not os.path.exists(filepath):
            del cfg_sounds[soundname]
            await self.config.guild(ctx.guild).sounds.set(cfg_sounds)
            await ctx.send('Looks like this sound\'s file has gone missing! I\'ve removed it from the list of sounds.')
            return

        await self._play_sfx(ctx.author.voice.channel, filepath)

    @commands.command()
    @checks.mod()
    async def addsfx(self, ctx, name: str, link: str=None):
        """Adds a new sound.

        Either upload the file as a Discord attachment and make your comment
        `[p]addsfx <name>`, or use `[p]addsfx <name> <direct-URL-to-file>`.
        """
        cfg_sounds = await self.config.guild(ctx.guild).sounds()
        cfg_padding = await self.config.guild(ctx.guild).padding()

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        attach = ctx.message.attachments
        if len(attach) > 1 or (attach and link):
            await ctx.send('Please only add one sound at a time.')
            return

        url = ''
        filename = ''
        if attach:
            a = attach[0]
            url = a.url
            filename = a.filename
        elif link:
            url = ''.join(link)
            filename = os.path.basename(
                '_'.join(url.split()).replace('%20', '_'))
        else:
            await ctx.send('You must provide either a Discord attachment or a direct link to a sound.')
            return

        _, file_extension = os.path.splitext(filename)
        if file_extension != '.wav' and file_extension != '.mp3':
            await ctx.send('Only .wav and .mp3 sounds are currently supported.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), filename)

        if name in cfg_sounds.keys():
            await ctx.send('A sound with that name already exists. Please choose another name and try again.')
            return

        if os.path.exists(filepath):
            await ctx.send('A sound with that filename already exists. Please change the filename and try again.')
            return

        async with self.session.get(url) as new_sound:
            f = open(filepath, 'wb')
            f.write(await new_sound.read())
            f.close()

        audio_data = pydub.AudioSegment.from_file(filepath, format=file_extension[1:])
        silence = pydub.AudioSegment.silent(duration=cfg_padding)
        padded_audio = silence + audio_data + silence
        padded_audio.export(filepath, format=file_extension[1:])

        cfg_sounds[name] = filename
        await self.config.guild(ctx.guild).sounds.set(cfg_sounds)

        await ctx.send(f'Sound {name} added.')

    @commands.command()
    @checks.mod()
    async def delsfx(self, ctx, soundname: str):
        """Deletes an existing sound."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])

        if os.path.exists(filepath):
            os.remove(filepath)

        del cfg_sounds[soundname]
        await self.config.guild(ctx.guild).sounds.set(cfg_sounds)

        await ctx.send(f'Sound {soundname} deleted.')

    @commands.command()
    @commands.cooldown(rate=1, per=3, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def allsfx(self, ctx):
        """Prints all available sounds for this server."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if len(cfg_sounds.items()) == 0:
            await ctx.send(f'No sounds found. Use `{ctx.prefix}addsfx` to add one.')
            return

        paginator = discord.ext.commands.help.Paginator()
        for soundname, filepath in cfg_sounds.items():
            paginator.add_line(soundname)

        await ctx.send('Sounds for this server:')
        for page in paginator.pages:
            await ctx.send(page)

    @commands.command(no_pm=True, pass_context=True, aliases=['getsound'])
    @commands.cooldown(rate=1, per=2, type=discord.ext.commands.cooldowns.BucketType.guild)
    async def getsfx(self, ctx, soundname: str):
        """Uploads the given sound."""

        if str(ctx.guild.id) not in os.listdir(self.sound_base):
            os.makedirs(os.path.join(self.sound_base, str(ctx.guild.id)))

        cfg_sounds = await self.config.guild(ctx.guild).sounds()

        if soundname not in cfg_sounds.keys():
            await ctx.send(f'Sound `{soundname}` does not exist. Try `{ctx.prefix}allsfx` for a list.')
            return

        filepath = os.path.join(self.sound_base, str(ctx.guild.id), cfg_sounds[soundname])
        if not os.path.exists(filepath):
            del cfg_sounds[soundname]
            await self.config.guild(ctx.guild).sounds.set(cfg_sounds)
            await ctx.send('Looks like this sound\'s file has gone missing! I\'ve removed it from the list of sounds.')
            return

        await ctx.send(file=discord.File(filepath))

    async def _play_sfx(self, vc, filepath, is_tts=False):
        player = await lavalink.connect(vc)
        track = (await player.get_tracks(query=filepath))[0]

        if player.current is None:
            player.queue.append(track)
            self.current_sfx = (track, is_tts)
            await player.play()
            return

        if self.current_sfx is not None:
            player.queue.insert(0, track)
            await player.skip()
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = (track, is_tts)
            return

        self.last_track_info = (player.current, player.position)
        self.current_sfx = (track, is_tts)
        player.queue.insert(0, track)
        player.queue.insert(1, player.current)
        await player.skip()

    async def ll_check(self, player, event, reason):
        if self.current_sfx is None and self.last_track_info is None:
            return

        if event == lavalink.LavalinkEvents.TRACK_EXCEPTION and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            return

        if event == lavalink.LavalinkEvents.TRACK_STUCK and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            await player.skip()
            return

        if event == lavalink.LavalinkEvents.TRACK_END and player.current is None and self.current_sfx is not None:
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            return

        if event == lavalink.LavalinkEvents.TRACK_END and self.last_track_info is not None and player.current.track_identifier == self.last_track_info[0].track_identifier:
            print(str(self.last_track_info[0].uri))
            if self.current_sfx[1]:
                os.remove(self.current_sfx[0].uri)
            self.current_sfx = None
            await player.pause()
            await player.seek(self.last_track_info[1])
            await player.pause(False)
            self.last_track_info = None
<|endoftext|>


<commit_before>import pytest
import numpy as np
from astropy import units as u
from astropy import convolution
from astropy.wcs import WCS
from astropy import wcs

from .test_spectral_cube import cube_and_raw

try:
    from radio_beam import beam,Beam
    RADIO_BEAM_INSTALLED = True
except ImportError:
    RADIO_BEAM_INSTALLED = False

try:
    import reproject
    REPROJECT_INSTALLED = True
except ImportError:
    REPROJECT_INSTALLED = False

@pytest.mark.skipif('not RADIO_BEAM_INSTALLED')
def test_convolution():
    cube, data = cube_and_raw('255_delta.fits')

    # 1" convolved with 1.5" -> 1.8027....
    target_beam = Beam(1.802775637731995*u.arcsec, 1.802775637731995*u.arcsec,
                       0*u.deg)

    conv_cube = cube.convolve_to(target_beam)

    expected = convolution.Gaussian2DKernel((1.5*u.arcsec /
                                             beam.SIGMA_TO_FWHM /
                                             (5.555555555555e-4*u.deg)).decompose().value,
                                            x_size=5, y_size=5,
                                           )

    np.testing.assert_almost_equal(expected.array,
                                   conv_cube.filled_data[0,:,:].value)

    # 2nd layer is all zeros
    assert np.all(conv_cube.filled_data[1,:,:] == 0.0)

@pytest.mark.skipif('not RADIO_BEAM_INSTALLED')
def test_beams_convolution():
    cube, data = cube_and_raw('455_delta_beams.fits')

    # 1" convolved with 1.5" -> 1.8027....
    target_beam = Beam(1.802775637731995*u.arcsec, 1.802775637731995*u.arcsec,
                       0*u.deg)

    conv_cube = cube.convolve_to(target_beam)

    pixscale = wcs.utils.proj_plane_pixel_area(cube.wcs.celestial)**0.5*u.deg

    for ii,bm in enumerate(cube.beams):
        expected = target_beam.deconvolve(bm).as_kernel(pixscale, x_size=5,
                                                        y_size=5)

        np.testing.assert_almost_equal(expected.array,
                                       conv_cube.filled_data[ii,:,:].value)

@pytest.mark.skipif('not REPROJECT_INSTALLED')
def test_reproject():

    cube, data = cube_and_raw('adv.fits')

    wcs_in = WCS(cube.header)
    wcs_out = wcs_in.deepcopy()
    wcs_out.wcs.ctype = ['GLON-SIN', 'GLAT-SIN', wcs_in.wcs.ctype[2]]
    wcs_out.wcs.crval = [134.37608, -31.939241, wcs_in.wcs.crval[2]]
    wcs_out.wcs.crpix = [2., 2., wcs_in.wcs.crpix[2]]

    header_out = cube.header
    header_out['NAXIS1'] = 4
    header_out['NAXIS2'] = 5
    header_out['NAXIS3'] = cube.shape[0]
    header_out.update(wcs_out.to_header())

    result = cube.reproject(header_out)

    assert result.shape == (cube.shape[0], 5, 4)

def test_spectral_smooth():

    cube, data = cube_and_raw('522_delta.fits')

    result = cube.spectral_smooth(kernel=convolution.Gaussian1DKernel(1.0))

    np.testing.assert_almost_equal(result[:,0,0].value,
                                   convolution.Gaussian1DKernel(1.0,
                                                                x_size=5).array,
                                   4)

def test_spectral_smooth_fail():

    cube, data = cube_and_raw('522_delta_beams.fits')

    with pytest.raises(AttributeError) as exc:
        cube.spectral_smooth(kernel=convolution.Gaussian1DKernel(1.0))

    assert exc.value.args[0] == ("VaryingResolutionSpectralCubes can't be "
                                 "spectrally smoothed.  Convolve to a "
                                 "common resolution with `convolve_to` before "
                                 "attempting spectral smoothed.")

def test_spectral_interpolate():

    cube, data = cube_and_raw('522_delta.fits')

    # midpoint between each position
    sg = (cube.spectral_axis[1:] + cube.spectral_axes[:-1])/2.

    result = cube.spectral_interpolate(spectral_grid=sg)

    np.testing.assert_almost_equal(result[:,0,0].value,
                                   [0.0, 0.5, 0.5, 0.0])

def test_spectral_interpolate_fail():

    cube, data = cube_and_raw('522_delta_beams.fits')

    with pytest.raises(AttributeError) as exc:
        cube.spectral_interpolate(5)

    assert exc.value.args[0] == ("VaryingResolutionSpectralCubes can't be "
                                 "spectrally interpolated.  Convolve to a "
                                 "common resolution with `convolve_to` before "
                                 "attempting spectral interpolation.")
<commit_msg>typo fix<commit_after>import pytest
import numpy as np
from astropy import units as u
from astropy import convolution
from astropy.wcs import WCS
from astropy import wcs

from .test_spectral_cube import cube_and_raw

try:
    from radio_beam import beam,Beam
    RADIO_BEAM_INSTALLED = True
except ImportError:
    RADIO_BEAM_INSTALLED = False

try:
    import reproject
    REPROJECT_INSTALLED = True
except ImportError:
    REPROJECT_INSTALLED = False

@pytest.mark.skipif('not RADIO_BEAM_INSTALLED')
def test_convolution():
    cube, data = cube_and_raw('255_delta.fits')

    # 1" convolved with 1.5" -> 1.8027....
    target_beam = Beam(1.802775637731995*u.arcsec, 1.802775637731995*u.arcsec,
                       0*u.deg)

    conv_cube = cube.convolve_to(target_beam)

    expected = convolution.Gaussian2DKernel((1.5*u.arcsec /
                                             beam.SIGMA_TO_FWHM /
                                             (5.555555555555e-4*u.deg)).decompose().value,
                                            x_size=5, y_size=5,
                                           )

    np.testing.assert_almost_equal(expected.array,
                                   conv_cube.filled_data[0,:,:].value)

    # 2nd layer is all zeros
    assert np.all(conv_cube.filled_data[1,:,:] == 0.0)

@pytest.mark.skipif('not RADIO_BEAM_INSTALLED')
def test_beams_convolution():
    cube, data = cube_and_raw('455_delta_beams.fits')

    # 1" convolved with 1.5" -> 1.8027....
    target_beam = Beam(1.802775637731995*u.arcsec, 1.802775637731995*u.arcsec,
                       0*u.deg)

    conv_cube = cube.convolve_to(target_beam)

    pixscale = wcs.utils.proj_plane_pixel_area(cube.wcs.celestial)**0.5*u.deg

    for ii,bm in enumerate(cube.beams):
        expected = target_beam.deconvolve(bm).as_kernel(pixscale, x_size=5,
                                                        y_size=5)

        np.testing.assert_almost_equal(expected.array,
                                       conv_cube.filled_data[ii,:,:].value)

@pytest.mark.skipif('not REPROJECT_INSTALLED')
def test_reproject():

    cube, data = cube_and_raw('adv.fits')

    wcs_in = WCS(cube.header)
    wcs_out = wcs_in.deepcopy()
    wcs_out.wcs.ctype = ['GLON-SIN', 'GLAT-SIN', wcs_in.wcs.ctype[2]]
    wcs_out.wcs.crval = [134.37608, -31.939241, wcs_in.wcs.crval[2]]
    wcs_out.wcs.crpix = [2., 2., wcs_in.wcs.crpix[2]]

    header_out = cube.header
    header_out['NAXIS1'] = 4
    header_out['NAXIS2'] = 5
    header_out['NAXIS3'] = cube.shape[0]
    header_out.update(wcs_out.to_header())

    result = cube.reproject(header_out)

    assert result.shape == (cube.shape[0], 5, 4)

def test_spectral_smooth():

    cube, data = cube_and_raw('522_delta.fits')

    result = cube.spectral_smooth(kernel=convolution.Gaussian1DKernel(1.0))

    np.testing.assert_almost_equal(result[:,0,0].value,
                                   convolution.Gaussian1DKernel(1.0,
                                                                x_size=5).array,
                                   4)

def test_spectral_smooth_fail():

    cube, data = cube_and_raw('522_delta_beams.fits')

    with pytest.raises(AttributeError) as exc:
        cube.spectral_smooth(kernel=convolution.Gaussian1DKernel(1.0))

    assert exc.value.args[0] == ("VaryingResolutionSpectralCubes can't be "
                                 "spectrally smoothed.  Convolve to a "
                                 "common resolution with `convolve_to` before "
                                 "attempting spectral smoothed.")

def test_spectral_interpolate():

    cube, data = cube_and_raw('522_delta.fits')

    # midpoint between each position
    sg = (cube.spectral_axis[1:] + cube.spectral_axis[:-1])/2.

    result = cube.spectral_interpolate(spectral_grid=sg)

    np.testing.assert_almost_equal(result[:,0,0].value,
                                   [0.0, 0.5, 0.5, 0.0])

def test_spectral_interpolate_fail():

    cube, data = cube_and_raw('522_delta_beams.fits')

    with pytest.raises(AttributeError) as exc:
        cube.spectral_interpolate(5)

    assert exc.value.args[0] == ("VaryingResolutionSpectralCubes can't be "
                                 "spectrally interpolated.  Convolve to a "
                                 "common resolution with `convolve_to` before "
                                 "attempting spectral interpolation.")
<|endoftext|>


<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from citest.service_testing import (base_agent, http_agent)

class JenkinsOperationStatus(base_agent.AgentOperationStatus):
  """Specialization of AgentOperationStatus for Jenkins operations.

  Jenkins operations are interesting to the effect that we use them to
  trigger steps in other services, meaning the status of Jenkins operation
  depends often on the status of another endpoint. To achieve this,
  this status class is effectively a wrapper around a second agent/status
  pair which collect information on the action resulting from the Jenkins
  trigger.
  """
  @property
  def id(self):
    return self.__trigger_status.id

  @property
  def finished(self):
    return self.__trigger_status.finished

  @property
  def finished_ok(self):
    return self.__trigger_status.finished_ok

  @property
  def detail(self):
    return self.__trigger_status.detail

  @property
  def error(self):
    return self.__trigger_status.error

  @property
  def status_agent(self):
    """
    The endpoint where the status of the Jenkins operation can be found.
    """
    return self.__trigger_status

  @property
  def timed_out(self):
    return self.__trigger_status.timed_out

  def refresh(self, trace=True):
    return self.__trigger_status.refresh(trace)

  def __init__(self, operation, status_class, path, http_response):
    """Constructs a JenkinsOperationStatus object.

    Args:
      operation [BaseJenkinsOperation]: The Jenkins operation that this is for.
      status_class [AgentOperationClass]: The status class used to monitor the
          action that the jenkins trigger kicked off.
      path [string]: The path the status_agent should poll on.
      http_response [HttpResponseType]: Response given by Jenkins to the
        trigger.
    """
    super(JenkinsOperationStatus, self).__init__(operation)
    self.__trigger_status = status_class(operation, http_response)
    self.__trigger_status._bind_id("(sync request has no ID)")
    self.__trigger_status._bind_detail_path(path)

  def __cmp__(self, response):
    return self.__trigger_status.__cmp__(response.__trigger_status)

  def __str__(self):
    return 'jenkins_status_agent={0}'.format(self.__trigger_status)

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_output(
        entity, 'Jenkins Status', self.__trigger_status)
    super(JenkinsOperationStatus, self).export_to_json_snapshot(
        snapshot, entity)

class JenkinsAgent(base_agent.BaseAgent):
  """A specialization of BaseAgent for interacting with Jenkins."""
  def __init__(self, baseUrl, auth_path, owner_agent):
    super(JenkinsAgent, self).__init__()
    self.__http_agent = http_agent.HttpAgent(baseUrl)
    self.__owner_agent = owner_agent

    # Allow up to 13 minutes to wait on operations.
    # 13 minutes is arbitrary. The current test takes around 6-7 minutes
    # end-to-end. Other use cases might make it more clear what this should be.
    self.default_max_wait_secs = 780

    # pylint: disable=bad-indentation
    if auth_path is None:
        auth_info = [os.environ.get('JENKINS_USER', None),
                     os.environ.get('JENKINS_PASSWORD', None)]
        if auth_info[0] is None or auth_info[1] is None:
          raise ValueError(
              'You must either supply --jenins_auth_path'
              ' or JENKINS_USER and JENKINS_PASSWORD environment variables.')
    else:
        with open(auth_path, 'r') as f:
          auth_info = f.read().split()

          if len(auth_info) != 2:
            raise ValueError(
                '--jenkins_auth_path={path} is not in the correct '
                'format. You must supply a file with the contents '
                '<username> <password'
                .format(path=auth_path))

    self.__http_agent.add_basic_auth_header(auth_info[0], auth_info[1])

  def get(self, path, trace=True):
    return self.__owner_agent.get(path, trace)

  def _trigger_jenkins_build(self, job, token):
    jenkins_path = 'job/{job}/build/?token={token}'.format(job=job,
                                                           token=token)
    self.logger.info('Triggering Jenkins %s', jenkins_path)
    result = self.__http_agent.get(jenkins_path, trace=True)
    result.check_ok()
    return result

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_control(
        entity, 'baseUrl', self.__http_agent.base_url)
    super(JenkinsAgent, self).export_to_json_snapshot(snapshot, entity)

  def new_jenkins_trigger_operation(self, title, job, token, status_class,
                                    status_path):
    """Returns a new JenkinsTriggerOperation

    Args:
      title [string]: Name of the operation.
      job [string]: Name of the Jenkins job to trigger.
      token [string]: Token required for triggering the job.
      status_class [AgentOperationStatus]: The status object in charge of
        recording the success of the action resulting from the trigger.
      status_path [string]: The path the status_class must poll on for
        success of the trigger
    """
    return JenkinsTriggerOperation(
        title=title, jenkins_agent=self,
        status_class=status_class, job=job, token=token,
        status_path=status_path)

class BaseJenkinsOperation(base_agent.AgentOperation):
  @property
  def status_class(self):
    return self.__status_class

  @property
  def data(self):
    return self.__data

  def __init__(self, title, jenkins_agent,
               status_class=JenkinsOperationStatus):
    """Construct a BaseJenkinsOperation

    Args:
      title [string]: Name of operation.
      jenkins_agent [JenkinsAgent]: JenkinsAgent class that has the Jenkins
        configuration stored.
      status_class [AgentOperationStatus]: The status class that will
       confirm success of the action resulting from the Jenkins trigger.
    """
    super(BaseJenkinsOperation, self).__init__(title, jenkins_agent)
    if not jenkins_agent or not isinstance(jenkins_agent, JenkinsAgent):
      raise TypeError('agent not a  JenkinsAgent: '
                      + jenkins_agent.__class__.__name__)

    self.__status_class = status_class
    self.__data = {}

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_mechanism(entity, 'Jenkins Agent', self.agent)
    # TODO(ewiseblatt): Not sure if this is input or output.
    snapshot.edge_builder.make_data(
        entity, 'Payload Data', self.__data, format='json')
    super(BaseJenkinsOperation, self).export_to_json_snapshot(snapshot, entity)

  def execute(self, agent=None, trace=True):
    if not self.agent:
      if not isinstance(agent, JenkinsAgent):
        raise TypeError('agent not a JenkinsAgent: '
                        + agent.__class__.__name__)
      self.bind_agent(agent)

    status = self._do_execute(agent, trace)
    if trace:
      agent.logger.debug('Returning status %s', status)
    return status

  def _do_execute(self, agent, trace=True):
    raise UnimplementedError('{0}._do_execute'.format(type(self)))

class JenkinsTriggerOperation(BaseJenkinsOperation):
  """Specialization of AgentOperation that triggers a build at the specified
  project.
  """
  def __init__(self, title, jenkins_agent, token, job, status_class,
               status_path):
    """Construct a JenkinsTriggerOperation.

    Args:
      title [string]: Name of the operation.
      jenkins_agent [JenkinsAgent]: The JenkensAgent class that has the Jenkins
        configuration stored.
      job [string]: Name of the Jenkins job to be triggered.
      token [string]: The token required to trigger the Jenkins job.
      status_class [AgentOperationClass]: The status class used to monitor the
          action that the jenkins trigger kicked off.
      status_path [string]: The path the status should poll for success on.
    """
    super(JenkinsTriggerOperation, self).__init__(
        jenkins_agent=jenkins_agent, status_class=status_class, title=title)

    self.__token = token
    self.__job = job
    self.__status_class = status_class
    self.__status_path = status_path

  def _do_execute(self, agent, trace=True):
    http_response = self.agent._trigger_jenkins_build(job=self.__job,
                                                       token=self.__token)
    agent.logger.debug(
        'Checking for effect of jenkins trigger at url={url}'.format(
            url=self.__status_path))

    return JenkinsOperationStatus(
        self, self.__status_class, self.__status_path, http_response)
<commit_msg>Fix typo<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os

from citest.service_testing import (base_agent, http_agent)

class JenkinsOperationStatus(base_agent.AgentOperationStatus):
  """Specialization of AgentOperationStatus for Jenkins operations.

  Jenkins operations are interesting to the effect that we use them to
  trigger steps in other services, meaning the status of Jenkins operation
  depends often on the status of another endpoint. To achieve this,
  this status class is effectively a wrapper around a second agent/status
  pair which collect information on the action resulting from the Jenkins
  trigger.
  """
  @property
  def id(self):
    return self.__trigger_status.id

  @property
  def finished(self):
    return self.__trigger_status.finished

  @property
  def finished_ok(self):
    return self.__trigger_status.finished_ok

  @property
  def detail(self):
    return self.__trigger_status.detail

  @property
  def error(self):
    return self.__trigger_status.error

  @property
  def status_agent(self):
    """
    The endpoint where the status of the Jenkins operation can be found.
    """
    return self.__trigger_status

  @property
  def timed_out(self):
    return self.__trigger_status.timed_out

  def refresh(self, trace=True):
    return self.__trigger_status.refresh(trace)

  def __init__(self, operation, status_class, path, http_response):
    """Constructs a JenkinsOperationStatus object.

    Args:
      operation [BaseJenkinsOperation]: The Jenkins operation that this is for.
      status_class [AgentOperationClass]: The status class used to monitor the
          action that the jenkins trigger kicked off.
      path [string]: The path the status_agent should poll on.
      http_response [HttpResponseType]: Response given by Jenkins to the
        trigger.
    """
    super(JenkinsOperationStatus, self).__init__(operation)
    self.__trigger_status = status_class(operation, http_response)
    self.__trigger_status._bind_id("(sync request has no ID)")
    self.__trigger_status._bind_detail_path(path)

  def __cmp__(self, response):
    return self.__trigger_status.__cmp__(response.__trigger_status)

  def __str__(self):
    return 'jenkins_status_agent={0}'.format(self.__trigger_status)

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_output(
        entity, 'Jenkins Status', self.__trigger_status)
    super(JenkinsOperationStatus, self).export_to_json_snapshot(
        snapshot, entity)

class JenkinsAgent(base_agent.BaseAgent):
  """A specialization of BaseAgent for interacting with Jenkins."""
  def __init__(self, baseUrl, auth_path, owner_agent):
    super(JenkinsAgent, self).__init__()
    self.__http_agent = http_agent.HttpAgent(baseUrl)
    self.__owner_agent = owner_agent

    # Allow up to 13 minutes to wait on operations.
    # 13 minutes is arbitrary. The current test takes around 6-7 minutes
    # end-to-end. Other use cases might make it more clear what this should be.
    self.default_max_wait_secs = 780

    # pylint: disable=bad-indentation
    if auth_path is None:
        auth_info = [os.environ.get('JENKINS_USER', None),
                     os.environ.get('JENKINS_PASSWORD', None)]
        if auth_info[0] is None or auth_info[1] is None:
          raise ValueError(
              'You must either supply --jenkins_auth_path'
              ' or JENKINS_USER and JENKINS_PASSWORD environment variables.')
    else:
        with open(auth_path, 'r') as f:
          auth_info = f.read().split()

          if len(auth_info) != 2:
            raise ValueError(
                '--jenkins_auth_path={path} is not in the correct '
                'format. You must supply a file with the contents '
                '<username> <password'
                .format(path=auth_path))

    self.__http_agent.add_basic_auth_header(auth_info[0], auth_info[1])

  def get(self, path, trace=True):
    return self.__owner_agent.get(path, trace)

  def _trigger_jenkins_build(self, job, token):
    jenkins_path = 'job/{job}/build/?token={token}'.format(job=job,
                                                           token=token)
    self.logger.info('Triggering Jenkins %s', jenkins_path)
    result = self.__http_agent.get(jenkins_path, trace=True)
    result.check_ok()
    return result

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_control(
        entity, 'baseUrl', self.__http_agent.base_url)
    super(JenkinsAgent, self).export_to_json_snapshot(snapshot, entity)

  def new_jenkins_trigger_operation(self, title, job, token, status_class,
                                    status_path):
    """Returns a new JenkinsTriggerOperation

    Args:
      title [string]: Name of the operation.
      job [string]: Name of the Jenkins job to trigger.
      token [string]: Token required for triggering the job.
      status_class [AgentOperationStatus]: The status object in charge of
        recording the success of the action resulting from the trigger.
      status_path [string]: The path the status_class must poll on for
        success of the trigger
    """
    return JenkinsTriggerOperation(
        title=title, jenkins_agent=self,
        status_class=status_class, job=job, token=token,
        status_path=status_path)

class BaseJenkinsOperation(base_agent.AgentOperation):
  @property
  def status_class(self):
    return self.__status_class

  @property
  def data(self):
    return self.__data

  def __init__(self, title, jenkins_agent,
               status_class=JenkinsOperationStatus):
    """Construct a BaseJenkinsOperation

    Args:
      title [string]: Name of operation.
      jenkins_agent [JenkinsAgent]: JenkinsAgent class that has the Jenkins
        configuration stored.
      status_class [AgentOperationStatus]: The status class that will
       confirm success of the action resulting from the Jenkins trigger.
    """
    super(BaseJenkinsOperation, self).__init__(title, jenkins_agent)
    if not jenkins_agent or not isinstance(jenkins_agent, JenkinsAgent):
      raise TypeError('agent not a  JenkinsAgent: '
                      + jenkins_agent.__class__.__name__)

    self.__status_class = status_class
    self.__data = {}

  def export_to_json_snapshot(self, snapshot, entity):
    snapshot.edge_builder.make_mechanism(entity, 'Jenkins Agent', self.agent)
    # TODO(ewiseblatt): Not sure if this is input or output.
    snapshot.edge_builder.make_data(
        entity, 'Payload Data', self.__data, format='json')
    super(BaseJenkinsOperation, self).export_to_json_snapshot(snapshot, entity)

  def execute(self, agent=None, trace=True):
    if not self.agent:
      if not isinstance(agent, JenkinsAgent):
        raise TypeError('agent not a JenkinsAgent: '
                        + agent.__class__.__name__)
      self.bind_agent(agent)

    status = self._do_execute(agent, trace)
    if trace:
      agent.logger.debug('Returning status %s', status)
    return status

  def _do_execute(self, agent, trace=True):
    raise UnimplementedError('{0}._do_execute'.format(type(self)))

class JenkinsTriggerOperation(BaseJenkinsOperation):
  """Specialization of AgentOperation that triggers a build at the specified
  project.
  """
  def __init__(self, title, jenkins_agent, token, job, status_class,
               status_path):
    """Construct a JenkinsTriggerOperation.

    Args:
      title [string]: Name of the operation.
      jenkins_agent [JenkinsAgent]: The JenkensAgent class that has the Jenkins
        configuration stored.
      job [string]: Name of the Jenkins job to be triggered.
      token [string]: The token required to trigger the Jenkins job.
      status_class [AgentOperationClass]: The status class used to monitor the
          action that the jenkins trigger kicked off.
      status_path [string]: The path the status should poll for success on.
    """
    super(JenkinsTriggerOperation, self).__init__(
        jenkins_agent=jenkins_agent, status_class=status_class, title=title)

    self.__token = token
    self.__job = job
    self.__status_class = status_class
    self.__status_path = status_path

  def _do_execute(self, agent, trace=True):
    http_response = self.agent._trigger_jenkins_build(job=self.__job,
                                                       token=self.__token)
    agent.logger.debug(
        'Checking for effect of jenkins trigger at url={url}'.format(
            url=self.__status_path))

    return JenkinsOperationStatus(
        self, self.__status_class, self.__status_path, http_response)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

__author__ = 'Filipe Ximenes'
__email__ = 'filipeximenes@gmail.com'
__version__ = '0.4.1'

from .tapioca_facebook import Facebook
<commit_msg>updating version<commit_after># -*- coding: utf-8 -*-

__author__ = 'Filipe Ximenes'
__email__ = 'filipeximenes@gmail.com'
__version__ = '0.5.0'

from .tapioca_facebook import Facebook
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2014-2016 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Base class for sources' REST calls

"""

import sys
import os
import urllib2
try:
    #added to allow GAE to work
    from google.appengine.api import urlfetch
    GAE_ENABLED = True
except ImportError:
    GAE_ENABLED = False
    import ssl

try:
    import simplejson as json
except ImportError:
    import json

from threading import Thread

PYTHON_2_7_9 = len(urllib2.urlopen.__defaults__) > 2
PYTHON_2 = sys.version_info < (3, 0)

if PYTHON_2:
    from poster.encode import multipart_encode, MultipartParam
if PYTHON_2_7_9:
    from bigml.sslposter import StreamingHTTPSHandler, register_openers
elif PYTHON_2:
    from poster.streaminghttp import StreamingHTTPSHandler, register_openers

from bigml.util import (localize, clear_console_line, reset_console_line,
                        console_log, is_url)
from bigml.bigmlconnection import (
    HTTP_CREATED, HTTP_ACCEPTED, HTTP_BAD_REQUEST,
    HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_NOT_FOUND,
    HTTP_TOO_MANY_REQUESTS, HTTP_FORBIDDEN,
    HTTP_INTERNAL_SERVER_ERROR, GAE_ENABLED, SEND_JSON)
from bigml.resourcehandler import (check_resource_type,
                                   resource_is_ready,
                                   get_source_id)
from bigml.constants import SOURCE_PATH, UPLOADING
from bigml.resourcehandler import ResourceHandler, LOGGER

if PYTHON_2:
    register_openers()
else:
    import requests
    from bigml.util import maybe_save

class SourceHandler(ResourceHandler):

    """This class is used by the BigML class as
       a mixin that provides the REST calls to sources. It should not
       be instantiated independently.

    """

    def __init__(self):
        """Initializes the SourceHandler. This class is intended to be
           used as a mixin on ResourceHandler, that inherits its
           attributes and basic method from BigMLConnection, and must not be
           instantiated independently.

        """
        self.source_url = self.url + SOURCE_PATH

    def _create_remote_source(self, url, args=None):
        """Creates a new source using a URL

        """
        create_args = {}
        if args is not None:
            create_args.update(args)
        create_args.update({"remote": url})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _create_inline_source(self, src_obj, args=None):
        """Create source from inline data

           The src_obj data should be a list of rows stored as dict or
           list objects.
        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        # some basic validation
        if (not isinstance(src_obj, list) or (
                not all([isinstance(row, dict) for row in src_obj]) and
                not all([isinstance(row, list) for row in src_obj]))):
            raise TypeError(
                'ERROR: inline source must be a list of dicts or a '
                'list of lists')

        create_args.update({"data": json.dumps(src_obj)})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _upload_source(self, args, source, out=sys.stdout):
        """Uploads a source asynchronously.

        """

        def update_progress(param, current, total):
            """Updates source's progress.

            """
            progress = round(current * 1.0 / total, 2)
            if progress < 1.0:
                source['object']['status']['progress'] = progress

        resource = self._process_source(source['resource'], source['location'],
                                        source['object'],
                                        args=args, progress_bar=True,
                                        callback=update_progress, out=out)
        source['code'] = resource['code']
        source['resource'] = resource['resource']
        source['location'] = resource['location']
        source['object'] = resource['object']
        source['error'] = resource['error']

    def _stream_source(self, file_name, args=None, async=False,
                       progress_bar=False, out=sys.stdout):
        """Creates a new source.

        """

        def draw_progress_bar(param, current, total):
            """Draws a text based progress report.

            """
            pct = 100 - ((total - current) * 100) / (total)
            console_log("Uploaded %s out of %s bytes [%s%%]" % (
                localize(current), localize(total), pct), reset=True)
        create_args = {}
        if args is not None:
            create_args.update(args)
        if 'source_parser' in create_args:
            create_args['source_parser'] = json.dumps(
                create_args['source_parser'])

        resource_id = None
        location = None
        resource = None
        error = None

        try:
            if isinstance(file_name, basestring):
                create_args.update({os.path.basename(file_name):
                                    open(file_name, "rb")})
            else:
                create_args = create_args.items()
                name = '<none>'
                create_args.append(MultipartParam(name, filename=name,
                                                  fileobj=file_name))

        except IOError, exception:
            raise IOError("Error: cannot read training set. %s" %
                          str(exception))

        if async:
            source = {
                'code': HTTP_ACCEPTED,
                'resource': resource_id,
                'location': location,
                'object': {'status': {'message': 'The upload is in progress',
                                      'code': UPLOADING,
                                      'progress': 0.0}},
                'error': error}
            upload_args = (create_args, source)
            thread = Thread(target=self._upload_source,
                            args=upload_args,
                            kwargs={'out': out})
            thread.start()
            return source
        return self._process_source(resource_id, location, resource,
                                    args=create_args,
                                    progress_bar=progress_bar,
                                    callback=draw_progress_bar, out=out)

    def _process_source(self, resource_id, location, resource,
                        args=None, progress_bar=False, callback=None,
                        out=sys.stdout):
        """Creates a new source.

        """
        code = HTTP_INTERNAL_SERVER_ERROR
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        if progress_bar and callback is not None:
            body, headers = multipart_encode(args, cb=callback)
        else:
            body, headers = multipart_encode(args)

        if GAE_ENABLED:
            try:
                response = urlfetch.fetch(url=self.source_url + self.auth,
                                          payload="".join(body),
                                          method=urlfetch.POST,
                                          headers=headers)
                code = response.status_code
                content = response.content
                if code in [HTTP_CREATED]:
                    if 'location' in response.headers:
                        location = response.headers['location']
                    resource = json.loads(response.content.decode(), 'utf-8')
                    resource_id = resource['resource']
                    error = {}
                elif code in [HTTP_BAD_REQUEST,
                              HTTP_UNAUTHORIZED,
                              HTTP_PAYMENT_REQUIRED,
                              HTTP_FORBIDDEN,
                              HTTP_NOT_FOUND,
                              HTTP_TOO_MANY_REQUESTS]:
                    error = json.loads(response.content.decode(), 'utf-8')
                    LOGGER.error(self.error_message(error, method='create'))
                elif code != HTTP_ACCEPTED:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR
            except urlfetch.Error, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
        else:
            try:
                request = urllib2.Request(self.source_url + self.auth,
                                          body, headers)
                # try using the new SSL checking in python 2.7.9
                try:
                    if not self.verify and PYTHON_2_7_9:
                        context = ssl.create_default_context(
                            ssl.Purpose.CLIENT_AUTH)
                        context.verify_mode = ssl.CERT_NONE
                        https_handler = StreamingHTTPSHandler(context=context)
                        opener = urllib2.build_opener(https_handler)
                        urllib2.install_opener(opener)
                        response = urllib2.urlopen(request)
                    else:
                        response = urllib2.urlopen(request)
                except AttributeError:
                    response = urllib2.urlopen(request)
                clear_console_line(out=out)
                reset_console_line(out=out)
                code = response.getcode()
                if code == HTTP_CREATED:
                    location = response.headers['location']
                    content = response.read()
                    resource = json.loads(content, 'utf-8')
                    resource_id = resource['resource']
                    error = {}
            except ValueError:
                LOGGER.error("Malformed response.")
            except urllib2.HTTPError, exception:
                code = exception.code
                if code in [HTTP_BAD_REQUEST,
                            HTTP_UNAUTHORIZED,
                            HTTP_PAYMENT_REQUIRED,
                            HTTP_NOT_FOUND,
                            HTTP_TOO_MANY_REQUESTS]:
                    content = exception.read()
                    error = json.loads(content.decode(), 'utf-8')
                    LOGGER.error(self.error_message(error, method='create'))
                else:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR

            except urllib2.URLError, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
                error = exception.args
        return {
            'code': code,
            'resource': resource_id,
            'location': location,
            'object': resource,
            'error': error}

    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list) or
                                      isinstance(value, dict)):
                create_args[key] = json.dumps(value)

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        try:

            if isinstance(file_name, basestring):
                files = {os.path.basename(file_name): open(file_name, "rb")}
            else:
                files = {'stdin': file_name}
        except IOError:
            sys.exit("ERROR: cannot read training set")

        if GAE_ENABLED:
            try:
                req_options = {
                    'url': self.source_url + self.auth,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': files,
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s",
                             str(exception))
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
        else:
            try:
                response = requests.post(self.source_url + self.auth,
                                         files=files,
                                         data=create_args, verify=self.verify)
            except (requests.ConnectionError,
                    requests.Timeout,
                    requests.RequestException), exc:
                LOGGER.error("HTTP request error: %s", str(exc))
                code = HTTP_INTERNAL_SERVER_ERROR
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
        try:
            code = response.status_code
            if code == HTTP_CREATED:
                location = response.headers['location']
                resource = json.loads(response.content.decode(), 'utf-8')
                resource_id = resource['resource']
                error = None
            elif code in [HTTP_BAD_REQUEST,
                          HTTP_UNAUTHORIZED,
                          HTTP_PAYMENT_REQUIRED,
                          HTTP_NOT_FOUND,
                          HTTP_TOO_MANY_REQUESTS]:
                error = json.loads(response.content.decode(), 'utf-8')
            else:
                LOGGER.error("Unexpected error (%s)" % code)
                code = HTTP_INTERNAL_SERVER_ERROR

        except ValueError:
            LOGGER.error("Malformed response")

        return maybe_save(resource_id, self.storage, code,
                          location, resource, error)

    def create_source(self, path=None, args=None, async=False,
                      progress_bar=False, out=sys.stdout):
        """Creates a new source.

           The source can be a local file path or a URL.

        """

        if path is None:
            raise Exception('A local path or a valid URL must be provided.')

        if is_url(path):
            return self._create_remote_source(path, args=args)
        elif isinstance(path, list):
            return self._create_inline_source(path, args=args)
        elif PYTHON_2:
            return self._stream_source(file_name=path, args=args, async=async,
                                       progress_bar=progress_bar, out=out)
        else:
            return self._create_local_source(file_name=path, args=args)

    def get_source(self, source, query_string=''):
        """Retrieves a remote source.
           The source parameter should be a string containing the
           source id or the dict returned by create_source.
           As source is an evolving object that is processed
           until it reaches the FINISHED or FAULTY state, thet function will
           return a dict that encloses the source values and state info
           available at the time it is called.
        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            return self._get("%s%s" % (self.url, source_id),
                             query_string=query_string)

    def source_is_ready(self, source):
        """Checks whether a source' status is FINISHED.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source = self.get_source(source)
        return resource_is_ready(source)

    def list_sources(self, query_string=''):
        """Lists all your remote sources.

        """
        return self._list(self.source_url, query_string)

    def update_source(self, source, changes):
        """Updates a source.

        Updates remote `source` with `changes'.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            body = json.dumps(changes)
            return self._update("%s%s" % (self.url, source_id), body)

    def delete_source(self, source):
        """Deletes a remote source permanently.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            return self._delete("%s%s" % (self.url, source_id))
<commit_msg>Bug fixing: adapting to changes in Python 3.6 for source API calls<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/env python
#
# Copyright 2014-2016 BigML
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

"""Base class for sources' REST calls

"""

import sys
import os
import urllib2
try:
    #added to allow GAE to work
    from google.appengine.api import urlfetch
    GAE_ENABLED = True
except ImportError:
    GAE_ENABLED = False
    import ssl

try:
    import simplejson as json
except ImportError:
    import json

from threading import Thread

PYTHON_2_7_9 = len(urllib2.urlopen.__defaults__) > 2
PYTHON_2 = sys.version_info < (3, 0)

if PYTHON_2:
    from poster.encode import multipart_encode, MultipartParam
if PYTHON_2_7_9:
    from bigml.sslposter import StreamingHTTPSHandler, register_openers
elif PYTHON_2:
    from poster.streaminghttp import StreamingHTTPSHandler, register_openers

from bigml.util import (localize, clear_console_line, reset_console_line,
                        console_log, is_url)
from bigml.bigmlconnection import (
    HTTP_CREATED, HTTP_ACCEPTED, HTTP_BAD_REQUEST,
    HTTP_UNAUTHORIZED, HTTP_PAYMENT_REQUIRED, HTTP_NOT_FOUND,
    HTTP_TOO_MANY_REQUESTS, HTTP_FORBIDDEN,
    HTTP_INTERNAL_SERVER_ERROR, GAE_ENABLED, SEND_JSON)
from bigml.bigmlconnection import json_load
from bigml.resourcehandler import (check_resource_type,
                                   resource_is_ready,
                                   get_source_id)
from bigml.constants import SOURCE_PATH, UPLOADING
from bigml.resourcehandler import ResourceHandler, LOGGER

if PYTHON_2:
    register_openers()
else:
    import requests
    from bigml.util import maybe_save

class SourceHandler(ResourceHandler):

    """This class is used by the BigML class as
       a mixin that provides the REST calls to sources. It should not
       be instantiated independently.

    """

    def __init__(self):
        """Initializes the SourceHandler. This class is intended to be
           used as a mixin on ResourceHandler, that inherits its
           attributes and basic method from BigMLConnection, and must not be
           instantiated independently.

        """
        self.source_url = self.url + SOURCE_PATH

    def _create_remote_source(self, url, args=None):
        """Creates a new source using a URL

        """
        create_args = {}
        if args is not None:
            create_args.update(args)
        create_args.update({"remote": url})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _create_inline_source(self, src_obj, args=None):
        """Create source from inline data

           The src_obj data should be a list of rows stored as dict or
           list objects.
        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        # some basic validation
        if (not isinstance(src_obj, list) or (
                not all([isinstance(row, dict) for row in src_obj]) and
                not all([isinstance(row, list) for row in src_obj]))):
            raise TypeError(
                'ERROR: inline source must be a list of dicts or a '
                'list of lists')

        create_args.update({"data": json.dumps(src_obj)})
        body = json.dumps(create_args)
        return self._create(self.source_url, body)

    def _upload_source(self, args, source, out=sys.stdout):
        """Uploads a source asynchronously.

        """

        def update_progress(param, current, total):
            """Updates source's progress.

            """
            progress = round(current * 1.0 / total, 2)
            if progress < 1.0:
                source['object']['status']['progress'] = progress

        resource = self._process_source(source['resource'], source['location'],
                                        source['object'],
                                        args=args, progress_bar=True,
                                        callback=update_progress, out=out)
        source['code'] = resource['code']
        source['resource'] = resource['resource']
        source['location'] = resource['location']
        source['object'] = resource['object']
        source['error'] = resource['error']

    def _stream_source(self, file_name, args=None, async=False,
                       progress_bar=False, out=sys.stdout):
        """Creates a new source.

        """

        def draw_progress_bar(param, current, total):
            """Draws a text based progress report.

            """
            pct = 100 - ((total - current) * 100) / (total)
            console_log("Uploaded %s out of %s bytes [%s%%]" % (
                localize(current), localize(total), pct), reset=True)
        create_args = {}
        if args is not None:
            create_args.update(args)
        if 'source_parser' in create_args:
            create_args['source_parser'] = json.dumps(
                create_args['source_parser'])

        resource_id = None
        location = None
        resource = None
        error = None

        try:
            if isinstance(file_name, basestring):
                create_args.update({os.path.basename(file_name):
                                    open(file_name, "rb")})
            else:
                create_args = create_args.items()
                name = '<none>'
                create_args.append(MultipartParam(name, filename=name,
                                                  fileobj=file_name))

        except IOError, exception:
            raise IOError("Error: cannot read training set. %s" %
                          str(exception))

        if async:
            source = {
                'code': HTTP_ACCEPTED,
                'resource': resource_id,
                'location': location,
                'object': {'status': {'message': 'The upload is in progress',
                                      'code': UPLOADING,
                                      'progress': 0.0}},
                'error': error}
            upload_args = (create_args, source)
            thread = Thread(target=self._upload_source,
                            args=upload_args,
                            kwargs={'out': out})
            thread.start()
            return source
        return self._process_source(resource_id, location, resource,
                                    args=create_args,
                                    progress_bar=progress_bar,
                                    callback=draw_progress_bar, out=out)

    def _process_source(self, resource_id, location, resource,
                        args=None, progress_bar=False, callback=None,
                        out=sys.stdout):
        """Creates a new source.

        """
        code = HTTP_INTERNAL_SERVER_ERROR
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        if progress_bar and callback is not None:
            body, headers = multipart_encode(args, cb=callback)
        else:
            body, headers = multipart_encode(args)

        if GAE_ENABLED:
            try:
                response = urlfetch.fetch(url=self.source_url + self.auth,
                                          payload="".join(body),
                                          method=urlfetch.POST,
                                          headers=headers)
                code = response.status_code
                content = response.content
                if code in [HTTP_CREATED]:
                    if 'location' in response.headers:
                        location = response.headers['location']
                    resource = json_load(response.content)
                    resource_id = resource['resource']
                    error = {}
                elif code in [HTTP_BAD_REQUEST,
                              HTTP_UNAUTHORIZED,
                              HTTP_PAYMENT_REQUIRED,
                              HTTP_FORBIDDEN,
                              HTTP_NOT_FOUND,
                              HTTP_TOO_MANY_REQUESTS]:
                    error = json_load(response.content)
                    LOGGER.error(self.error_message(error, method='create'))
                elif code != HTTP_ACCEPTED:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR
            except urlfetch.Error, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
        else:
            try:
                request = urllib2.Request(self.source_url + self.auth,
                                          body, headers)
                # try using the new SSL checking in python 2.7.9
                try:
                    if not self.verify and PYTHON_2_7_9:
                        context = ssl.create_default_context(
                            ssl.Purpose.CLIENT_AUTH)
                        context.verify_mode = ssl.CERT_NONE
                        https_handler = StreamingHTTPSHandler(context=context)
                        opener = urllib2.build_opener(https_handler)
                        urllib2.install_opener(opener)
                        response = urllib2.urlopen(request)
                    else:
                        response = urllib2.urlopen(request)
                except AttributeError:
                    response = urllib2.urlopen(request)
                clear_console_line(out=out)
                reset_console_line(out=out)
                code = response.getcode()
                if code == HTTP_CREATED:
                    location = response.headers['location']
                    content = response.read()
                    resource = json_load(content)
                    resource_id = resource['resource']
                    error = {}
            except ValueError:
                LOGGER.error("Malformed response.")
            except urllib2.HTTPError, exception:
                code = exception.code
                if code in [HTTP_BAD_REQUEST,
                            HTTP_UNAUTHORIZED,
                            HTTP_PAYMENT_REQUIRED,
                            HTTP_NOT_FOUND,
                            HTTP_TOO_MANY_REQUESTS]:
                    content = exception.read()
                    error = json_load(content)
                    LOGGER.error(self.error_message(error, method='create'))
                else:
                    LOGGER.error("Unexpected error (%s)", code)
                    code = HTTP_INTERNAL_SERVER_ERROR

            except urllib2.URLError, exception:
                LOGGER.error("Error establishing connection: %s",
                             str(exception))
                error = exception.args
        return {
            'code': code,
            'resource': resource_id,
            'location': location,
            'object': resource,
            'error': error}

    def _create_local_source(self, file_name, args=None):
        """Creates a new source using a local file.

        This function is only used from Python 3. No async-prepared.

        """
        create_args = {}
        if args is not None:
            create_args.update(args)

        for key, value in create_args.items():
            if value is not None and (isinstance(value, list) or
                                      isinstance(value, dict)):
                create_args[key] = json.dumps(value)

        code = HTTP_INTERNAL_SERVER_ERROR
        resource_id = None
        location = None
        resource = None
        error = {
            "status": {
                "code": code,
                "message": "The resource couldn't be created"}}

        try:

            if isinstance(file_name, basestring):
                files = {os.path.basename(file_name): open(file_name, "rb")}
            else:
                files = {'stdin': file_name}
        except IOError:
            sys.exit("ERROR: cannot read training set")

        if GAE_ENABLED:
            try:
                req_options = {
                    'url': self.source_url + self.auth,
                    'method': urlfetch.POST,
                    'headers': SEND_JSON,
                    'data': create_args,
                    'files': files,
                    'validate_certificate': self.verify
                }
                response = urlfetch.fetch(**req_options)
            except urlfetch.Error, exception:
                LOGGER.error("HTTP request error: %s",
                             str(exception))
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
        else:
            try:
                response = requests.post(self.source_url + self.auth,
                                         files=files,
                                         data=create_args, verify=self.verify)
            except (requests.ConnectionError,
                    requests.Timeout,
                    requests.RequestException), exc:
                LOGGER.error("HTTP request error: %s", str(exc))
                code = HTTP_INTERNAL_SERVER_ERROR
                return maybe_save(resource_id, self.storage, code,
                                  location, resource, error)
        try:
            code = response.status_code
            if code == HTTP_CREATED:
                location = response.headers['location']
                resource = json_load(response.content)
                resource_id = resource['resource']
                error = None
            elif code in [HTTP_BAD_REQUEST,
                          HTTP_UNAUTHORIZED,
                          HTTP_PAYMENT_REQUIRED,
                          HTTP_NOT_FOUND,
                          HTTP_TOO_MANY_REQUESTS]:
                error = json_load(response.content)
            else:
                LOGGER.error("Unexpected error (%s)" % code)
                code = HTTP_INTERNAL_SERVER_ERROR

        except ValueError:
            LOGGER.error("Malformed response")

        return maybe_save(resource_id, self.storage, code,
                          location, resource, error)

    def create_source(self, path=None, args=None, async=False,
                      progress_bar=False, out=sys.stdout):
        """Creates a new source.

           The source can be a local file path or a URL.

        """

        if path is None:
            raise Exception('A local path or a valid URL must be provided.')

        if is_url(path):
            return self._create_remote_source(path, args=args)
        elif isinstance(path, list):
            return self._create_inline_source(path, args=args)
        elif PYTHON_2:
            return self._stream_source(file_name=path, args=args, async=async,
                                       progress_bar=progress_bar, out=out)
        else:
            return self._create_local_source(file_name=path, args=args)

    def get_source(self, source, query_string=''):
        """Retrieves a remote source.
           The source parameter should be a string containing the
           source id or the dict returned by create_source.
           As source is an evolving object that is processed
           until it reaches the FINISHED or FAULTY state, thet function will
           return a dict that encloses the source values and state info
           available at the time it is called.
        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            return self._get("%s%s" % (self.url, source_id),
                             query_string=query_string)

    def source_is_ready(self, source):
        """Checks whether a source' status is FINISHED.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source = self.get_source(source)
        return resource_is_ready(source)

    def list_sources(self, query_string=''):
        """Lists all your remote sources.

        """
        return self._list(self.source_url, query_string)

    def update_source(self, source, changes):
        """Updates a source.

        Updates remote `source` with `changes'.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            body = json.dumps(changes)
            return self._update("%s%s" % (self.url, source_id), body)

    def delete_source(self, source):
        """Deletes a remote source permanently.

        """
        check_resource_type(source, SOURCE_PATH,
                            message="A source id is needed.")
        source_id = get_source_id(source)
        if source_id:
            return self._delete("%s%s" % (self.url, source_id))
<|endoftext|>