<commit_before># Copyright 2012-2014 Mattias Fliesberg
#
# This file is part of opmuse.
#
# opmuse is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# opmuse is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with opmuse.  If not, see <http://www.gnu.org/licenses/>.

import os
import time
import subprocess
import re
import cherrypy
import fcntl
import select
import logging
import signal
from subprocess import TimeoutExpired

def debug(msg):
    cherrypy.log.error(msg, context='transcoding', severity=logging.DEBUG)

def log(msg):
    cherrypy.log(msg, context='transcoding')

class FFMPEGError(Exception):
    pass

class FFMPEGTranscoderSubprocessTool(cherrypy.Tool):
    """
    This tool makes sure the ffmpeg subprocess is ended
    properly when a request is cancelled
    """
    def __init__(self):
        cherrypy.Tool.__init__(self, 'on_end_request',
                               self.end, priority=20)

    def end(self):
        transcoding_transcoder = None

        if (hasattr(cherrypy.request, 'transcoding_transcoder') and
                cherrypy.request.transcoding_transcoder is not None):

            transcoding_transcoder = cherrypy.request.transcoding_transcoder

            cherrypy.request.transcoding_transcoder = None

            transcoding_transcoder.stop()

        if (hasattr(cherrypy.request, 'transcoding_track') and
                cherrypy.request.transcoding_track is not None):

            transcoding_track = cherrypy.request.transcoding_track

            cherrypy.request.transcoding_track = None

            cherrypy.engine.publish('transcoding.end',
                                    track=transcoding_track,
                                    transcoder=transcoding_transcoder)

            debug('"%s" transcoding ended.' % transcoding_track)

class Transcoder:
    def transcode(self):
        raise NotImplementedError()

    @staticmethod
    def outputs():
        raise NotImplementedError()

class FFMPEGTranscoder(Transcoder):

    def __init__(self, track, skip_seconds):
        self.track = track
        self.skip_seconds = skip_seconds
        self.stderr = None
        self.success = False
        self.error = None
        self.stopped = False
        self.process = None

        if cherrypy.request.app is not None:
            ffmpeg_cmd = cherrypy.request.app.config.get('opmuse').get('transcoding.ffmpeg_cmd')
        else:
            ffmpeg_cmd = None

        if ffmpeg_cmd is not None:
            self.ffmpeg_cmd = ffmpeg_cmd
        else:
            self.ffmpeg_cmd = 'ffmpeg'

    def __enter__(self):
        cherrypy.engine.publish('transcoding.start', transcoder=self, track=self.track)

        self.filename = self.track.paths[0].path
        self.pretty_filename = self.track.paths[0].pretty_path

        ext = os.path.splitext(os.path.basename(self.filename))[1].lower()[1:]

        artist = self.track.artist.name if self.track.artist is not None else ''
        album = self.track.album.name if self.track.album is not None else ''
        title = self.track.name
        track_number = self.track.number if self.track.number is not None else 0

        if self.skip_seconds is not None:
            skip_seconds_args = ['-ss', str(self.skip_seconds)]
        else:
            skip_seconds_args = []

        args = ([self.ffmpeg_cmd] +
                skip_seconds_args +
                self.ffmpeg_input_args +
                # reads input at native frame rate, e.g. very handy for streaming.
                ['-re'] +
                ['-i', self.filename] +
                # always produce stereo output
                ['-ac', '2'] +
                # strip any video streams
                ['-vn'] +
                self.ffmpeg_output_args + [
                    '-metadata', 'artist=%s' % artist,
                    '-metadata', 'album=%s' % album,
                    '-metadata', 'title=%s' % title,
                    '-metadata', 'tracknumber=%s' % track_number,
                    '-'])

        for index, arg in enumerate(args):
            if not isinstance(arg, bytes):
                arg = arg.encode('utf8')

            args[index] = arg.replace(b'EXT', ext)

        cherrypy.request.transcoding_track = self.track
        cherrypy.request.transcoding_transcoder = self

        try:
            self.process = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE, stdin=None)
        except Exception as e:
            self.error = 'Got "%s" when starting ffmpeg.' % str(e)
            return

        debug('transcoding with: %s' % b' '.join(args).decode('utf8', 'replace'))

        return self.transcode

    def __exit__(self, type, value, traceback):
        if self.process is not None:
            try:
                self.process.wait(10)
            except TimeoutExpired:
                self.stop()

        if self.process is not None and not self.stopped and self.process.returncode != 0:
            stderr_lines = self.stderr.decode('utf8', 'replace').split("\n")

            try:
                stderr_lines.remove("")
            except ValueError:
                pass

            self.error = stderr_lines[-1]

            log('ffmpeg returned non-zero status "%d" and "%s".' % (self.process.returncode, self.error))
        elif self.error is not None:
            log('Got exception "%s".' % (self.error))
        else:
            self.success = True

        cherrypy.engine.publish('transcoding.done', track=self.track)

        debug('"%s" transcoding done.' % self.track)

    def stop(self):
        if self.stopped:
            return

        if self.process is not None:
            try:
                self.process.send_signal(signal.SIGTERM)
                self.process.stdout.read()
                self.process.wait()
            except ProcessLookupError:
                pass

        self.stopped = True

    @staticmethod
    def set_nonblocking(fileno):
        fcntl.fcntl(
            fileno, fcntl.F_SETFL, fcntl.fcntl(fileno, fcntl.F_GETFL) | os.O_NONBLOCK,
        )

    def read_process(self):
        FFMPEGTranscoder.set_nonblocking(self.process.stderr.fileno())

        poll = select.poll()

        poll.register(self.process.stdout, select.POLLIN | select.POLLHUP)
        poll.register(self.process.stderr, select.POLLIN | select.POLLHUP)

        pollc = 2

        events = poll.poll()

        initial_bitrate = self.initial_bitrate()

        bitrate = initial_bitrate
        seconds = 0

        while pollc > 0 and len(events) > 0:
            info = data = None

            for event in events:
                rfd, event = event

                if event & select.POLLIN:
                    if rfd == self.process.stdout.fileno():
                        data = self.process.stdout.read(bitrate)

                    if rfd == self.process.stderr.fileno():
                        readx = select.select([self.process.stderr.fileno()], [], [])[0]

                        if readx:
                            chunk = self.process.stderr.read()

                            self.stderr = chunk

                            if len(chunk) > 0:
                                match = re.match(b'.*time=[ ]*(?P<time>[0-9.:]+).*bitrate=[ ]*(?P<bitrate>[0-9.]+)',
                                                 chunk)

                                if match:
                                    info = match.groupdict()

                if event & select.POLLHUP:
                    poll.unregister(rfd)
                    pollc = pollc - 1

                if pollc > 0:
                    events = poll.poll()

            # this is here for when we don't get the output from ffmpeg and we assume we've read 1 second
            # of data seeing as we should have an approximate bitrate at least.
            seconds += 1

            yield data, bitrate, seconds

            if info is not None:
                bitrate = int(float(info['bitrate'].decode()) * 1024 / 8)

                match = re.match(b'(?P<hours>[0-9]+):(?P<minutes>[0-9]+):(?P<seconds>[0-9.]+)', info['time'])

                if match:
                    time_info = match.groupdict()

                    if time_info is not None:
                        hours = time_info['hours'].decode()
                        minutes = time_info['minutes'].decode()
                        seconds = time_info['seconds'].decode()
                        seconds = float(hours) * 60 * 60 + float(minutes) * 60 + float(seconds)

    def transcode(self):
        # we wait this long before we start streaming to the client as a way to fool ffmpeg
        # to keep the stream a couple of seconds before the client. this way the client can
        # buffer more than it's regular wait-before-starting-playing-cache-filling thingie.
        # this hopefully fixes choppy play for certain tracks in certain players.
        #
        # it would be nice if we could do this with ffmpeg directly so we could skip this
        # artificial wait, but i don't know how.
        seconds_keepahead = 2

        time.sleep(seconds_keepahead)

        start_time = time.time()

        for data, bitrate, seconds in self.read_process():

            yield data

            wall_time = time.time() - start_time

            if self.skip_seconds is not None:
                seconds += self.skip_seconds
                wall_time += self.skip_seconds

            seconds_ahead = seconds - wall_time

            cherrypy.engine.publish('transcoding.progress', progress={
                'seconds': seconds,
                'bitrate': bitrate,
                'seconds_ahead': seconds_ahead
            }, transcoder=self, track=self.track)

            debug('"%s" transcoding at %d b/s, we\'re %.2fs ahead (total %ds).' %
                  (self.pretty_filename, bitrate, seconds_ahead, seconds))

    def initial_bitrate(self):
        """
        initial bitrate to use before we get one from ffmpeg.
        """
        raise NotImplementedError()

class CopyFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "copy", "-f", "EXT"]
    ffmpeg_input_args = []

    def outputs():
        return None

    def initial_bitrate(self):
        if self.track.bitrate is not None and self.track.bitrate != 0:
            return int(self.track.bitrate / 8)
        else:
            # this value is veeery arbitrary because we don't know the format
            # audio quality or nothing... but on the other hand all tracks should
            # have a bitrate
            return int(192000 / 8)

class Mp3FFMPEGTranscoder(FFMPEGTranscoder):
    # -aq 0 should be v0
    ffmpeg_output_args = ["-acodec", "libmp3lame", "-f", "mp3", "-aq", "0"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/mp3'

    def initial_bitrate(self):
        # lame v0's target bitrate is 245kbit/s (but is of course VBR)
        return int(245000 / 8)

class OggFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "libvorbis", "-f", "ogg", "-aq", "6"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/ogg'

    def initial_bitrate(self):
        # -aq 6 is about 192kbit/s
        return int(192000 / 8)

class Transcoding:
    players = [
        ['Music Player Daemon', ['audio/mp3', 'audio/ogg']],
        ['foobar2000', ['audio/mp3', 'audio/ogg']],
        ['Windows.*Chrome', ['audio/mp3', 'audio/ogg']],
        ['Linux.*Chrome', ['audio/mp3', 'audio/ogg']],
        ['VLC', ['audio/mp3', 'audio/ogg']],
        # This matches Nokia N9s default player
        ['GStreamer', ['audio/mp3', 'audio/ogg']]
    ]
    """
    if a player doesn't supply a usable Accept header we will look through this
    to figure out what formats it supports

     - we don't put audio/flac here, always transcode it to mp3 or ogg
     - we prioritze mp3 over ogg because we've had some lag-issues with ogg,
       at least when transcoding from flac to ogg...
     - we default to mp3
    """

    transcoders = [Mp3FFMPEGTranscoder, OggFFMPEGTranscoder]

    def determine_transcoder(self, track, user_agent, accepts):
        if not (len(accepts) == 0 or len(accepts) == 1 and accepts[0] == '*/*'):
            transcoder, format = self._determine_transcoder(track, accepts)
            if transcoder is not None:
                return transcoder, format

        for player, formats in self.players:
            if re.search(player, user_agent):
                transcoder, format = self._determine_transcoder(track, formats)

                if transcoder is not None:
                    return transcoder, format

                break

        if track.format == 'audio/mp3':
            return CopyFFMPEGTranscoder, 'audio/mp3'
        else:
            return Mp3FFMPEGTranscoder, 'audio/mp3'

    def _determine_transcoder(self, track, formats):
        if track.format in formats:
            return CopyFFMPEGTranscoder, track.format

        for format in formats:
            for transcoder in self.transcoders:
                if transcoder.outputs() == format:
                    return transcoder, format

        return None, None

    def transcode(self, tracks, transcoder=None):
        if transcoder is None:
            transcoder = CopyFFMPEGTranscoder

        for track, skip_seconds in tracks:
            if isinstance(transcoder, Transcoder):
                raise Exception("transcoder must be an instance of Transcoder")

            with transcoder(track, skip_seconds) as transcode:
                # error occured in __enter__
                if transcode is None:
                    return

                for data in transcode():
                    yield data

transcoding = Transcoding()
<commit_msg>kill ffmpeg when server is shut down<commit_after># Copyright 2012-2014 Mattias Fliesberg
#
# This file is part of opmuse.
#
# opmuse is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# opmuse is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with opmuse.  If not, see <http://www.gnu.org/licenses/>.

import os
import time
import subprocess
import re
import cherrypy
import fcntl
import select
import logging
import signal
from subprocess import TimeoutExpired

def debug(msg):
    cherrypy.log.error(msg, context='transcoding', severity=logging.DEBUG)

def log(msg):
    cherrypy.log(msg, context='transcoding')

class FFMPEGError(Exception):
    pass

class FFMPEGTranscoderSubprocessTool(cherrypy.Tool):
    """
    This tool makes sure the ffmpeg subprocess is ended
    properly when a request is cancelled
    """
    def __init__(self):
        cherrypy.Tool.__init__(self, 'on_end_request',
                               self.end, priority=20)

    def end(self):
        transcoding_transcoder = None

        if (hasattr(cherrypy.request, 'transcoding_transcoder') and
                cherrypy.request.transcoding_transcoder is not None):

            transcoding_transcoder = cherrypy.request.transcoding_transcoder

            cherrypy.request.transcoding_transcoder = None

            transcoding_transcoder.stop()

        if (hasattr(cherrypy.request, 'transcoding_track') and
                cherrypy.request.transcoding_track is not None):

            transcoding_track = cherrypy.request.transcoding_track

            cherrypy.request.transcoding_track = None

            cherrypy.engine.publish('transcoding.end',
                                    track=transcoding_track,
                                    transcoder=transcoding_transcoder)

            debug('"%s" transcoding ended.' % transcoding_track)

class Transcoder:
    def transcode(self):
        raise NotImplementedError()

    @staticmethod
    def outputs():
        raise NotImplementedError()

class FFMPEGTranscoder(Transcoder):
    def __init__(self, track, skip_seconds):
        self.track = track
        self.skip_seconds = skip_seconds
        self.stderr = None
        self.success = False
        self.error = None
        self.stopped = False
        self.process = None

        if cherrypy.request.app is not None:
            ffmpeg_cmd = cherrypy.request.app.config.get('opmuse').get('transcoding.ffmpeg_cmd')
        else:
            ffmpeg_cmd = None

        if ffmpeg_cmd is not None:
            self.ffmpeg_cmd = ffmpeg_cmd
        else:
            self.ffmpeg_cmd = 'ffmpeg'

    def __enter__(self):
        cherrypy.engine.publish('transcoding.start', transcoder=self, track=self.track)

        # http server has priority of 25, so specify 20 so this can
        # run before and stop this one which would otherwise cause
        # a "deadlock" with the http servers stopping.
        #
        # that's why we do this here, because otherwise close wont be called on
        # this generator and a sort of deadlock would occur.
        #
        # there's still one remaining issue here though and that is that
        # while the server is shutting down it still accepts new requests so
        # when this one is killed the client (i.e. browser) will just start
        # another request which will require another call to "stop" (e.g. SIGTERM).
        # dont know what to do about that one :/
        cherrypy.engine.subscribe('stop', self.stop, priority=20)

        self.filename = self.track.paths[0].path
        self.pretty_filename = self.track.paths[0].pretty_path

        ext = os.path.splitext(os.path.basename(self.filename))[1].lower()[1:]

        artist = self.track.artist.name if self.track.artist is not None else ''
        album = self.track.album.name if self.track.album is not None else ''
        title = self.track.name
        track_number = self.track.number if self.track.number is not None else 0

        if self.skip_seconds is not None:
            skip_seconds_args = ['-ss', str(self.skip_seconds)]
        else:
            skip_seconds_args = []

        args = ([self.ffmpeg_cmd] +
                skip_seconds_args +
                self.ffmpeg_input_args +
                # reads input at native frame rate, e.g. very handy for streaming.
                ['-re'] +
                ['-i', self.filename] +
                # always produce stereo output
                ['-ac', '2'] +
                # strip any video streams
                ['-vn'] +
                self.ffmpeg_output_args + [
                    '-metadata', 'artist=%s' % artist,
                    '-metadata', 'album=%s' % album,
                    '-metadata', 'title=%s' % title,
                    '-metadata', 'tracknumber=%s' % track_number,
                    '-'])

        for index, arg in enumerate(args):
            if not isinstance(arg, bytes):
                arg = arg.encode('utf8')

            args[index] = arg.replace(b'EXT', ext)

        cherrypy.request.transcoding_track = self.track
        cherrypy.request.transcoding_transcoder = self

        try:
            self.process = subprocess.Popen(args, shell=False, stdout=subprocess.PIPE,
                                            stderr=subprocess.PIPE, stdin=None)
        except Exception as e:
            self.error = 'Got "%s" when starting ffmpeg.' % str(e)
            return

        debug('transcoding with: %s' % b' '.join(args).decode('utf8', 'replace'))

        return self.transcode

    def __exit__(self, type, value, traceback):
        if self.process is not None:
            try:
                self.process.wait(10)
            except TimeoutExpired:
                self.stop()

        if self.process is not None and not self.stopped and self.process.returncode != 0:
            stderr_lines = self.stderr.decode('utf8', 'replace').split("\n")

            try:
                stderr_lines.remove("")
            except ValueError:
                pass

            self.error = stderr_lines[-1]

            log('ffmpeg returned non-zero status "%d" and "%s".' % (self.process.returncode, self.error))
        elif self.error is not None:
            log('Got exception "%s".' % (self.error))
        else:
            self.success = True

        cherrypy.engine.publish('transcoding.done', track=self.track)

        debug('"%s" transcoding done.' % self.track)

    def stop(self):
        if self.stopped:
            return

        try:
            if self.process is not None:
                debug("Stopping ffmpeg %d" % self.process.pid)

                self.process.send_signal(signal.SIGTERM)
                self.process.stdout.read()
                self.process.wait()
        except ProcessLookupError:
            pass
        finally:
            cherrypy.engine.unsubscribe('stop', self.stop)
            self.stopped = True

    @staticmethod
    def set_nonblocking(fileno):
        fcntl.fcntl(
            fileno, fcntl.F_SETFL, fcntl.fcntl(fileno, fcntl.F_GETFL) | os.O_NONBLOCK,
        )

    def read_process(self):
        FFMPEGTranscoder.set_nonblocking(self.process.stderr.fileno())

        poll = select.poll()

        poll.register(self.process.stdout, select.POLLIN | select.POLLHUP)
        poll.register(self.process.stderr, select.POLLIN | select.POLLHUP)

        pollc = 2

        events = poll.poll()

        initial_bitrate = self.initial_bitrate()

        bitrate = initial_bitrate
        seconds = 0

        while pollc > 0 and len(events) > 0:
            info = data = None

            for event in events:
                rfd, event = event

                if event & select.POLLIN:
                    if rfd == self.process.stdout.fileno():
                        data = self.process.stdout.read(bitrate)

                    if rfd == self.process.stderr.fileno():
                        readx = select.select([self.process.stderr.fileno()], [], [])[0]

                        if readx:
                            chunk = self.process.stderr.read()

                            self.stderr = chunk

                            if len(chunk) > 0:
                                match = re.match(b'.*time=[ ]*(?P<time>[0-9.:]+).*bitrate=[ ]*(?P<bitrate>[0-9.]+)',
                                                 chunk)

                                if match:
                                    info = match.groupdict()

                if event & select.POLLHUP:
                    poll.unregister(rfd)
                    pollc = pollc - 1

                if pollc > 0:
                    events = poll.poll()

            # this is here for when we don't get the output from ffmpeg and we assume we've read 1 second
            # of data seeing as we should have an approximate bitrate at least.
            seconds += 1

            yield data, bitrate, seconds

            if info is not None:
                bitrate = int(float(info['bitrate'].decode()) * 1024 / 8)

                match = re.match(b'(?P<hours>[0-9]+):(?P<minutes>[0-9]+):(?P<seconds>[0-9.]+)', info['time'])

                if match:
                    time_info = match.groupdict()

                    if time_info is not None:
                        hours = time_info['hours'].decode()
                        minutes = time_info['minutes'].decode()
                        seconds = time_info['seconds'].decode()
                        seconds = float(hours) * 60 * 60 + float(minutes) * 60 + float(seconds)

    def transcode(self):
        # we wait this long before we start streaming to the client as a way to fool ffmpeg
        # to keep the stream a couple of seconds before the client. this way the client can
        # buffer more than it's regular wait-before-starting-playing-cache-filling thingie.
        # this hopefully fixes choppy play for certain tracks in certain players.
        #
        # it would be nice if we could do this with ffmpeg directly so we could skip this
        # artificial wait, but i don't know how.
        seconds_keepahead = 2

        time.sleep(seconds_keepahead)

        start_time = time.time()

        for data, bitrate, seconds in self.read_process():

            yield data

            wall_time = time.time() - start_time

            if self.skip_seconds is not None:
                seconds += self.skip_seconds
                wall_time += self.skip_seconds

            seconds_ahead = seconds - wall_time

            cherrypy.engine.publish('transcoding.progress', progress={
                'seconds': seconds,
                'bitrate': bitrate,
                'seconds_ahead': seconds_ahead
            }, transcoder=self, track=self.track)

            debug('"%s" transcoding at %d b/s, we\'re %.2fs ahead (total %ds).' %
                  (self.pretty_filename, bitrate, seconds_ahead, seconds))

    def initial_bitrate(self):
        """
        initial bitrate to use before we get one from ffmpeg.
        """
        raise NotImplementedError()

class CopyFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "copy", "-f", "EXT"]
    ffmpeg_input_args = []

    def outputs():
        return None

    def initial_bitrate(self):
        if self.track.bitrate is not None and self.track.bitrate != 0:
            return int(self.track.bitrate / 8)
        else:
            # this value is veeery arbitrary because we don't know the format
            # audio quality or nothing... but on the other hand all tracks should
            # have a bitrate
            return int(192000 / 8)

class Mp3FFMPEGTranscoder(FFMPEGTranscoder):
    # -aq 0 should be v0
    ffmpeg_output_args = ["-acodec", "libmp3lame", "-f", "mp3", "-aq", "0"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/mp3'

    def initial_bitrate(self):
        # lame v0's target bitrate is 245kbit/s (but is of course VBR)
        return int(245000 / 8)

class OggFFMPEGTranscoder(FFMPEGTranscoder):
    ffmpeg_output_args = ["-acodec", "libvorbis", "-f", "ogg", "-aq", "6"]
    ffmpeg_input_args = []

    def outputs():
        return 'audio/ogg'

    def initial_bitrate(self):
        # -aq 6 is about 192kbit/s
        return int(192000 / 8)

class Transcoding:
    players = [
        ['Music Player Daemon', ['audio/mp3', 'audio/ogg']],
        ['foobar2000', ['audio/mp3', 'audio/ogg']],
        ['Windows.*Chrome', ['audio/mp3', 'audio/ogg']],
        ['Linux.*Chrome', ['audio/mp3', 'audio/ogg']],
        ['VLC', ['audio/mp3', 'audio/ogg']],
        # This matches Nokia N9s default player
        ['GStreamer', ['audio/mp3', 'audio/ogg']]
    ]
    """
    if a player doesn't supply a usable Accept header we will look through this
    to figure out what formats it supports

     - we don't put audio/flac here, always transcode it to mp3 or ogg
     - we prioritze mp3 over ogg because we've had some lag-issues with ogg,
       at least when transcoding from flac to ogg...
     - we default to mp3
    """

    transcoders = [Mp3FFMPEGTranscoder, OggFFMPEGTranscoder]

    def determine_transcoder(self, track, user_agent, accepts):
        if not (len(accepts) == 0 or len(accepts) == 1 and accepts[0] == '*/*'):
            transcoder, format = self._determine_transcoder(track, accepts)
            if transcoder is not None:
                return transcoder, format

        for player, formats in self.players:
            if re.search(player, user_agent):
                transcoder, format = self._determine_transcoder(track, formats)

                if transcoder is not None:
                    return transcoder, format

                break

        if track.format == 'audio/mp3':
            return CopyFFMPEGTranscoder, 'audio/mp3'
        else:
            return Mp3FFMPEGTranscoder, 'audio/mp3'

    def _determine_transcoder(self, track, formats):
        if track.format in formats:
            return CopyFFMPEGTranscoder, track.format

        for format in formats:
            for transcoder in self.transcoders:
                if transcoder.outputs() == format:
                    return transcoder, format

        return None, None

    def transcode(self, tracks, transcoder=None):
        if transcoder is None:
            transcoder = CopyFFMPEGTranscoder

        for track, skip_seconds in tracks:
            if isinstance(transcoder, Transcoder):
                raise Exception("transcoder must be an instance of Transcoder")

            with transcoder(track, skip_seconds) as transcode:
                # error occured in __enter__
                if transcode is None:
                    return

                for data in transcode():
                    yield data

transcoding = Transcoding()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import uuid
import os
from datetime import datetime

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone

from taggit.models import TaggedItemBase
from taggit.managers import TaggableManager

from opps.core.models import Publishable

def get_file_path(instance, filename):
    ext = filename.split('.')[-1]
    filename = "{0}-{1}.{2}".format(uuid.uuid4(), instance.slug, ext)
    d = datetime.now()
    folder = "images/{0}".format(d.strftime("%Y/%m/%d/"))
    return os.path.join(folder, filename)

class TaggedImage(TaggedItemBase):
    """Tag for images """
    content_object = models.ForeignKey('images.Image')
    pass

class Image(Publishable):

    title = models.CharField(_(u"Title"), max_length=140, db_index=True)
    slug = models.SlugField(_(u"Slug"), max_length=150, blank=True,
                            db_index=True)
    image = models.ImageField(upload_to=get_file_path)
    description = models.TextField(_(u"Description"), null=True, blank=True)
    tags = TaggableManager(blank=True, through=TaggedImage, verbose_name=u'Tags')

    source = models.ForeignKey('sources.Source', null=True, blank=True)

    class META:
        verbose_name = _('Image')
        verbose_name_plural = _('Images')

    def __unicode__(self):
        return self.slug

    def get_absolute_url(self):
        if self.date_available <= timezone.now() and self.published:
            return self.image.url
        return u""
<commit_msg>create class mdoel Cropping on Images app<commit_after># -*- coding: utf-8 -*-
import uuid
import os
from datetime import datetime

from django.db import models
from django.utils.translation import ugettext_lazy as _
from django.utils import timezone

from taggit.models import TaggedItemBase
from taggit.managers import TaggableManager

from opps.core.models import Publishable, Slugged

HALIGN_CHOICES = (
    ('left', _('Left')),
    ('center', _('Center')),
    ('right', _('Right'))
)
VALIGN_CHOICES = (
    ('top', _('Top')),
    ('middle', _('Middle')),
    ('bottom', _('Bottom'))
)

def get_file_path(instance, filename):
    ext = filename.split('.')[-1]
    filename = "{0}-{1}.{2}".format(uuid.uuid4(), instance.slug, ext)
    d = datetime.now()
    folder = "images/{0}".format(d.strftime("%Y/%m/%d/"))
    return os.path.join(folder, filename)

class TaggedImage(TaggedItemBase):
    """Tag for images """
    content_object = models.ForeignKey('images.Image')
    pass

class Cropping(models.Model):
    crop_example = models.CharField(_(u"Crop Example"), max_length=140,
                                   null=True, blank=True)
    flip = models.BooleanField(_(u'Flip'), default=False,
                               help_text=_(u'Flag that indicates that '
                                           u'thumbor should flip '
                                           u'horizontally (on the vertical '
                                           u'axis) the image'))
    flop = models.BooleanField(_(u'Flop'), default=False,
                               help_text=_(u'Flag that indicates that '
                                           u'thumbor should flip '
                                           u'vertically (on the horizontal '
                                           u'axis) the image'))
    halign = models.CharField(_(u'Horizontal Align'), default=False,
                              max_length=6,
                              null=True, blank=True,
                              choices=HALIGN_CHOICES,
                              help_text=_(u'Horizontal alignment that '
                                          u'thumbor should use for cropping'))
    valign = models.CharField(_(u'Vertical Align'), default=False,
                              max_length=6,
                              null=True, blank=True,
                              choices=VALIGN_CHOICES,
                              help_text=_(u'Vertical alignment that '
                                          u'thumbor should use for cropping'))

    fit_in = models.BooleanField(_(u'Fit in'), default=False,
                               help_text=_(u'flag that indicates that '
                                           u'thumbor should fit the image in '
                                           u'the box defined by width x '
                                           u'height'))

    smart = models.BooleanField(_(u'Smart'), default=False,
                                help_text=_(u'Flag that indicates that'
                                            u'thumbor should use smart '
                                            u'cropping;'))

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        self.crop_example = self.image.url
        super(Cropping, self).save(*args, **kwargs)

class Image(Publishable, Slugged, Cropping):

    title = models.CharField(_(u"Title"), max_length=140, db_index=True)
    image = models.ImageField(upload_to=get_file_path)
    description = models.TextField(_(u"Description"), null=True, blank=True)
    tags = TaggableManager(blank=True, through=TaggedImage, verbose_name=u'Tags')

    source = models.ForeignKey('sources.Source', null=True, blank=True)

    class META:
        verbose_name = _('Image')
        verbose_name_plural = _('Images')
        unique_together = ['site', 'slug']

    def __unicode__(self):
        return u"{}-{}".format(self.site, self.slug)

    def get_absolute_url(self):
        if self.date_available <= timezone.now() and self.published:
            return self.image.url
        return u""
<|endoftext|>


<commit_before>from django.db import models

class OrderableManager(models.Manager):
    '''
    Adds additional functionality to `Orderable.objects` allow access to the next and
    previous ordered object within the queryset.
    '''
    def before(self, orderable):
        return self.filter(sort_order__lt=orderable.sort_order).last()

    def after(self, orderable):
        return self.filter(sort_order__gt=orderable.sort_order).first()
<commit_msg>fix typo<commit_after>from django.db import models

class OrderableManager(models.Manager):
    '''
    Adds additional functionality to `Orderable.objects` providing access to the next and
    previous ordered object within the queryset.
    '''
    def before(self, orderable):
        return self.filter(sort_order__lt=orderable.sort_order).last()

    def after(self, orderable):
        return self.filter(sort_order__gt=orderable.sort_order).first()
<|endoftext|>


<commit_before>import os
import math
import logging
log = logging.getLogger(__file__)

def check_create_db(db_url, is_geospatial=False):
    """ generic check a database ... and create it if it doesn't exist
        @TODO: add other database supports beyond postgres
    """
    #import pdb; pdb.set_trace()
    ret_val = True
    if 'postgres' in db_url:
        db_name = database_name_from_url(db_url)
        db_user = username_from_url(db_url, def_val=os.getenv('USERNAME'))
        ret_val = postgres_check_create_db(db_name, db_user, is_geospatial)

    return ret_val

def make_url(db_url, def_val=None):
    """ wrapper around sqlalchemy's make_url
    """
    ret_val = def_val
    try:
        from sqlalchemy.engine.url import make_url
        url = make_url(db_url)
        log.debug("{} {} {} {} {}".format(url.username, url.password, url.host, url.port, url.database))
        ret_val = url
    except Exception as e:
        log.error("DB URL {} PARSE ERROR : {}".format(db_url, e))
    return ret_val

def database_name_from_url(db_url, def_val=None):
    ret_val = def_val
    u = make_url(db_url)
    if u and u.database:
        ret_val = u.database
    return ret_val

def username_from_url(db_url, def_val=None):
    ret_val = def_val
    u = make_url(db_url)
    if u and u.username:
        ret_val = u.username
    return ret_val

def db_conn(url):
    """  create a generic scoped session to a database as defined by the param url
         via sqlalchemy.  This routine meant as a quick way to grab a session / engine
         @param url: sqlite:///gtfs.db or postgresql+psycopg2://postgres@localhost:5432/transit
         @return: session, engine
    """
    from sqlalchemy.orm import scoped_session
    from sqlalchemy.orm import sessionmaker
    session = scoped_session(sessionmaker())

    # create the engine
    from sqlalchemy import create_engine
    engine = create_engine(url)
    session.configure(bind=engine)

    return session, engine

def db_args():
    """ create a generic database commandline arg PARSER """
    import argparse
    parser = argparse.ArgumentParser(prog='gtfs data loader', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--database_url', '-d',  default='sqlite:///gtfs.db', help='DATABASE URL with appropriate privileges')
    parser.add_argument('--is_geospatial', '-g', default=False, action='store_true', help='Database supports GEOSPATIAL functions')
    parser.add_argument('--schema','-s', default=None, help='Database SCHEMA name')
    parser.add_argument('--gtfs','-u', default="DATAd", help='URL or local path to GTFS(RT) data')
    return parser

def db_args_gtfsdb(just_engine=True):
    parser = db_args()
    args = parser.parse_args()
    session, engine = db_conn(args.database_url)
    if just_engine:
        return engine
    else:
        return session, engine

def db_gtfs_rt(just_engine=False):
    """ get a command line PARSER and db connection to query gtfsrdb data
        NOTE: meant as a quick dirty way to grab a connection for test apps
    """
    parser = db_args()
    parser.add_argument('--route',  '-r', default="12", help='what route?')
    parser.add_argument('--agency', '-a', default="TriMet", help='what agency?')
    args = parser.parse_args()

    from ott.data.gtfsrdb import model
    model.add_schema(args.schema)
    session, engine = db_conn(args.database_url)
    if just_engine:
        return engine
    else:
        return session, engine

def closest_stops_tuple_to_dict(tup):
    ret_val = {
        'stop_id'    : tup[1],
        'stop_name'  : tup[0],
        'stop_lat'   : tup[2],
        'stop_lon'   : tup[3],
        'dist_miles' : tup[4]
    }
    return ret_val

def add_math_to_sqllite(conn, conn_record):
    """ This method is called for each new SQLAlchemy database connection. I'm using it as a connection decorator to
        add math routines to a sqllite database

        @note: check out the call to (above): event.listen(self.engine, 'connect', Database.connection)
        @see:  http://docs.sqlalchemy.org/en/rel_0_8/core/events.html#sqlalchemy.events.PoolEvents
    """
    if 'sqlite' in type(conn).__module__:
        print 'in method that adds math to sqllite'
        conn.create_function("sin",     1, math.sin)
        conn.create_function("cos",     1, math.cos)
        conn.create_function("acos",    1, math.acos)
        conn.create_function("sqrt",    1, math.sqrt)
        conn.create_function("pow",     2, math.pow)
        conn.create_function("radians", 1, math.radians)

def decorate_engine(engine, method=add_math_to_sqllite):
    """ for
    """
    from sqlalchemy import event
    event.listen(engine, 'connect', method)

def sqlite_nearest_query(schema=None, table='stops'):
    """
        http://www.plumislandmedia.net/mysql/haversine-mysql-nearest-loc/

        SELECT stop_name, stop_code, stop_lat, stop_lon,
               (3963.17 * ACOS(COS(RADIANS(45.587))
                 * COS(RADIANS(stop_lat))
                 * COS(RADIANS(-122.593) - RADIANS(stop_lon))
                 + SIN(RADIANS(45.587))
                 * SIN(RADIANS(stop_lat))))
                 AS miles_away
        FROM ott.stops
        ORDER BY miles_away
        LIMIT 15
    """
    if schema:
        table = "{0}.{1}".format(schema, table)

    clause = ( "SELECT stop_code, stop_name, stop_lat, stop_lon, "
               "(3963.17 * ACOS(COS(RADIANS(:lat))  "
               " * COS(RADIANS(stop_lat))  "
               " * COS(RADIANS(:lon) - RADIANS(stop_lon)) "
               " + SIN(RADIANS(:lat)) "
               " * SIN(RADIANS(stop_lat)))) "
               " AS dist_miles "
               " FROM {0} "
               " ORDER BY dist_miles "
    ).format(table)
    return clause

def sqlite_query_nearest_stops(conn, lon, lat, count, schema, table):
    """ execute the query...
    """
    if count is None or count > 250:
        count = 10
    clause  = sqlite_nearest_query(schema, table)
    ret_val = conn.execute(clause, {'lon':lon, 'lat':lat}).fetchmany(count)
    return ret_val

def postgres_check_db(db_name, db_user, db_table=None):
    """
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect

        # step 1: check the database connection
        con = connect(user=db_user, dbname=db_name)

        # step 2: (optionally) check a value from a given database
        if db_table:
            cur = con.cursor()
            cur.execute('SELECT 1 from {}'.format(db_table))
            item = cur.fetchone()
            log.debug("POSTGRES item {} from table {} of db {}".format(item, db_table, db_name))
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_add_postgis(db_name, db_user):
    """ add postgis extension on a named database
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect
        from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

        con = connect(user=db_user, dbname=db_name)
        con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
        cur = con.cursor()
        cur.execute('CREATE EXTENSION postgis')
        cur.close()
        con.close()
        con = None
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_create_db(db_name, db_user):
    """ create a postgres db...and also maybe add the postgis extension
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect
        from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

        # step 1: connect to global postgres database and then create new db
        con = connect(user=db_user, dbname='postgres')
        con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
        cur = con.cursor()
        cur.execute('CREATE DATABASE {}'.format(db_name))
        cur.close()
        con.close()
        con = None
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_check_create_db(db_name, db_user, is_geospatial=False):
    """
    """
    ret_val = True
    try:
        exists = postgres_check_db(db_name, db_user)
        if not exists:
            ret_val = postgres_create_db(db_name, db_user)
            if ret_val and is_geospatial:
                ret_val = postgres_add_postgis(db_name, db_user)
    except Exception as e:
        log.error("POSTGRES CREATE DATABASE ERROR : {}".format(e))
        ret_val = False

    return ret_val

def postgres_drop_schema(db_url, db_user, move=False):
    """ drop / rename
        TODO: more work needed here
    """
    ret_val = True
    try:
        if move:
            ret_val = postgres_add_postgis(db_name, db_user)
        else:
            pass
    except Exception as e:
        log.error("POSTGRES DROP (RENAME) DATABASE ERROR : {}".format(e))
        ret_val = False

    return ret_val

def postgres_add_shp_file(db_url, db_user, is_geospatial=False):
    """
        @see http://geospatialpython.com/2016/08/pure-python-loading-shapefiles-into.html
    """
    ret_val = True
<commit_msg>fix up the gtfsdb stuff<commit_after>import os
import math
import logging
log = logging.getLogger(__file__)

def check_create_db(db_url, is_geospatial=False):
    """ generic check a database ... and create it if it doesn't exist
        @TODO: add other database supports beyond postgres
    """
    #import pdb; pdb.set_trace()
    ret_val = True
    if 'postgres' in db_url:
        db_name = database_name_from_url(db_url)
        db_user = username_from_url(db_url, def_val=os.getenv('USERNAME'))
        ret_val = postgres_check_create_db(db_name, db_user, is_geospatial)

    return ret_val

def make_url(db_url, def_val=None):
    """ wrapper around sqlalchemy's make_url
    """
    ret_val = def_val
    try:
        from sqlalchemy.engine.url import make_url
        url = make_url(db_url)
        log.debug("{} {} {} {} {}".format(url.username, url.password, url.host, url.port, url.database))
        ret_val = url
    except Exception as e:
        log.error("DB URL {} PARSE ERROR : {}".format(db_url, e))
    return ret_val

def database_name_from_url(db_url, def_val=None):
    ret_val = def_val
    u = make_url(db_url)
    if u and u.database:
        ret_val = u.database
    return ret_val

def username_from_url(db_url, def_val=None):
    ret_val = def_val
    u = make_url(db_url)
    if u and u.username:
        ret_val = u.username
    return ret_val

def db_conn(url):
    """  create a generic scoped session to a database as defined by the param url
         via sqlalchemy.  This routine meant as a quick way to grab a session / engine
         @param url: sqlite:///gtfs.db or postgresql+psycopg2://postgres@localhost:5432/transit
         @return: session, engine
    """
    from sqlalchemy.orm import scoped_session
    from sqlalchemy.orm import sessionmaker
    session = scoped_session(sessionmaker())

    # create the engine
    from sqlalchemy import create_engine
    engine = create_engine(url)
    session.configure(bind=engine)

    return session, engine

def gtfsdb_conn(kwargs):
    from gtfsdb import Database
    db = Database(**kwargs)
    return db

def db_args():
    """ create a generic database commandline arg PARSER """
    import argparse
    parser = argparse.ArgumentParser(prog='gtfs data loader', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('--database_url', '-d',  default='sqlite:///gtfs.db', help='DATABASE URL with appropriate privileges')
    parser.add_argument('--is_geospatial', '-g', default=False, action='store_true', help='Database supports GEOSPATIAL functions')
    parser.add_argument('--schema','-s', default=None, help='Database SCHEMA name')
    parser.add_argument('--gtfs','-u', default="DATAd", help='URL or local path to GTFS(RT) data')
    return parser

def db_args_gtfsdb():
    parser = db_args()
    args = parser.parse_args()
    kwargs = dict(
        url=args.database_url,
        is_geospatial=args.is_geospatial,
        schema=args.schema
    )

    return gtfsdb_conn(kwargs)

def db_gtfs_rt():
    """
    get a command line PARSER and db connection to query gtfsrdb data
    :requires ott.data project:
    NOTE: meant as a quick dirty way to grab a connection for test apps
    """
    parser = db_args()
    parser.add_argument('--route',  '-r', default="12", help='what route?')
    parser.add_argument('--agency', '-a', default="TriMet", help='what agency?')
    args = parser.parse_args()

    from ott.data.gtfsrdb import model
    model.add_schema(args.schema)
    session, engine = db_conn(args.database_url)
    return session, engine

def closest_stops_tuple_to_dict(tup):
    ret_val = {
        'stop_id'    : tup[1],
        'stop_name'  : tup[0],
        'stop_lat'   : tup[2],
        'stop_lon'   : tup[3],
        'dist_miles' : tup[4]
    }
    return ret_val

def add_math_to_sqllite(conn, conn_record):
    """ This method is called for each new SQLAlchemy database connection. I'm using it as a connection decorator to
        add math routines to a sqllite database

        @note: check out the call to (above): event.listen(self.engine, 'connect', Database.connection)
        @see:  http://docs.sqlalchemy.org/en/rel_0_8/core/events.html#sqlalchemy.events.PoolEvents
    """
    if 'sqlite' in type(conn).__module__:
        print 'in method that adds math to sqllite'
        conn.create_function("sin",     1, math.sin)
        conn.create_function("cos",     1, math.cos)
        conn.create_function("acos",    1, math.acos)
        conn.create_function("sqrt",    1, math.sqrt)
        conn.create_function("pow",     2, math.pow)
        conn.create_function("radians", 1, math.radians)

def decorate_engine(engine, method=add_math_to_sqllite):
    """ for
    """
    from sqlalchemy import event
    event.listen(engine, 'connect', method)

def sqlite_nearest_query(schema=None, table='stops'):
    """
        http://www.plumislandmedia.net/mysql/haversine-mysql-nearest-loc/

        SELECT stop_name, stop_code, stop_lat, stop_lon,
               (3963.17 * ACOS(COS(RADIANS(45.587))
                 * COS(RADIANS(stop_lat))
                 * COS(RADIANS(-122.593) - RADIANS(stop_lon))
                 + SIN(RADIANS(45.587))
                 * SIN(RADIANS(stop_lat))))
                 AS miles_away
        FROM ott.stops
        ORDER BY miles_away
        LIMIT 15
    """
    if schema:
        table = "{0}.{1}".format(schema, table)

    clause = ( "SELECT stop_code, stop_name, stop_lat, stop_lon, "
               "(3963.17 * ACOS(COS(RADIANS(:lat))  "
               " * COS(RADIANS(stop_lat))  "
               " * COS(RADIANS(:lon) - RADIANS(stop_lon)) "
               " + SIN(RADIANS(:lat)) "
               " * SIN(RADIANS(stop_lat)))) "
               " AS dist_miles "
               " FROM {0} "
               " ORDER BY dist_miles "
    ).format(table)
    return clause

def sqlite_query_nearest_stops(conn, lon, lat, count, schema, table):
    """ execute the query...
    """
    if count is None or count > 250:
        count = 10
    clause  = sqlite_nearest_query(schema, table)
    ret_val = conn.execute(clause, {'lon':lon, 'lat':lat}).fetchmany(count)
    return ret_val

def postgres_check_db(db_name, db_user, db_table=None):
    """
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect

        # step 1: check the database connection
        con = connect(user=db_user, dbname=db_name)

        # step 2: (optionally) check a value from a given database
        if db_table:
            cur = con.cursor()
            cur.execute('SELECT 1 from {}'.format(db_table))
            item = cur.fetchone()
            log.debug("POSTGRES item {} from table {} of db {}".format(item, db_table, db_name))
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_add_postgis(db_name, db_user):
    """ add postgis extension on a named database
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect
        from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

        con = connect(user=db_user, dbname=db_name)
        con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
        cur = con.cursor()
        cur.execute('CREATE EXTENSION postgis')
        cur.close()
        con.close()
        con = None
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_create_db(db_name, db_user):
    """ create a postgres db...and also maybe add the postgis extension
    """
    ret_val = True
    con = None
    try:
        from psycopg2 import connect
        from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT

        # step 1: connect to global postgres database and then create new db
        con = connect(user=db_user, dbname='postgres')
        con.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)
        cur = con.cursor()
        cur.execute('CREATE DATABASE {}'.format(db_name))
        cur.close()
        con.close()
        con = None
    except Exception as e:
        log.error("POSTGRES DATABASE ERROR : {}".format(e))
        ret_val = False
    finally:
        if con:
            con.close()
    return ret_val

def postgres_check_create_db(db_name, db_user, is_geospatial=False):
    """
    """
    ret_val = True
    try:
        exists = postgres_check_db(db_name, db_user)
        if not exists:
            ret_val = postgres_create_db(db_name, db_user)
            if ret_val and is_geospatial:
                ret_val = postgres_add_postgis(db_name, db_user)
    except Exception as e:
        log.error("POSTGRES CREATE DATABASE ERROR : {}".format(e))
        ret_val = False

    return ret_val

def postgres_drop_schema(db_url, db_user, move=False):
    """ drop / rename
        TODO: more work needed here
    """
    ret_val = True
    try:
        if move:
            ret_val = postgres_add_postgis(db_name, db_user)
        else:
            pass
    except Exception as e:
        log.error("POSTGRES DROP (RENAME) DATABASE ERROR : {}".format(e))
        ret_val = False

    return ret_val

def postgres_add_shp_file(db_url, db_user, is_geospatial=False):
    """
        @see http://geospatialpython.com/2016/08/pure-python-loading-shapefiles-into.html
    """
    ret_val = True
<|endoftext|>


<commit_before>import sys

if sys.version < 3:
    from overrides import overrides
    from overrides import __VERSION__
else:
    from overrides.overrides import overrides
    from overrides.overrides import __VERSION__

<commit_msg>allow import in python 2<commit_after>import sys

if sys.version < '3':
    from overrides import overrides
    from overrides import __VERSION__
else:
    from overrides.overrides import overrides
    from overrides.overrides import __VERSION__

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
File:    signals.py
Author:  Patrick Chasco
Created: July 26, 2005

Purpose: A signals implementation
"""

#========================================================
# Implementation
#========================================================
import weakref
from threading import Lock
import inspect

class Signal:
    """
    class Signal

    A simple implementation of the Signal/Slot pattern. To use, simply
    create a Signal instance. The instance may be a member of a class,
    a global, or a local; it makes no difference what scope it resides
    within. Connect slots to the signal using the "connect()" method.
    The slot may be a member of a class or a simple function. If the
    slot is a member of a class, Signal will automatically detect when
    the method's class instance has been deleted and remove it from
    its list of connected slots.
    """
    def __init__(self):
        self.slots = []

        # for keeping references to _WeakMethod_FuncHost objects.
        # If we didn't, then the weak references would die for
        # non-method slots that we've created.
        self.funchost = []

    def __call__(self, *args, **kwargs):
        for i in range(len(self.slots)):
            slot = self.slots[i]
            if slot != None:
                slot(*args, **kwargs)
            else:
                del self.slots[i]

    def call(self, *args, **kwargs):
        self.__call__(*args, **kwargs)

    def connect(self, slot):
        self.disconnect(slot)
        if inspect.ismethod(slot):
            self.slots.append(WeakMethod(slot))
        else:
            o = _WeakMethod_FuncHost(slot)
            self.slots.append(WeakMethod(o.func))
            # we stick a copy in here just to keep the instance alive
            self.funchost.append(o)

    def disconnect(self, slot):
        try:
            for i in range(len(self.slots)):
                wm = self.slots[i]
                if inspect.ismethod(slot):
                    if wm.f == slot.im_func and wm.c() == slot.im_self:
                        del self.slots[i]
                        return
                else:
                    if wm.c().hostedFunction == slot:
                        del self.slots[i]
                        return
        except:
            pass

    def disconnectAll(self):
        del self.slots
        del self.funchost
        self.slots = []
        self.funchost = []

class _WeakMethod_FuncHost:
    def __init__(self, func):
        self.hostedFunction = func
    def func(self, *args, **kwargs):
        self.hostedFunction(*args, **kwargs)

# this class was generously donated by a poster on ASPN (aspn.activestate.com)
class WeakMethod:
	def __init__(self, f):
            self.f = f.im_func
            self.c = weakref.ref(f.im_self)
	def __call__(self, *args, **kwargs):
            if self.c() == None : return
            self.f(self.c(), *args, **kwargs)

#========================================================
# Example usage
#========================================================
if __name__ == "__main__":
    class Button:
        def __init__(self):
            # Creating a signal as a member of a class
            self.sigClick = Signal()

    class Listener:
        # a sample method that will be connected to the signal
        def onClick(self):
            print "onClick ", repr(self)

    # a sample function to connect to the signal
    def listenFunction():
        print "listenFunction"

    # a function that accepts arguments
    def listenWithArgs(text):
        print "listenWithArgs: ", text

    b = Button()
    l = Listener()

    # Demonstrating connecting and calling signals
    print
    print "should see one message"
    b.sigClick.connect(l.onClick)
    b.sigClick()

    # Disconnecting all signals
    print
    print "should see no messages"
    b.sigClick.disconnectAll()
    b.sigClick()

    # connecting multiple functions to a signal
    print
    print "should see two messages"
    l2 = Listener()
    b.sigClick.connect(l.onClick)
    b.sigClick.connect(l2.onClick)
    b.sigClick()

    # disconnecting individual functions
    print
    print "should see two messages"
    b.sigClick.disconnect(l.onClick)
    b.sigClick.connect(listenFunction)
    b.sigClick()

    # signals disconnecting automatically
    print
    print "should see one message"
    b.sigClick.disconnectAll()
    b.sigClick.connect(l.onClick)
    b.sigClick.connect(l2.onClick)
    del l2
    b.sigClick()

    # example with arguments and a local signal
    print
    print "should see one message"
    sig = Signal()
    sig.connect(listenWithArgs)
    sig("Hello, World!")
<commit_msg>improve code layout<commit_after>#!/usr/bin/env python

"""
File:    signals.py
Author:  Patrick Chasco
Created: July 26, 2005

Purpose: A signals implementation
"""

#--------------------------------------------------------------------------------------
import weakref
import threading
import inspect

#--------------------------------------------------------------------------------------
class _WeakMethod_FuncHost:
    def __init__(self, func):
        self.hostedFunction = func
    def func(self, *args, **kwargs):
        self.hostedFunction(*args, **kwargs)

#--------------------------------------------------------------------------------------
# This class was generously donated by a poster on ASPN (aspn.activestate.com)
class WeakMethod:
	def __init__(self, f):
            self.f = f.im_func
            self.c = weakref.ref(f.im_self)
	def __call__(self, *args, **kwargs):
            if self.c() == None : return
            self.f(self.c(), *args, **kwargs)

#--------------------------------------------------------------------------------------
class Signal:
    """
    class Signal

    A simple implementation of the Signal/Slot pattern. To use, simply
    create a Signal instance. The instance may be a member of a class,
    a global, or a local; it makes no difference what scope it resides
    within. Connect slots to the signal using the "connect()" method.
    The slot may be a member of a class or a simple function. If the
    slot is a member of a class, Signal will automatically detect when
    the method's class instance has been deleted and remove it from
    its list of connected slots.
    """
    #--------------------------------------------------------------------------------------
    def __init__( self, threadsafe = True ):
        self.slots = []
        pass

    #--------------------------------------------------------------------------------------
    def __call__(self, *args, **kwargs):
        for i in range(len(self.slots)):
            slot = self.slots[i]
            if slot != None:
                slot(*args, **kwargs)
            else:
                del self.slots[i]
                pass
            pass
        pass

    #--------------------------------------------------------------------------------------
    def connect(self, slot):
        self.disconnect(slot)
        if inspect.ismethod(slot):
            self.slots.append(WeakMethod(slot))
        else:
            self.slots.append(slot)
            pass
        pass

    #--------------------------------------------------------------------------------------
    def disconnect(self, slot):
        try:
            for i in range(len(self.slots)):
                wm = self.slots[i]
                if inspect.ismethod(slot):
                    if wm.f == slot.im_func and wm.c() == slot.im_self:
                        del self.slots[i]
                        return
                    pass
                else:
                    if wm == slot:
                        del self.slots[i]
                        return
                    pass
                pass
            pass
        except:
            pass
        pass

    #--------------------------------------------------------------------------------------
    def disconnectAll(self):
        del self.slots[ : ]
        pass

    #--------------------------------------------------------------------------------------
    pass

#--------------------------------------------------------------------------------------
if __name__ == "__main__":
    class Button:
        def __init__(self):
            # Creating a signal as a member of a class
            self.sigClick = Signal()

    class Listener:
        # a sample method that will be connected to the signal
        def onClick(self):
            print "onClick ", repr(self)

    # a sample function to connect to the signal
    def listenFunction():
        print "listenFunction"

    # a function that accepts arguments
    def listenWithArgs(text):
        print "listenWithArgs: ", text

    b = Button()
    l = Listener()

    # Demonstrating connecting and calling signals
    print
    print "should see one message"
    b.sigClick.connect(l.onClick)
    b.sigClick()

    # Disconnecting all signals
    print
    print "should see no messages"
    b.sigClick.disconnectAll()
    b.sigClick()

    # connecting multiple functions to a signal
    print
    print "should see two messages"
    l2 = Listener()
    b.sigClick.connect(l.onClick)
    b.sigClick.connect(l2.onClick)
    b.sigClick()

    # disconnecting individual functions
    print
    print "should see two messages"
    b.sigClick.disconnect(l.onClick)
    b.sigClick.connect(listenFunction)
    b.sigClick()

    # signals disconnecting automatically
    print
    print "should see one message"
    b.sigClick.disconnectAll()
    b.sigClick.connect(l.onClick)
    b.sigClick.connect(l2.onClick)
    del l2
    b.sigClick()

    # example with arguments and a local signal
    print
    print "should see one message"
    sig = Signal()
    sig.connect(listenWithArgs)
    sig("Hello, World!")

#--------------------------------------------------------------------------------------
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Copyright (c) 2006-2012 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import re

from lib.core.common import extractRegexResult
from lib.core.common import getFilteredPageContent
from lib.core.common import listToStrValue
from lib.core.common import removeDynamicContent
from lib.core.common import wasLastRequestDBMSError
from lib.core.common import wasLastRequestHTTPError
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.exception import sqlmapNoneDataException
from lib.core.settings import DEFAULT_PAGE_ENCODING
from lib.core.settings import DIFF_TOLERANCE
from lib.core.settings import HTML_TITLE_REGEX
from lib.core.settings import MIN_RATIO
from lib.core.settings import MAX_RATIO
from lib.core.settings import LOWER_RATIO_BOUND
from lib.core.settings import UPPER_RATIO_BOUND
from lib.core.threads import getCurrentThreadData

def comparison(page, headers, code=None, getRatioValue=False, pageLength=None):
    _ = _adjust(_comparison(page, headers, code, getRatioValue, pageLength), getRatioValue)
    return _

def _adjust(condition, getRatioValue):
    if not any([conf.string, conf.regexp, conf.code]):
        # Negative logic approach is used in raw page comparison scheme as that what is "different" than original
        # PAYLOAD.WHERE.NEGATIVE response is considered as True; in switch based approach negative logic is not
        # applied as that what is by user considered as True is that what is returned by the comparison mechanism
        # itself
        retVal = not condition if kb.negativeLogic and condition is not None else condition
    else:
        retVal = condition if not getRatioValue else (MAX_RATIO if condition else MIN_RATIO)

    return retVal

def _comparison(page, headers, code, getRatioValue, pageLength):
    threadData = getCurrentThreadData()

    if kb.testMode:
        threadData.lastComparisonPage = page

    if page is None and pageLength is None:
        return None

    seqMatcher = threadData.seqMatcher
    seqMatcher.set_seq1(kb.pageTemplate)

    if any([conf.string, conf.regexp]):
        rawResponse = "%s%s" % (listToStrValue(headers.headers if headers else ""), page)

        # String to match in page when the query is valid
        if conf.string:
            return conf.string in rawResponse

        # Regular expression to match in page when the query is valid
        if conf.regexp:
            return re.search(conf.regexp, rawResponse, re.I | re.M) is not None

    # HTTP code to match when the query is valid
    if isinstance(code, int) and conf.code:
        return conf.code == code

    if page:
        # In case of an DBMS error page return None
        if kb.errorIsNone and (wasLastRequestDBMSError() or wasLastRequestHTTPError()):
            return None

        # Dynamic content lines to be excluded before comparison
        if not kb.nullConnection:
            page = removeDynamicContent(page)
            seqMatcher.set_seq1(removeDynamicContent(kb.pageTemplate))

        if not pageLength:
            pageLength = len(page)

    if kb.nullConnection and pageLength:
        if not seqMatcher.a:
            errMsg = "problem occured while retrieving original page content "
            errMsg += "which prevents sqlmap from continuation. Please rerun, "
            errMsg += "and if the problem persists turn off any optimization switches"
            raise sqlmapNoneDataException, errMsg

        ratio = 1. * pageLength / len(seqMatcher.a)

        if ratio > 1.:
            ratio = 1. / ratio
    else:
        # Preventing "Unicode equal comparison failed to convert both arguments to Unicode"
        # (e.g. if one page is PDF and the other is HTML)
        if isinstance(seqMatcher.a, str) and isinstance(page, unicode):
            page = page.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')
        elif isinstance(seqMatcher.a, unicode) and isinstance(page, str):
            seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')

        seq1, seq2 = None, None

        if conf.titles:
            seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a)
            seq2 = extractRegexResult(HTML_TITLE_REGEX, page)
        else:
            seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a
            seq2 = getFilteredPageContent(page, True) if conf.textOnly else page

        if seq1 is not None:
            seqMatcher.set_seq1(seq1)

        if seq2 is not None:
            seqMatcher.set_seq2(seq2)

        if seq1 is None or seq2 is None:
            return None
        else:
            ratio = round(seqMatcher.quick_ratio(), 3)

    # If the url is stable and we did not set yet the match ratio and the
    # current injected value changes the url page content
    if kb.matchRatio is None:
        if ratio >= LOWER_RATIO_BOUND and ratio <= UPPER_RATIO_BOUND:
            kb.matchRatio = ratio
            logger.debug("setting match ratio for current parameter to %.3f" % kb.matchRatio)

    # If it has been requested to return the ratio and not a comparison
    # response
    if getRatioValue:
        return ratio

    elif ratio > UPPER_RATIO_BOUND:
        return True

    elif kb.matchRatio is None:
        return None

    else:
        return (ratio - kb.matchRatio) > DIFF_TOLERANCE
<commit_msg>Minor fix related to Issue #94<commit_after>#!/usr/bin/env python

"""
Copyright (c) 2006-2012 sqlmap developers (http://sqlmap.org/)
See the file 'doc/COPYING' for copying permission
"""

import re

from lib.core.common import extractRegexResult
from lib.core.common import getFilteredPageContent
from lib.core.common import listToStrValue
from lib.core.common import removeDynamicContent
from lib.core.common import wasLastRequestDBMSError
from lib.core.common import wasLastRequestHTTPError
from lib.core.data import conf
from lib.core.data import kb
from lib.core.data import logger
from lib.core.exception import sqlmapNoneDataException
from lib.core.settings import DEFAULT_PAGE_ENCODING
from lib.core.settings import DIFF_TOLERANCE
from lib.core.settings import HTML_TITLE_REGEX
from lib.core.settings import MIN_RATIO
from lib.core.settings import MAX_RATIO
from lib.core.settings import LOWER_RATIO_BOUND
from lib.core.settings import UPPER_RATIO_BOUND
from lib.core.threads import getCurrentThreadData

def comparison(page, headers, code=None, getRatioValue=False, pageLength=None):
    _ = _adjust(_comparison(page, headers, code, getRatioValue, pageLength), getRatioValue)
    return _

def _adjust(condition, getRatioValue):
    if not any([conf.string, conf.regexp, conf.code]):
        # Negative logic approach is used in raw page comparison scheme as that what is "different" than original
        # PAYLOAD.WHERE.NEGATIVE response is considered as True; in switch based approach negative logic is not
        # applied as that what is by user considered as True is that what is returned by the comparison mechanism
        # itself
        retVal = not condition if kb.negativeLogic and condition is not None and not getRatioValue else condition
    else:
        retVal = condition if not getRatioValue else (MAX_RATIO if condition else MIN_RATIO)

    return retVal

def _comparison(page, headers, code, getRatioValue, pageLength):
    threadData = getCurrentThreadData()

    if kb.testMode:
        threadData.lastComparisonPage = page

    if page is None and pageLength is None:
        return None

    seqMatcher = threadData.seqMatcher
    seqMatcher.set_seq1(kb.pageTemplate)

    if any([conf.string, conf.regexp]):
        rawResponse = "%s%s" % (listToStrValue(headers.headers if headers else ""), page)

        # String to match in page when the query is valid
        if conf.string:
            return conf.string in rawResponse

        # Regular expression to match in page when the query is valid
        if conf.regexp:
            return re.search(conf.regexp, rawResponse, re.I | re.M) is not None

    # HTTP code to match when the query is valid
    if isinstance(code, int) and conf.code:
        return conf.code == code

    if page:
        # In case of an DBMS error page return None
        if kb.errorIsNone and (wasLastRequestDBMSError() or wasLastRequestHTTPError()):
            return None

        # Dynamic content lines to be excluded before comparison
        if not kb.nullConnection:
            page = removeDynamicContent(page)
            seqMatcher.set_seq1(removeDynamicContent(kb.pageTemplate))

        if not pageLength:
            pageLength = len(page)

    if kb.nullConnection and pageLength:
        if not seqMatcher.a:
            errMsg = "problem occured while retrieving original page content "
            errMsg += "which prevents sqlmap from continuation. Please rerun, "
            errMsg += "and if the problem persists turn off any optimization switches"
            raise sqlmapNoneDataException, errMsg

        ratio = 1. * pageLength / len(seqMatcher.a)

        if ratio > 1.:
            ratio = 1. / ratio
    else:
        # Preventing "Unicode equal comparison failed to convert both arguments to Unicode"
        # (e.g. if one page is PDF and the other is HTML)
        if isinstance(seqMatcher.a, str) and isinstance(page, unicode):
            page = page.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')
        elif isinstance(seqMatcher.a, unicode) and isinstance(page, str):
            seqMatcher.a = seqMatcher.a.encode(kb.pageEncoding or DEFAULT_PAGE_ENCODING, 'ignore')

        seq1, seq2 = None, None

        if conf.titles:
            seq1 = extractRegexResult(HTML_TITLE_REGEX, seqMatcher.a)
            seq2 = extractRegexResult(HTML_TITLE_REGEX, page)
        else:
            seq1 = getFilteredPageContent(seqMatcher.a, True) if conf.textOnly else seqMatcher.a
            seq2 = getFilteredPageContent(page, True) if conf.textOnly else page

        if seq1 is not None:
            seqMatcher.set_seq1(seq1)

        if seq2 is not None:
            seqMatcher.set_seq2(seq2)

        if seq1 is None or seq2 is None:
            return None
        else:
            ratio = round(seqMatcher.quick_ratio(), 3)

    # If the url is stable and we did not set yet the match ratio and the
    # current injected value changes the url page content
    if kb.matchRatio is None:
        if ratio >= LOWER_RATIO_BOUND and ratio <= UPPER_RATIO_BOUND:
            kb.matchRatio = ratio
            logger.debug("setting match ratio for current parameter to %.3f" % kb.matchRatio)

    # If it has been requested to return the ratio and not a comparison
    # response
    if getRatioValue:
        return ratio

    elif ratio > UPPER_RATIO_BOUND:
        return True

    elif kb.matchRatio is None:
        return None

    else:
        return (ratio - kb.matchRatio) > DIFF_TOLERANCE
<|endoftext|>


<commit_before>class GtkSharpPackage (Package):
	def __init__ (self):
		Package.__init__ (self, 'gtk-sharp', '2-12-branch')
		self.commit = 'b078aacaf263af84605812d778b62afbdf3e1b59'
		self.source_dir_name = 'mono-gtk-sharp-%s' % self.commit[:7]
		self.configure = './bootstrap-2.12 --prefix="%{prefix}"'
		self.sources = [
			'http://github.com/mono/gtk-sharp/tarball/%{commit}'
		]

GtkSharpPackage ()
<commit_msg>[gtksharp] Include container leak fix<commit_after>class GtkSharpPackage (Package):
	def __init__ (self):
		Package.__init__ (self, 'gtk-sharp', '2-12-branch')
		self.commit = 'b078aacaf263af84605812d778b62afbdf3e1b59'
		self.source_dir_name = 'mono-gtk-sharp-%s' % self.commit[:7]
		self.configure = './bootstrap-2.12 --prefix="%{prefix}"'
		self.sources = [
			'http://github.com/mono/gtk-sharp/tarball/%{commit}',

			#container leak fix
			'https://github.com/mono/gtk-sharp/pull/50.patch'
		]

	def prep (self):
		Package.prep (self)
		for p in range (1, len (self.sources)):
			self.sh ('patch -p1 < "%{sources[' + str (p) + ']}"')

GtkSharpPackage ()
<|endoftext|>


<commit_before>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
from op_test import OpTest
import paddle.fluid.core as core
import paddle.fluid as fluid
from paddle.fluid import compiler, Program, program_guard

def stable_softmax(x):
    """Compute the softmax of vector x in a numerically stable way."""
    # clip to shiftx, otherwise, when calc loss with
    # log(exp(shiftx)), may get log(0)=INF
    shiftx = (x - np.max(x)).clip(-64.)
    exps = np.exp(shiftx)
    return exps / np.sum(exps)

class TestSoftmaxOp(OpTest):
    def get_x_shape(self):
        return [10, 10]

    def get_axis(self):
        return -1

    def setUp(self):
        self.op_type = "softmax"
        self.use_cudnn = False
        self.use_mkldnn = False
        self.dtype = np.float64
        self.init_kernel_type()
        self.shape = self.get_x_shape()
        self.axis = self.get_axis()

        x = np.random.uniform(0.1, 1, self.shape).astype(self.dtype)
        out = np.apply_along_axis(stable_softmax, self.axis, x)

        self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(x)}
        self.outputs = {'Out': out}
        self.attrs = {
            'axis': self.axis,
            'use_cudnn': self.use_cudnn,
            'use_mkldnn': self.use_mkldnn
        }

    def init_kernel_type(self):
        pass

    def test_check_output(self):
        # TODO(wangzhongpu): support mkldnn op in dygraph mode
        if self.use_cudnn:
            place = core.CUDAPlace(0)
            self.check_output_with_place(
                place, atol=1e-5, check_dygraph=(self.use_mkldnn == False))
        else:
            self.check_output(check_dygraph=(self.use_mkldnn == False))

    def test_check_grad(self):
        # TODO(wangzhongpu): support mkldnn op in dygraph mode
        if self.use_cudnn or self.dtype == np.float16:
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_grad_with_place(
                    place, ["X"],
                    "Out",
                    max_relative_error=0.01,
                    check_dygraph=(self.use_mkldnn == False))
        else:
            self.check_grad(
                ["X"],
                "Out",
                max_relative_error=0.01,
                check_dygraph=(self.use_mkldnn == False))

class TestSoftmaxOpError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):
            # The input type of softmax_op must be Variable.
            x1 = fluid.create_lod_tensor(
                np.array([[-1]]), [[1]], fluid.CPUPlace())
            self.assertRaises(TypeError, fluid.layers.softmax, x1)
            # The input dtype of softmax_op must be float16, float32 or float64.
            x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32")
            self.assertRaises(TypeError, fluid.layers.softmax, x2)
            x3 = fluid.layers.data(name='x3', shape=[4], dtype="float16")
            fluid.layers.softmax(x3)

class TestSoftmaxOp2(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

class TestSoftmaxOp3(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 0

class TestSoftmaxOp4(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 1

class TestSoftmaxOp5(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 2

class TestSoftmaxOp6(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 3

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp(TestSoftmaxOp):
    def init_kernel_type(self):
        self.use_cudnn = True

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp2(TestSoftmaxCUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp5(TestSoftmaxCUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 3

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16Op(TestSoftmaxOp):
    def init_kernel_type(self):
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

    # FIXME: If the x_shape is [10, 10], gradient failed.
    def test_check_grad(self):
        pass

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16Op2(TestSoftmaxOp):
    def init_kernel_type(self):
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

    def get_x_shape(self):
        return [2, 3, 4, 5]

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp):
    def init_kernel_type(self):
        self.use_cudnn = True
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16CUDNNOp2(TestSoftmaxFP16CUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

if __name__ == "__main__":
    unittest.main()
<commit_msg>test=develop softmax op fp16 test case pass grad check (#24130)<commit_after>#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
from op_test import OpTest
import paddle.fluid.core as core
import paddle.fluid as fluid
from paddle.fluid import compiler, Program, program_guard

def stable_softmax(x):
    """Compute the softmax of vector x in a numerically stable way."""
    # clip to shiftx, otherwise, when calc loss with
    # log(exp(shiftx)), may get log(0)=INF
    shiftx = (x - np.max(x)).clip(-64.)
    exps = np.exp(shiftx)
    return exps / np.sum(exps)

class TestSoftmaxOp(OpTest):
    def get_x_shape(self):
        return [10, 10]

    def get_axis(self):
        return -1

    def setUp(self):
        self.op_type = "softmax"
        self.use_cudnn = False
        self.use_mkldnn = False
        self.dtype = np.float64
        self.init_kernel_type()
        self.shape = self.get_x_shape()
        self.axis = self.get_axis()

        x = np.random.uniform(0.1, 1, self.shape).astype(self.dtype)
        out = np.apply_along_axis(stable_softmax, self.axis, x)

        self.inputs = {'X': OpTest.np_dtype_to_fluid_dtype(x)}
        self.outputs = {'Out': out}
        self.attrs = {
            'axis': self.axis,
            'use_cudnn': self.use_cudnn,
            'use_mkldnn': self.use_mkldnn
        }

    def init_kernel_type(self):
        pass

    def test_check_output(self):
        # TODO(wangzhongpu): support mkldnn op in dygraph mode
        if self.use_cudnn:
            place = core.CUDAPlace(0)
            self.check_output_with_place(
                place, atol=1e-5, check_dygraph=(self.use_mkldnn == False))
        else:
            self.check_output(check_dygraph=(self.use_mkldnn == False))

    def test_check_grad(self):
        # TODO(wangzhongpu): support mkldnn op in dygraph mode
        if self.use_cudnn or self.dtype == np.float16:
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_grad_with_place(
                    place, ["X"],
                    "Out",
                    max_relative_error=0.01,
                    check_dygraph=(self.use_mkldnn == False))
        else:
            self.check_grad(
                ["X"],
                "Out",
                max_relative_error=0.01,
                check_dygraph=(self.use_mkldnn == False))

class TestSoftmaxOpError(unittest.TestCase):
    def test_errors(self):
        with program_guard(Program(), Program()):
            # The input type of softmax_op must be Variable.
            x1 = fluid.create_lod_tensor(
                np.array([[-1]]), [[1]], fluid.CPUPlace())
            self.assertRaises(TypeError, fluid.layers.softmax, x1)
            # The input dtype of softmax_op must be float16, float32 or float64.
            x2 = fluid.layers.data(name='x2', shape=[4], dtype="int32")
            self.assertRaises(TypeError, fluid.layers.softmax, x2)
            x3 = fluid.layers.data(name='x3', shape=[4], dtype="float16")
            fluid.layers.softmax(x3)

class TestSoftmaxOp2(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

class TestSoftmaxOp3(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 0

class TestSoftmaxOp4(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 1

class TestSoftmaxOp5(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 2

class TestSoftmaxOp6(TestSoftmaxOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 3

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp(TestSoftmaxOp):
    def init_kernel_type(self):
        self.use_cudnn = True

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp2(TestSoftmaxCUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxCUDNNOp5(TestSoftmaxCUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

    def get_axis(self):
        return 3

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16Op(TestSoftmaxOp):
    def init_kernel_type(self):
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

    # FIXME: If the x_shape is [10, 10], gradient failed.
    def test_check_grad(self):
        pass

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16Op2(TestSoftmaxOp):
    def init_kernel_type(self):
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

    def get_x_shape(self):
        return [2, 3, 4, 5]

    def test_check_grad(self):
        pass

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16CUDNNOp(TestSoftmaxOp):
    def init_kernel_type(self):
        self.use_cudnn = True
        self.dtype = np.float16

    def test_check_output(self):
        if core.is_compiled_with_cuda():
            place = core.CUDAPlace(0)
            if core.is_float16_supported(place):
                self.check_output_with_place(place, atol=1e-3)

@unittest.skipIf(not core.is_compiled_with_cuda(),
                 "core is not compiled with CUDA")
class TestSoftmaxFP16CUDNNOp2(TestSoftmaxFP16CUDNNOp):
    def get_x_shape(self):
        return [2, 3, 4, 5]

if __name__ == "__main__":
    unittest.main()
<|endoftext|>