<commit_before>#!/usr/bin/env python
# Copyright (c) 2010, Aaron DeVore
# Released under the Don't Be A Douchbag License.
# Use responsibly. Contribute changes if you feel like it. No CP!

import urllib2
import os
import posixpath
import json
import time
from optparse import OptionParser
from BeautifulSoup import BeautifulSoup

options = OptionParser()
options.add_option("-b", "--board", dest="board",
    default='b', help="board name")
options.add_option("-o", "--overwrite-images", dest="overwriteImages",
    default=False, help="Overwrite non-empty images", action="store_true")
options.add_option("-u", "--update", dest="update", action="store_true",
    default=False, help="update the thread")
options.add_option("-p", "--pause-update", type="int", dest="pauseUpdate",
    default=100, help="Wait time between thread updates")
options.add_option("--pause-image", type="int", dest="pauseImage",
    default=1, help="Wait time between image downloads")

class Post(object):
    def __init__(self, postID, text, poster, timestamp, image, imageTitle):
        self.id = postID
        self.text = text
        self.poster = poster
        self.timestamp = timestamp
        if image:
            self.image = posixpath.basename(image)
            self.imageURL = image
            self.imageTitle = imageTitle
        else:
            self.imageURL = self.image = self.imageTitle = None
    def __repr__(self):
        if self.image:
            return u"%(id)s by %(poster)s with %(image)s" % self.__dict__
        else:
            return u"%(id)s by %(poster)s with no image" % self.__dict__

def getSoup(board, thread):
    url = "http://boards.4chan.org/%s/res/%s" % (board, thread)
    print "downloading thread %s for board %s at %s" % (thread, board, url)
    f = urllib2.urlopen(url)
    soup = BeautifulSoup(f)
    f.close()
    return soup

def getPosts(soup):
    posts = []
    imageCount = 0
    postTables = (td.findParent('table')
        for td in soup.findAll('td', 'doubledash'))

    for postTable in postTables:
        postID = postTable.find('td', id=True)['id']
        text = postTable.find('blockquote').renderContents().strip()
        posterSpan = postTable.find('span', 'commentpostername')
        poster = posterSpan.string
        timestamp = posterSpan.findNext(text=True)

        filespan = postTable.find('span', 'filesize')
        if filespan:
            imageCount += 1
            imageURL = filespan.find('a')['href']
            imageTitle = postTable.find('span', title=True).string
        else:
            imageURL = None
            imageTitle = None

        post = Post(postID, text, poster, timestamp, imageURL, imageTitle)
        print u"found %s" % post
        posts.append(post)
    print u"found %i posts with %i images" % (len(posts), imageCount)
    return posts

def downloadImages(posts, dest, overwriteImages, pauseImage):
    imageDir = os.path.join(dest, "images")
    if not os.path.exists(imageDir):
        os.mkdir(imageDir)
    print "pause time between image requests:", pauseImage
    for post in posts:
        if post.image:
            localPath = os.path.join(imageDir, post.image)
            if os.path.exists(localPath):
                if not overwriteImages and os.path.getsize(localPath) != 0:
                    print u"Skip: image %s already exists" % post.image
                    continue
            print u"downloading %s to %s" % (post.imageURL, post.image)
            with open(localPath, 'w') as f:
                try:
                    remote = urllib2.urlopen(post.imageURL)
                except urllib2.HTTPError, e:
                    if e.code == 404:
                        print "image 404ed"
                    raise
                f.write(remote.read())
            time.sleep(pauseImage) # be nice to the servers

def writeData(thread, posts, dest):
    target = os.path.join(dest, "thread.js")
    jsonPosts = []
    jsonCode = {}
    jsonCode['id'] = thread
    jsonCode['posts'] = jsonPosts
    for post in posts:
        jsonPosts.append({
            'id': post.id,
            'poster': post.poster,
            'image': post.image,
            'timestamp': post.timestamp,
            'text': post.text,
            'imageTitle': post.imageTitle,
            })
    print u"writing thread data for %s to %s" % (thread, target)
    with open(target, 'w') as f:
        json.dump(jsonCode, f)

def main():
    opts, args = options.parse_args()
    if len(args) != 2:
        print options.usage
    thread = args[0]
    baseDest = args[1]
    board = opts.board
    overwriteImages = opts.overwriteImages
    if opts.update:
        updates = -1
    else:
        updates = 1
    dest = os.path.join(baseDest, u"%s-%s" % (board, thread))
    if not os.path.exists(dest):
        os.makedirs(dest)
    try:
        while updates != 0:
            updates -= 1
            soup = getSoup(opts.board, thread)
            posts = getPosts(soup)
            downloadImages(posts, dest, overwriteImages, opts.pauseImage)
            writeData(thread, posts, dest)
            if updates != 0:
                print "waiting %i seconds for next update" % opts.pauseUpdate
                print "-" * 40
                time.sleep(opts.pauseUpdate)
    except KeyboardInterrupt:
        print "Keyboard Interrupt, ending archiving"
    except urllib2.HTTPError, e:
        if e.code == 404:
            print "Thread or image 404ed"
        else:
            raise

if __name__ == "__main__":
    main()<commit_msg>various fixes, use prettify for json<commit_after>#!/usr/bin/env python
# Copyright (c) 2010, Aaron DeVore
# Released under the Don't Be A Douchbag License.
# Use responsibly. Contribute changes if you feel like it. No CP!

import urllib2
import os
import posixpath
import json
import time
from optparse import OptionParser
from BeautifulSoup import BeautifulSoup, Tag
import htmlentitydefs

USAGE = "%prog [options] <thread ID> <base directory>"

options = OptionParser(usage=USAGE)
options.add_option("-b", "--board", dest="board",
    default='b', help="board name")
options.add_option("-o", "--overwrite-images", dest="overwriteImages",
    default=False, help="Overwrite non-empty images", action="store_true")
options.add_option("-u", "--update", dest="update", action="store_true",
    default=False, help="update the thread")
options.add_option("-p", "--pause-update", type="int", dest="pauseUpdate",
    default=100, help="Wait time between thread updates")
options.add_option("--pause-image", type="int", dest="pauseImage",
    default=1, help="Wait time between image downloads")
options.add_option("-n", "--no-pics", action="store_false", dest="pics", default=True,
    help="Do not download pictures")

class Post(object):
    def __init__(self, postID, text, poster, timestamp, image, imageTitle):
        self.id = postID
        self.text = text
        self.poster = poster
        self.timestamp = timestamp
        if image:
            self.image = posixpath.basename(image)
            self.imageURL = image
            self.imageTitle = imageTitle
        else:
            self.imageURL = self.image = self.imageTitle = None
    def __repr__(self):
        if self.image:
            return u"%(id)s by %(poster)s with %(image)s" % self.__dict__
        else:
            return u"%(id)s by %(poster)s with no image" % self.__dict__

def getSoup(board, thread):
    url = "http://boards.4chan.org/%s/res/%s" % (board, thread)
    print "downloading thread %s for board %s at %s" % (thread, board, url)
    f = urllib2.urlopen(url)
    soup = BeautifulSoup(f)
    f.close()
    return soup

def getText(tag, seperator=u""):
    """
    Get all child text for a tag.
    """
    text = []
    for node in tag.recursiveChildGenerator():
        if isinstance(node, Tag) and node.name == "br":
            text.append(u"\n")
            continue
        elif not isinstance(node, unicode):
            continue
        for find, replace in htmlentitydefs.name2codepoint.items():
            node = node.replace(u"&%s;" % find, unichr(replace))
        text.append(node)
    return seperator.join(text)

def getOP(soup):
    threadNode = soup.find("form", {'name': "delform"})
    timestamp = threadNode.find("span", 'posttime').string
    poster = threadNode.find("span", "postername").string
    imageNode = threadNode.find("span", "filesize")
    imageURL = imageNode.a["href"]
    imageTitle = imageNode.findNext("span", "filetitle").string
    postID = threadNode.find("input", type="checkbox", value="delete")["name"]
    text = getText(threadNode.blockquote, " ")
    return Post(postID, text, poster, timestamp, imageURL, imageTitle)

def getRegularPosts(soup, posts):
    postTables = (td.findParent('table')
        for td in soup.findAll('td', 'doubledash'))
    imageCount = 0

    for postTable in postTables:
        postID = postTable.find('td', id=True)['id']
        text = getText(postTable.find('blockquote'), u" ")
        posterSpan = postTable.find('span', 'commentpostername')
        poster = posterSpan.string
        timestamp = posterSpan.findNextSibling(text=True)

        filespan = postTable.find('span', 'filesize')
        if filespan:
            imageCount += 1
            imageURL = filespan.find('a')['href']
            imageTitle = postTable.find('span', title=True).string
        else:
            imageURL = None
            imageTitle = None

        post = Post(postID, text, poster, timestamp, imageURL, imageTitle)
        print u"found %s" % post
        posts.append(post)
    return imageCount

def getPosts(soup):
    posts = [getOP(soup)]
    imageCount = 1 # Start at 1 for OP's image
    imageCount += getRegularPosts(soup, posts)
    print u"found %i posts with %i images" % (len(posts), imageCount)
    return posts

def downloadImages(posts, dest, overwriteImages, pauseImage):
    imageDir = os.path.join(dest, "images")
    if not os.path.exists(imageDir):
        os.mkdir(imageDir)
    print "pause time between image requests:", pauseImage
    for post in posts:
        if post.image:
            localPath = os.path.join(imageDir, post.image)
            if os.path.exists(localPath):
                if not overwriteImages and os.path.getsize(localPath) != 0:
                    print u"Skip: image %s already exists" % post.image
                    continue
            print u"downloading %s to %s" % (post.imageURL, post.image)
            with open(localPath, 'w') as f:
                try:
                    remote = urllib2.urlopen(post.imageURL)
                except urllib2.HTTPError, e:
                    if e.code == 404:
                        print "image 404ed"
                    raise
                f.write(remote.read())
            time.sleep(pauseImage) # be nice to the servers

def writeData(thread, posts, dest):
    target = os.path.join(dest, "thread.js")
    jsonPosts = []
    jsonCode = {}
    jsonCode['id'] = thread

    jsonCode['posts'] = jsonPosts
    for post in posts:
        jsonPosts.append({
            'id': post.id,
            'poster': post.poster,
            'image': post.image,
            'timestamp': post.timestamp,
            'text': post.text,
            'imageTitle': post.imageTitle,
            })
    print u"writing thread data for %s to %s" % (thread, target)
    with open(target, 'w') as f:
        json.dump(jsonCode, f, indent=4)

def main():
    opts, args = options.parse_args()
    if len(args) != 2:
        options.print_usage()
    thread = args[0]
    baseDest = args[1]
    board = opts.board
    overwriteImages = opts.overwriteImages
    if opts.update:
        updates = -1
    else:
        updates = 1
    dest = os.path.join(baseDest, u"%s-%s" % (board, thread))
    if not os.path.exists(dest):
        os.makedirs(dest)
    try:
        while updates != 0:
            updates -= 1
            soup = getSoup(opts.board, thread)
            posts = getPosts(soup)
            if opts.pics:
                downloadImages(posts, dest, overwriteImages, opts.pauseImage)
            writeData(thread, posts, dest)
            if updates != 0:
                print "waiting %i seconds for next update" % opts.pauseUpdate
                print "-" * 40
                time.sleep(opts.pauseUpdate)
    except KeyboardInterrupt:
        print "Keyboard Interrupt, ending archiving"
    except urllib2.HTTPError, e:
        if e.code == 404:
            print "Thread or image 404ed"
        else:
            raise

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding=utf-8

"""
This module contains the following classes:

* :class:`~aeneas.diagnostics.Diagnostics`,
  checking whether the setup of ``aeneas`` was successful.

This module can be executed from command line with::

    python -m aeneas.diagnostics

.. versionadded:: 1.4.1
"""

from __future__ import absolute_import
from __future__ import print_function
import sys

import aeneas.globalfunctions as gf

__author__ = "Alberto Pettarin"
__copyright__ = """
    Copyright 2012-2013, Alberto Pettarin (www.albertopettarin.it)
    Copyright 2013-2015, ReadBeyond Srl   (www.readbeyond.it)
    Copyright 2015-2016, Alberto Pettarin (www.albertopettarin.it)
    """
__license__ = "GNU AGPL v3"
__version__ = "1.5.1"
__email__ = "aeneas@readbeyond.it"
__status__ = "Production"

class Diagnostics(object):
    """
    Check whether the setup of ``aeneas`` was successful.
    """
    @classmethod
    def check_shell_encoding(cls):
        """
        Check whether ``sys.stdin`` and ``sys.stdout`` are UTF-8 encoded.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        is_in_utf8 = True
        is_out_utf8 = True
        if sys.stdin.encoding not in ["UTF-8", "UTF8"]:
            is_in_utf8 = False
        if sys.stdout.encoding not in ["UTF-8", "UTF8"]:
            is_out_utf8 = False
        if (is_in_utf8) and (is_out_utf8):
            gf.print_success(u"shell encoding OK")
        else:
            gf.print_warning(u"shell encoding WARNING")
            if not is_in_utf8:
                gf.print_warning(u"  The default input encoding of your shell is not UTF-8")
            if not is_out_utf8:
                gf.print_warning(u"  The default output encoding of your shell is not UTF-8")
            gf.print_info(u"  If you plan to use aeneas on the command line,")
            if gf.is_posix():
                gf.print_info(u"  you might want to 'export PYTHONIOENCODING=UTF-8' in your shell")
            else:
                gf.print_info(u"  you might want to 'set PYTHONIOENCODING=UTF-8' in your shell")
            return True
        return False

    @classmethod
    def check_ffprobe(cls):
        """
        Check whether ``ffprobe`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.ffprobewrapper import FFPROBEWrapper
            file_path = gf.absolute_path(u"tools/res/audio.mp3", __file__)
            prober = FFPROBEWrapper()
            properties = prober.read_properties(file_path)
            gf.print_success(u"ffprobe        OK")
            return False
        except:
            pass
        gf.print_error(u"ffprobe        ERROR")
        gf.print_info(u"  Please make sure you have ffprobe installed correctly")
        gf.print_info(u"  (usually it is provided by the ffmpeg installer)")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        return True

    @classmethod
    def check_ffmpeg(cls):
        """
        Check whether ``ffmpeg`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.ffmpegwrapper import FFMPEGWrapper
            input_file_path = gf.absolute_path(u"tools/res/audio.mp3", __file__)
            handler, output_file_path = gf.tmp_file(suffix=u".wav")
            converter = FFMPEGWrapper()
            result = converter.convert(input_file_path, output_file_path)
            gf.delete_file(handler, output_file_path)
            if result:
                gf.print_success(u"ffmpeg         OK")
                return False
        except:
            pass
        gf.print_error(u"ffmpeg         ERROR")
        gf.print_info(u"  Please make sure you have ffmpeg installed correctly")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        return True

    @classmethod
    def check_espeak(cls):
        """
        Check whether ``espeak`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.espeakwrapper import ESPEAKWrapper
            text = u"From fairest creatures we desire increase,"
            language = u"eng"
            handler, output_file_path = gf.tmp_file(suffix=u".wav")
            espeak = ESPEAKWrapper()
            result = espeak.synthesize_single(
                text,
                language,
                output_file_path
            )
            gf.delete_file(handler, output_file_path)
            if result:
                gf.print_success(u"espeak         OK")
                return False
        except:
            pass
        gf.print_error(u"espeak         ERROR")
        gf.print_info(u"  Please make sure you have espeak installed correctly")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        gf.print_info(u"  You might also want to check that the espeak-data directory")
        gf.print_info(u"  is set up correctly, for example, it has the correct permissions")
        return True

    @classmethod
    def check_tools(cls):
        """
        Check whether ``aeneas.tools.*`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.tools.convert_syncmap import ConvertSyncMapCLI
            # disabling this check, as it contains optional dependency pafy
            #from aeneas.tools.download import DownloadCLI
            from aeneas.tools.execute_job import ExecuteJobCLI
            from aeneas.tools.execute_task import ExecuteTaskCLI
            from aeneas.tools.extract_mfcc import ExtractMFCCCLI
            from aeneas.tools.ffmpeg_wrapper import FFMPEGWrapperCLI
            from aeneas.tools.ffprobe_wrapper import FFPROBEWrapperCLI
            # disabling this check, as it contains optional dependency Pillow
            #from aeneas.tools.plot_waveform import PlotWaveformCLI
            from aeneas.tools.read_audio import ReadAudioCLI
            from aeneas.tools.read_text import ReadTextCLI
            from aeneas.tools.run_sd import RunSDCLI
            from aeneas.tools.run_vad import RunVADCLI
            from aeneas.tools.synthesize_text import SynthesizeTextCLI
            from aeneas.tools.validate import ValidateCLI
            gf.print_success(u"aeneas.tools   OK")
            return False
        except:
            pass
        gf.print_error(u"aeneas.tools   ERROR")
        gf.print_info(u"  Unable to import one or more aeneas.tools")
        gf.print_info(u"  Please check that you installed aeneas properly")
        return True

    @classmethod
    def check_cdtw(cls):
        """
        Check whether Python C extension ``cdtw`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cdtw"):
            gf.print_success(u"aeneas.cdtw    AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cdtw    NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be significantly slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_cmfcc(cls):
        """
        Check whether Python C extension ``cmfcc`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cmfcc"):
            gf.print_success(u"aeneas.cmfcc   AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cmfcc   NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be significantly slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_cew(cls):
        """
        Check whether Python C extension ``cew`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cew"):
            gf.print_success(u"aeneas.cew     AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cew     NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be a bit slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_all(cls, tools=True, encoding=True, c_ext=True):
        """
        Perform all checks.

        Return a tuple of booleans ``(errors, warnings, c_ext_warnings)``.

        :param bool tools: if ``True``, check aeneas tools
        :param bool encoding: if ``True``, check shell encoding
        :param bool c_ext: if ``True``, check Python C extensions
        :rtype: (bool, bool, bool)
        """
        # errors are fatal
        if cls.check_ffprobe():
            return (True, False, False)
        if cls.check_ffmpeg():
            return (True, False, False)
        if cls.check_espeak():
            return (True, False, False)
        if (tools) and (cls.check_tools()):
            return (True, False, False)
        # warnings are non-fatal
        warnings = False
        c_ext_warnings = False
        if encoding:
            warnings = cls.check_shell_encoding()
        if c_ext:
            # we do not want lazy evaluation
            c_ext_warnings = cls.check_cdtw() or c_ext_warnings
            c_ext_warnings = cls.check_cmfcc() or c_ext_warnings
            c_ext_warnings = cls.check_cew() or c_ext_warnings
        # return results
        return (False, warnings, c_ext_warnings)

def main():
    errors, warnings, c_ext_warnings = Diagnostics.check_all()
    if errors:
        sys.exit(1)
    if c_ext_warnings:
        gf.print_warning(u"All required dependencies are met but at least one available Python C extension is not compiled")
        sys.exit(2)
    else:
        gf.print_success(u"All required dependencies are met and all available Python C extensions are compiled")
        sys.exit(0)

if __name__ == '__main__':
    main()

<commit_msg>Update to reflect new wording.<commit_after>#!/usr/bin/env python
# coding=utf-8

"""
This module contains the following classes:

* :class:`~aeneas.diagnostics.Diagnostics`,
  checking whether the setup of ``aeneas`` was successful.

This module can be executed from command line with::

    python -m aeneas.diagnostics

.. versionadded:: 1.4.1
"""

from __future__ import absolute_import
from __future__ import print_function
import sys

import aeneas.globalfunctions as gf

__author__ = "Alberto Pettarin"
__copyright__ = """
    Copyright 2012-2013, Alberto Pettarin (www.albertopettarin.it)
    Copyright 2013-2015, ReadBeyond Srl   (www.readbeyond.it)
    Copyright 2015-2016, Alberto Pettarin (www.albertopettarin.it)
    """
__license__ = "GNU AGPL v3"
__version__ = "1.5.1"
__email__ = "aeneas@readbeyond.it"
__status__ = "Production"

class Diagnostics(object):
    """
    Check whether the setup of ``aeneas`` was successful.
    """
    @classmethod
    def check_shell_encoding(cls):
        """
        Check whether ``sys.stdin`` and ``sys.stdout`` are UTF-8 encoded.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        is_in_utf8 = True
        is_out_utf8 = True
        if sys.stdin.encoding not in ["UTF-8", "UTF8"]:
            is_in_utf8 = False
        if sys.stdout.encoding not in ["UTF-8", "UTF8"]:
            is_out_utf8 = False
        if (is_in_utf8) and (is_out_utf8):
            gf.print_success(u"shell encoding OK")
        else:
            gf.print_warning(u"shell encoding WARNING")
            if not is_in_utf8:
                gf.print_warning(u"  The default input encoding of your shell is not UTF-8")
            if not is_out_utf8:
                gf.print_warning(u"  The default output encoding of your shell is not UTF-8")
            gf.print_info(u"  If you plan to use aeneas on the command line,")
            if gf.is_posix():
                gf.print_info(u"  you might want to 'export PYTHONIOENCODING=UTF-8' in your shell")
            else:
                gf.print_info(u"  you might want to 'set PYTHONIOENCODING=UTF-8' in your shell")
            return True
        return False

    @classmethod
    def check_ffprobe(cls):
        """
        Check whether ``ffprobe`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.ffprobewrapper import FFPROBEWrapper
            file_path = gf.absolute_path(u"tools/res/audio.mp3", __file__)
            prober = FFPROBEWrapper()
            properties = prober.read_properties(file_path)
            gf.print_success(u"ffprobe        OK")
            return False
        except:
            pass
        gf.print_error(u"ffprobe        ERROR")
        gf.print_info(u"  Please make sure you have ffprobe installed correctly")
        gf.print_info(u"  (usually it is provided by the ffmpeg installer)")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        return True

    @classmethod
    def check_ffmpeg(cls):
        """
        Check whether ``ffmpeg`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.ffmpegwrapper import FFMPEGWrapper
            input_file_path = gf.absolute_path(u"tools/res/audio.mp3", __file__)
            handler, output_file_path = gf.tmp_file(suffix=u".wav")
            converter = FFMPEGWrapper()
            result = converter.convert(input_file_path, output_file_path)
            gf.delete_file(handler, output_file_path)
            if result:
                gf.print_success(u"ffmpeg         OK")
                return False
        except:
            pass
        gf.print_error(u"ffmpeg         ERROR")
        gf.print_info(u"  Please make sure you have ffmpeg installed correctly")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        return True

    @classmethod
    def check_espeak(cls):
        """
        Check whether ``espeak`` can be called.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.espeakwrapper import ESPEAKWrapper
            text = u"From fairest creatures we desire increase,"
            language = u"eng"
            handler, output_file_path = gf.tmp_file(suffix=u".wav")
            espeak = ESPEAKWrapper()
            result = espeak.synthesize_single(
                text,
                language,
                output_file_path
            )
            gf.delete_file(handler, output_file_path)
            if result:
                gf.print_success(u"espeak         OK")
                return False
        except:
            pass
        gf.print_error(u"espeak         ERROR")
        gf.print_info(u"  Please make sure you have espeak installed correctly")
        gf.print_info(u"  and that its path is in your PATH environment variable")
        gf.print_info(u"  You might also want to check that the espeak-data directory")
        gf.print_info(u"  is set up correctly, for example, it has the correct permissions")
        return True

    @classmethod
    def check_tools(cls):
        """
        Check whether ``aeneas.tools.*`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        try:
            from aeneas.tools.convert_syncmap import ConvertSyncMapCLI
            # disabling this check, as it contains optional dependency pafy
            #from aeneas.tools.download import DownloadCLI
            from aeneas.tools.execute_job import ExecuteJobCLI
            from aeneas.tools.execute_task import ExecuteTaskCLI
            from aeneas.tools.extract_mfcc import ExtractMFCCCLI
            from aeneas.tools.ffmpeg_wrapper import FFMPEGWrapperCLI
            from aeneas.tools.ffprobe_wrapper import FFPROBEWrapperCLI
            # disabling this check, as it contains optional dependency Pillow
            #from aeneas.tools.plot_waveform import PlotWaveformCLI
            from aeneas.tools.read_audio import ReadAudioCLI
            from aeneas.tools.read_text import ReadTextCLI
            from aeneas.tools.run_sd import RunSDCLI
            from aeneas.tools.run_vad import RunVADCLI
            from aeneas.tools.synthesize_text import SynthesizeTextCLI
            from aeneas.tools.validate import ValidateCLI
            gf.print_success(u"aeneas.tools   OK")
            return False
        except:
            pass
        gf.print_error(u"aeneas.tools   ERROR")
        gf.print_info(u"  Unable to import one or more aeneas.tools")
        gf.print_info(u"  Please check that you installed aeneas properly")
        return True

    @classmethod
    def check_cdtw(cls):
        """
        Check whether Python C extension ``cdtw`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cdtw"):
            gf.print_success(u"aeneas.cdtw    AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cdtw    NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be significantly slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_cmfcc(cls):
        """
        Check whether Python C extension ``cmfcc`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cmfcc"):
            gf.print_success(u"aeneas.cmfcc   AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cmfcc   NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be significantly slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_cew(cls):
        """
        Check whether Python C extension ``cew`` can be imported.

        Return ``True`` on failure and ``False`` on success.

        :rtype: bool
        """
        if gf.can_run_c_extension("cew"):
            gf.print_success(u"aeneas.cew     AVAILABLE")
            return False
        gf.print_warning(u"aeneas.cew     NOT AVAILABLE")
        gf.print_info(u"  You can still run aeneas but it will be a bit slower")
        gf.print_info(u"  Please refer to the installation documentation for details")
        return True

    @classmethod
    def check_all(cls, tools=True, encoding=True, c_ext=True):
        """
        Perform all checks.

        Return a tuple of booleans ``(errors, warnings, c_ext_warnings)``.

        :param bool tools: if ``True``, check aeneas tools
        :param bool encoding: if ``True``, check shell encoding
        :param bool c_ext: if ``True``, check Python C extensions
        :rtype: (bool, bool, bool)
        """
        # errors are fatal
        if cls.check_ffprobe():
            return (True, False, False)
        if cls.check_ffmpeg():
            return (True, False, False)
        if cls.check_espeak():
            return (True, False, False)
        if (tools) and (cls.check_tools()):
            return (True, False, False)
        # warnings are non-fatal
        warnings = False
        c_ext_warnings = False
        if encoding:
            warnings = cls.check_shell_encoding()
        if c_ext:
            # we do not want lazy evaluation
            c_ext_warnings = cls.check_cdtw() or c_ext_warnings
            c_ext_warnings = cls.check_cmfcc() or c_ext_warnings
            c_ext_warnings = cls.check_cew() or c_ext_warnings
        # return results
        return (False, warnings, c_ext_warnings)

def main():
    errors, warnings, c_ext_warnings = Diagnostics.check_all()
    if errors:
        sys.exit(1)
    if c_ext_warnings:
        gf.print_warning(u"All required dependencies are met but at least one Python C extension is not available")
        sys.exit(2)
    else:
        gf.print_success(u"All required dependencies are met and all available Python C extensions are working")
        sys.exit(0)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>import numpy
import arrayfire
import numbers
from IPython.core.debugger import Tracer

dim_t = numpy.int64

__TypeMap__ = { float: arrayfire.f64,
                numpy.float32: arrayfire.f32,
                numpy.dtype('float32'): arrayfire.f32,
                numpy.float64: arrayfire.f64,
                numpy.dtype('float64'): arrayfire.f64,
                numpy.int8: arrayfire.b8,
                numpy.dtype('int8'): arrayfire.b8,
                numpy.uint8: arrayfire.u8,
                numpy.dtype('uint8'): arrayfire.u8,
                numpy.bool: arrayfire.b8,
                numpy.dtype('bool'): arrayfire.b8,
                numpy.int64: arrayfire.s64,
                numpy.dtype('int64'): arrayfire.s64,
                numpy.uint64: arrayfire.u64,
                numpy.dtype('uint64'): arrayfire.u64,
                numpy.uint32: arrayfire.u32,
                numpy.dtype('uint32'): arrayfire.u32,
                numpy.int32: arrayfire.s32,
                numpy.dtype('int32'): arrayfire.s32,
                numpy.complex128: arrayfire.c64,
                numpy.dtype('complex128'): arrayfire.c64,
            }

__TypeToString__ = { arrayfire.f64: 'f64',
                     arrayfire.f32: 'f32',
                     arrayfire.u32: 'u32',
                     arrayfire.s32: 's32',
                     arrayfire.u64: 'u64',
                     arrayfire.s64: 's64',
                     arrayfire.c32: 'c32',
                     arrayfire.c64: 'c64',
                     }

__dummy__ = object()

def _raw(x):
    if(isinstance(x,ndarray)):
        return x.d_array
    else:
        return x

def _af_shape(af_array):
    shape = ()
    for i in range(0,af_array.numdims()):
        shape = (af_array.dims(i),)+shape
    return shape

def vdot(a, b):
    s = arrayfire.dot(arrayfire.conjg(a.d_array), b.d_array)
    return ndarray(_af_shape(s), dtype=a.dtype, af_array=s)

def zeros(shape, dtype=float, order='C'):
    b = numpy.zeros(shape, dtype, order)
    return ndarray(b.shape, b.dtype, buffer=b,order=order)

def ones(shape, dtype=float, order='C'):
    b = numpy.ones(shape, dtype, order)
    return ndarray(b.shape, b.dtype, buffer=b,order=order)

def array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0):
    if(order is not None):
        raise NotImplementedError
    if(subok is not False):
        raise NotImplementedError
    # If it's not a numpy or afnumpy array first create a numpy array from it
    if(not isinstance(object, ndarray) and
       not isinstance(object, numpy.ndarray)):
        object = numpy.array(object, dtype=dtype, copy=copy, subok=subok, ndmin=ndmin)
#        return ndarray(a.shape, dtype=a.dtype, buffer=a)

    shape = object.shape
    while(ndmin > len(shape)):
        shape = (1,)+shape
    if(dtype is None):
        dtype = object.dtype
    if(isinstance(object, ndarray)):
        if(copy):
            s = object.d_array.copy().astype(__TypeMap__[dtype])
        else:
            s = object.d_array.astype(__TypeMap__[dtype])
        return ndarray(shape, dtype=dtype, af_array=s)
    elif(isinstance(object, numpy.ndarray)):
        return ndarray(shape, dtype=dtype, buffer=object.astype(dtype, copy=False))
    else:
        raise AssertionError

def where(condition, x=__dummy__, y=__dummy__):
    a = condition
    s = arrayfire.where(a.d_array)
    if(x is __dummy__ and y is __dummy__):
        return ndarray(_af_shape(s), dtype=numpy.uint32, af_array=s)
    elif(x is not __dummy__ and y is not __dummy__):
        if(x.dtype != y.dtype):
            raise TypeError('x and y must have same dtype')
        if(x.shape != y.shape):
            raise ValueError('x and y must have same shape')
        idx = ndarray(_af_shape(s), dtype=numpy.uint32, af_array=s)
        ret = array(y)
        ret[idx] = x[idx]
        return ret;
    else:
        raise ValueError('either both or neither of x and y should be given')

def all(a, axis=None, out=None, keepdims=False):
    if(out is not None):
        raise NotImplementedError
    if(keepdims is not False):
        raise NotImplementedError
    if(axis is None):
        for i in range(len(a.shape)-1,-1,-1):
            s = arrayfire.allTrue(a.d_array, c2f(a.shape, i))
            a = ndarray(_af_shape(s), dtype=bool, af_array=s)
    else:
        s = arrayfire.allTrue(a.d_array, c2f(a.shape, axis))
    a = ndarray(_af_shape(s), dtype=bool, af_array=s)
    if(axis == -1):
        if(keepdims):
            return numpy.array(a)
        else:
            return numpy.array(a)[0]
    else:
        return a

def sum(a, axis=None, dtype=None, out=None, keepdims=False):
    if(out is not None):
        raise NotImplementedError
    if(keepdims is not False):
        raise NotImplementedError
    if(axis is None):
        for i in range(len(a.shape)-1,-1,-1):
            s = arrayfire.sum(a.d_array, c2f(a.shape, i))
            a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    else:
        s = arrayfire.sum(a.d_array, c2f(a.shape, axis))
    a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    if(axis is None):
        if(keepdims):
            return numpy.array(a)
        else:
            return numpy.array(a)[0]
    else:
        return a

def reshape(a, newshape, order='C'):
    if(order is not 'C'):
        raise NotImplementedError
    newshape = numpy.array(c2f(newshape), dtype=dim_t)
    ret, handle = arrayfire.af_moddims(a.d_array.get(), newshape.size, newshape.ctypes.data)
    s = arrayfire.array_from_handle(handle)
    a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    return a

def c2f(shape, dim = None):
    if(dim is None):
        return shape[::-1]
    else:
        return len(shape)-dim-1

class ndarray(object):
    def __init__(self, shape, dtype=float, buffer=None, offset=0, strides=None, order=None, af_array=None):
        if(offset != 0):
            raise NotImplementedError('offset must be 0')
        if(strides is not None):
            raise NotImplementedError('strides must be None')
        if(order is not None and order != 'C'):
            raise NotImplementedError('order must be None')
        self.shape = shape
        self.dtype = dtype
        s_a = numpy.array(c2f(shape),dtype=dim_t)
        if(s_a.size < 1):
            raise NotImplementedError('0 dimension arrays are not yet supported')
        elif(s_a.size < 4):
            if(af_array is not None):
                self.handle = af_array.get()
                # We need to make sure to keep a copy of af_array
                # Otherwise python will free it and havoc ensues
                self.d_array = af_array
            else:
                if(buffer is not None):
                    ret, self.handle = arrayfire.af_create_array(buffer.ctypes.data, s_a.size, s_a.ctypes.data, __TypeMap__[dtype])
                else:
                    ret, self.handle = arrayfire.af_create_handle(s_a.size, s_a.ctypes.data, __TypeMap__[dtype])
                self.d_array = arrayfire.array_from_handle(self.handle)
        else:
            raise NotImplementedError('Only up to 4 dimensions are supported')
        self.h_array = numpy.ndarray(shape,dtype,buffer,offset,strides,order)

    def __repr__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return self.h_array.__repr__()

    def __str__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return self.h_array.__str__()

    def __add__(self, other):
        s = arrayfire.__add__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __iadd__(self, other):
        self.d_array += _raw(other)
        return self

    def __radd__(self, other):
        s = arrayfire.__add__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __sub__(self, other):
        s = arrayfire.__sub__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __isub__(self, other):
        self.d_array -= _raw(other)
        return self

    def __rsub__(self, other):
        s = arrayfire.__sub__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __mul__(self, other):
        s = arrayfire.__mul__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __imul__(self, other):
        self.d_array *= _raw(other)
        return self

    def __rmul__(self, other):
        s = arrayfire.__mul__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __div__(self, other):
        s = arrayfire.__div__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __idiv__(self, other):
        self.d_array /= _raw(other)
        return self

    def __rdiv__(self, other):
        s = arrayfire.__div__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __pow__(self, other):
        s = arrayfire.pow(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __rpow__(self, other):
        s = arrayfire.pow(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __lt__(self, other):
        s = arrayfire.__lt__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __le__(self, other):
        s = arrayfire.__le__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __gt__(self, other):
        s = arrayfire.__gt__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __ge__(self, other):
        s = arrayfire.__ge__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __eq__(self, other):
        s = arrayfire.__eq__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __ne__(self, other):
        s = arrayfire.__ne__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __nonzero__(self):
        return numpy.array(self).__nonzero__()

    def __len__(self):
        return len(h_array)

    @property
    def size(self):
        return len(h_array)

    def _convert_dim(self, args):
        if args < 0:
            args = len(self) + args
        return args

    def __getitem__(self, args):
        if(isinstance(args, ndarray)):
            s = (self.d_array[arrayfire.index(args.d_array)])
            return ndarray(_af_shape(s), dtype=self.dtype, af_array=s)
        elif(isinstance(args, tuple)):
            args = list(args)
            args[0] = self._convert_dim(args[0])
            return self._data[tuple(args)]
        else:
            idx = arrayfire.index(self._convert_dim(args))
            s = self.d_array.__getitem__(arrayfire.index(self._convert_dim(args)))
            return ndarray(_af_shape(s), dtype=self.dtype, af_array=s)

    def __convert_dim__(self, idx, maxlen):
        if(isinstance(idx, slice)):
            if idx.step is None:
                step = 1
            else:
                step = idx.step
            if idx.start is None:
                if step < 0:
                    start = maxlen-1
                else:
                    start = 0
            else:
                start = idx.start
            if idx.stop is None:
                if step < 0:
                    end = 0
                else:
                    end = maxlen-1
            else:
                end = idx.stop
            return slice(start,end,step)
        else:
            if idx < 0:
                return maxlen+idx
            else:
                return idx

    def __setitem__(self, idx, value):
        if(isinstance(idx, ndarray)):
            idx = arrayfire.index(idx.d_array)
        elif(isinstance(idx, slice)):
            idx = self.__convert_dim__(idx,self.shape[0])
            idx = arrayfire.index(arrayfire.seq(float(idx.start),float(idx.stop),float(idx.step)))
        else:
            raise NotImplementedError('indices must be a afnumpy.ndarray')
        if(isinstance(value, ndarray)):
            if(value.dtype != self.dtype):
                raise TypeError('left hand side must have same dtype as right hand side')
            self.d_array.setValue(idx, value.d_array)
        elif(isinstance(value, numbers.Number)):
            self.d_array.setValue(idx, value)
        else:
            raise NotImplementedError('values must be a afnumpy.ndarray')

    def __array__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return numpy.copy(self.h_array)

    def reshape(self, shape, order = 'C'):
        return reshape(self, shape, order)

    def max(self):
        type_max = getattr(arrayfire, 'max_'+__TypeToString__[self.d_array.type()])
        return type_max(self.d_array)

    def min(self):
        type_max = getattr(arrayfire, 'min_'+__TypeToString__[self.d_array.type()])
        return type_max(self.d_array)

#    def __getattr__(self,name):
#        print name
#        raise AttributeError

<commit_msg>Start implementing abs()<commit_after>import numpy
import arrayfire
import numbers
from IPython.core.debugger import Tracer

dim_t = numpy.int64

__TypeMap__ = { float: arrayfire.f64,
                numpy.float32: arrayfire.f32,
                numpy.dtype('float32'): arrayfire.f32,
                numpy.float64: arrayfire.f64,
                numpy.dtype('float64'): arrayfire.f64,
                numpy.int8: arrayfire.b8,
                numpy.dtype('int8'): arrayfire.b8,
                numpy.uint8: arrayfire.u8,
                numpy.dtype('uint8'): arrayfire.u8,
                numpy.bool: arrayfire.b8,
                numpy.dtype('bool'): arrayfire.b8,
                numpy.int64: arrayfire.s64,
                numpy.dtype('int64'): arrayfire.s64,
                numpy.uint64: arrayfire.u64,
                numpy.dtype('uint64'): arrayfire.u64,
                numpy.uint32: arrayfire.u32,
                numpy.dtype('uint32'): arrayfire.u32,
                numpy.int32: arrayfire.s32,
                numpy.dtype('int32'): arrayfire.s32,
                numpy.complex128: arrayfire.c64,
                numpy.dtype('complex128'): arrayfire.c64,
                numpy.complex64: arrayfire.c32,
                numpy.dtype('complex64'): arrayfire.c32,
            }

__TypeToString__ = { arrayfire.f64: 'f64',
                     arrayfire.f32: 'f32',
                     arrayfire.u32: 'u32',
                     arrayfire.s32: 's32',
                     arrayfire.u64: 'u64',
                     arrayfire.s64: 's64',
                     arrayfire.c32: 'c32',
                     arrayfire.c64: 'c64',
                     }

__dummy__ = object()

def _raw(x):
    if(isinstance(x,ndarray)):
        return x.d_array
    else:
        return x

def _af_shape(af_array):
    shape = ()
    for i in range(0,af_array.numdims()):
        shape = (af_array.dims(i),)+shape
    return shape

def vdot(a, b):
    s = arrayfire.dot(arrayfire.conjg(a.d_array), b.d_array)
    return ndarray(_af_shape(s), dtype=a.dtype, af_array=s)

def zeros(shape, dtype=float, order='C'):
    b = numpy.zeros(shape, dtype, order)
    return ndarray(b.shape, b.dtype, buffer=b,order=order)

def ones(shape, dtype=float, order='C'):
    b = numpy.ones(shape, dtype, order)
    return ndarray(b.shape, b.dtype, buffer=b,order=order)

def array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0):
    if(order is not None):
        raise NotImplementedError
    if(subok is not False):
        raise NotImplementedError
    # If it's not a numpy or afnumpy array first create a numpy array from it
    if(not isinstance(object, ndarray) and
       not isinstance(object, numpy.ndarray)):
        object = numpy.array(object, dtype=dtype, copy=copy, subok=subok, ndmin=ndmin)
#        return ndarray(a.shape, dtype=a.dtype, buffer=a)

    shape = object.shape
    while(ndmin > len(shape)):
        shape = (1,)+shape
    if(dtype is None):
        dtype = object.dtype
    if(isinstance(object, ndarray)):
        if(copy):
            s = object.d_array.copy().astype(__TypeMap__[dtype])
        else:
            s = object.d_array.astype(__TypeMap__[dtype])
        return ndarray(shape, dtype=dtype, af_array=s)
    elif(isinstance(object, numpy.ndarray)):
        return ndarray(shape, dtype=dtype, buffer=object.astype(dtype, copy=False))
    else:
        raise AssertionError

def where(condition, x=__dummy__, y=__dummy__):
    a = condition
    s = arrayfire.where(a.d_array)
    if(x is __dummy__ and y is __dummy__):
        return ndarray(_af_shape(s), dtype=numpy.uint32, af_array=s)
    elif(x is not __dummy__ and y is not __dummy__):
        if(x.dtype != y.dtype):
            raise TypeError('x and y must have same dtype')
        if(x.shape != y.shape):
            raise ValueError('x and y must have same shape')
        idx = ndarray(_af_shape(s), dtype=numpy.uint32, af_array=s)
        ret = array(y)
        ret[idx] = x[idx]
        return ret;
    else:
        raise ValueError('either both or neither of x and y should be given')

def all(a, axis=None, out=None, keepdims=False):
    if(out is not None):
        raise NotImplementedError
    if(keepdims is not False):
        raise NotImplementedError
    if(axis is None):
        for i in range(len(a.shape)-1,-1,-1):
            s = arrayfire.allTrue(a.d_array, c2f(a.shape, i))
            a = ndarray(_af_shape(s), dtype=bool, af_array=s)
    else:
        s = arrayfire.allTrue(a.d_array, c2f(a.shape, axis))
    a = ndarray(_af_shape(s), dtype=bool, af_array=s)
    if(axis == -1):
        if(keepdims):
            return numpy.array(a)
        else:
            return numpy.array(a)[0]
    else:
        return a

def sum(a, axis=None, dtype=None, out=None, keepdims=False):
    if(out is not None):
        raise NotImplementedError
    if(keepdims is not False):
        raise NotImplementedError
    if(axis is None):
        for i in range(len(a.shape)-1,-1,-1):
            s = arrayfire.sum(a.d_array, c2f(a.shape, i))
            a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    else:
        s = arrayfire.sum(a.d_array, c2f(a.shape, axis))
    a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    if(axis is None):
        if(keepdims):
            return numpy.array(a)
        else:
            return numpy.array(a)[0]
    else:
        return a

def reshape(a, newshape, order='C'):
    if(order is not 'C'):
        raise NotImplementedError
    newshape = numpy.array(c2f(newshape), dtype=dim_t)
    ret, handle = arrayfire.af_moddims(a.d_array.get(), newshape.size, newshape.ctypes.data)
    s = arrayfire.array_from_handle(handle)
    a = ndarray(_af_shape(s), dtype=a.dtype, af_array=s)
    return a

def c2f(shape, dim = None):
    if(dim is None):
        return shape[::-1]
    else:
        return len(shape)-dim-1

class ndarray(object):
    def __init__(self, shape, dtype=float, buffer=None, offset=0, strides=None, order=None, af_array=None):
        if(offset != 0):
            raise NotImplementedError('offset must be 0')
        if(strides is not None):
            raise NotImplementedError('strides must be None')
        if(order is not None and order != 'C'):
            raise NotImplementedError('order must be None')
        self.shape = shape
        self.dtype = dtype
        s_a = numpy.array(c2f(shape),dtype=dim_t)
        if(s_a.size < 1):
            raise NotImplementedError('0 dimension arrays are not yet supported')
        elif(s_a.size < 4):
            if(af_array is not None):
                self.handle = af_array.get()
                # We need to make sure to keep a copy of af_array
                # Otherwise python will free it and havoc ensues
                self.d_array = af_array
            else:
                if(buffer is not None):
                    ret, self.handle = arrayfire.af_create_array(buffer.ctypes.data, s_a.size, s_a.ctypes.data, __TypeMap__[dtype])
                else:
                    ret, self.handle = arrayfire.af_create_handle(s_a.size, s_a.ctypes.data, __TypeMap__[dtype])
                self.d_array = arrayfire.array_from_handle(self.handle)
        else:
            raise NotImplementedError('Only up to 4 dimensions are supported')
        self.h_array = numpy.ndarray(shape,dtype,buffer,offset,strides,order)

    def __repr__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return self.h_array.__repr__()

    def __str__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return self.h_array.__str__()

    def __add__(self, other):
        s = arrayfire.__add__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __iadd__(self, other):
        self.d_array += _raw(other)
        return self

    def __radd__(self, other):
        s = arrayfire.__add__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __sub__(self, other):
        s = arrayfire.__sub__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __isub__(self, other):
        self.d_array -= _raw(other)
        return self

    def __rsub__(self, other):
        s = arrayfire.__sub__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __mul__(self, other):
        s = arrayfire.__mul__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __imul__(self, other):
        self.d_array *= _raw(other)
        return self

    def __rmul__(self, other):
        s = arrayfire.__mul__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __div__(self, other):
        s = arrayfire.__div__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __idiv__(self, other):
        self.d_array /= _raw(other)
        return self

    def __rdiv__(self, other):
        s = arrayfire.__div__(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __pow__(self, other):
        s = arrayfire.pow(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __rpow__(self, other):
        s = arrayfire.pow(_raw(other), self.d_array)
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __lt__(self, other):
        s = arrayfire.__lt__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __le__(self, other):
        s = arrayfire.__le__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __gt__(self, other):
        s = arrayfire.__gt__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __ge__(self, other):
        s = arrayfire.__ge__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __eq__(self, other):
        s = arrayfire.__eq__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __ne__(self, other):
        s = arrayfire.__ne__(self.d_array, _raw(other))
        return ndarray(self.shape, dtype=numpy.bool, af_array=s)

    def __abs__(self):
        s = arrayfire.abs(self.d_array)
        # dtype is wrong for complex types
        return ndarray(self.shape, dtype=self.dtype, af_array=s)

    def __nonzero__(self):
        return numpy.array(self).__nonzero__()

    def __len__(self):
        return len(h_array)

    @property
    def size(self):
        return len(h_array)

    def _convert_dim(self, args):
        if args < 0:
            args = len(self) + args
        return args

    def __getitem__(self, args):
        if(isinstance(args, ndarray)):
            s = (self.d_array[arrayfire.index(args.d_array)])
            return ndarray(_af_shape(s), dtype=self.dtype, af_array=s)
        elif(isinstance(args, tuple)):
            args = list(args)
            args[0] = self._convert_dim(args[0])
            return self._data[tuple(args)]
        else:
            idx = arrayfire.index(self._convert_dim(args))
            s = self.d_array.__getitem__(arrayfire.index(self._convert_dim(args)))
            return ndarray(_af_shape(s), dtype=self.dtype, af_array=s)

    def __convert_dim__(self, idx, maxlen):
        if(isinstance(idx, slice)):
            if idx.step is None:
                step = 1
            else:
                step = idx.step
            if idx.start is None:
                if step < 0:
                    start = maxlen-1
                else:
                    start = 0
            else:
                start = idx.start
            if idx.stop is None:
                if step < 0:
                    end = 0
                else:
                    end = maxlen-1
            else:
                end = idx.stop
            return slice(start,end,step)
        else:
            if idx < 0:
                return maxlen+idx
            else:
                return idx

    def __setitem__(self, idx, value):
        if(isinstance(idx, ndarray)):
            idx = arrayfire.index(idx.d_array)
        elif(isinstance(idx, slice)):
            idx = self.__convert_dim__(idx,self.shape[0])
            idx = arrayfire.index(arrayfire.seq(float(idx.start),float(idx.stop),float(idx.step)))
        else:
            raise NotImplementedError('indices must be a afnumpy.ndarray')
        if(isinstance(value, ndarray)):
            if(value.dtype != self.dtype):
                raise TypeError('left hand side must have same dtype as right hand side')
            self.d_array.setValue(idx, value.d_array)
        elif(isinstance(value, numbers.Number)):
            self.d_array.setValue(idx, value)
        else:
            raise NotImplementedError('values must be a afnumpy.ndarray')

    def __array__(self):
        self.d_array.host(self.h_array.ctypes.data)
        return numpy.copy(self.h_array)

    def reshape(self, shape, order = 'C'):
        return reshape(self, shape, order)

    def max(self):
        type_max = getattr(arrayfire, 'max_'+__TypeToString__[self.d_array.type()])
        return type_max(self.d_array)

    def min(self):
        type_max = getattr(arrayfire, 'min_'+__TypeToString__[self.d_array.type()])
        return type_max(self.d_array)

#    def __getattr__(self,name):
#        print name
#        raise AttributeError

<|endoftext|>


<commit_before>from abstract import layer, _activate, _dropout
import numpy
import numpy.random as rng
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

class rotate_layer (layer):
    """
    This is a rotate layer. This takes a layer and an angle (rotation normalized in [0,1]) as input
    and rotates the batch of images by the specified rotation parameter.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        input_shape: ``(mini_batch_size, input_size)``
        angle: value from [0,1]
        borrow: ``theano`` borrow, typicall ``True``.
        input_params: Supply params or initializations from a pre-trained system.
    """

    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  angle = None,
                  borrow = True,
                  verbose = 2 ):
        super(rotate_layer,self).__init__(id = id, type = 'rotate', verbose = verbose)
        srng = RandomStreams(rng.randint(1,2147462468), use_cuda=None)
        if verbose >= 3:
            print "... Creating rotate layer"

        if input_shape == 4:
            if verbose >= 3:
                print "... Creating the rotate layer"

            if angle is None:
                angle = srng.uniform(size = input_shape[0], low =limits[0], high =limits[1],
                                                dtype = theano.config.floatX )

            theta = numpy.zeros((input_shape[0],2,3),dtype='float32')
            theta[:,0,0] = numpy.cos(angle[:,0]*180)
            theta[:,0,1] = -numpy.sin(angle[:,0]*180)
            theta[:,1,0] = numpy.sin(angle[:,0]*180)
            theta[:,1,1] = numpy.cos(angle[:,0]*180)
            theta = theta.reshape((input_shape[0], 6))

            self.output = self._transform_affine(theta, input)
            import pdb
            pdb.set_trace()
            self.output_shape = input_shape
            self.angle = angle

    def _transform_affine(self, theta, input):
        num_batch, num_channels, height, width = input.shape
        theta = T.reshape(theta, (-1, 2, 3))

        # grid of (x_t, y_t, 1)
        out_height = T.cast(height, 'int64')
        out_width = T.cast(width, 'int64')
        grid = _meshgrid(out_height, out_width)

        # Transform A x (x_t, y_t, 1)^T -> (x_s, y_s)
        T_g = T.dot(theta, grid)
        x_s = T_g[:, 0]
        y_s = T_g[:, 1]
        x_s_flat = x_s.flatten()
        y_s_flat = y_s.flatten()

        # dimshuffle input to  (bs, height, width, channels)
        input_dim = input.dimshuffle(0, 2, 3, 1)
        input_transformed = _interpolate(
            input_dim, x_s_flat, y_s_flat,
            out_height, out_width)

        output = T.reshape(
            input_transformed, (num_batch, out_height, out_width, num_channels))
        output = output.dimshuffle(0, 3, 1, 2)  # dimshuffle to conv format
        return output

    def _interpolate(self, im, x, y, out_height, out_width):
        # *_f are floats
        num_batch, height, width, channels = im.shape
        height_f = T.cast(height, theano.config.floatX)
        width_f = T.cast(width, theano.config.floatX)

        # clip coordinates to [-1, 1]
        x = T.clip(x, -1, 1)
        y = T.clip(y, -1, 1)

        # scale coordinates from [-1, 1] to [0, width/height - 1]
        x = (x + 1) / 2 * (width_f - 1)
        y = (y + 1) / 2 * (height_f - 1)

        # obtain indices of the 2x2 pixel neighborhood surrounding the coordinates;
        # we need those in floatX for interpolation and in int64 for indexing. for
        # indexing, we need to take care they do not extend past the image.
        x0_f = T.floor(x)
        y0_f = T.floor(y)
        x1_f = x0_f + 1
        y1_f = y0_f + 1
        x0 = T.cast(x0_f, 'int64')
        y0 = T.cast(y0_f, 'int64')
        x1 = T.cast(T.minimum(x1_f, width_f - 1), 'int64')
        y1 = T.cast(T.minimum(y1_f, height_f - 1), 'int64')

        # The input is [num_batch, height, width, channels]. We do the lookup in
        # the flattened input, i.e [num_batch*height*width, channels]. We need
        # to offset all indices to match the flat version
        dim2 = width
        dim1 = width*height
        base = T.repeat(
            T.arange(num_batch, dtype='int64')*dim1, out_height*out_width)
        base_y0 = base + y0*dim2
        base_y1 = base + y1*dim2
        idx_a = base_y0 + x0
        idx_b = base_y1 + x0
        idx_c = base_y0 + x1
        idx_d = base_y1 + x1

        # use indices to lookup pixels for all samples
        im_flat = im.reshape((-1, channels))
        Ia = im_flat[idx_a]
        Ib = im_flat[idx_b]
        Ic = im_flat[idx_c]
        Id = im_flat[idx_d]

        # calculate interpolated values
        wa = ((x1_f-x) * (y1_f-y)).dimshuffle(0, 'x')
        wb = ((x1_f-x) * (y-y0_f)).dimshuffle(0, 'x')
        wc = ((x-x0_f) * (y1_f-y)).dimshuffle(0, 'x')
        wd = ((x-x0_f) * (y-y0_f)).dimshuffle(0, 'x')
        output = T.sum([wa*Ia, wb*Ib, wc*Ic, wd*Id], axis=0)
        return output

    def _linspace(self, start, stop, num):
        # Theano linspace. Behaves similar to np.linspace
        start = T.cast(start, theano.config.floatX)
        stop = T.cast(stop, theano.config.floatX)
        num = T.cast(num, theano.config.floatX)
        step = (stop-start)/(num-1)
        return T.arange(num, dtype=theano.config.floatX)*step+start

    def _meshgrid(self, height, width):
        # This function is the grid generator.
        # It is equivalent to the following numpy code:
        #  x_t, y_t = np.meshgrid(np.linspace(-1, 1, width),
        #                         np.linspace(-1, 1, height))
        #  ones = np.ones(np.prod(x_t.shape))
        #  grid = np.vstack([x_t.flatten(), y_t.flatten(), ones])
        # It is implemented in Theano instead to support symbolic grid sizes.
        # Note: If the image size is known at layer construction time, we could
        # compute the meshgrid offline in numpy instead of doing it dynamically
        # in Theano. However, it hardly affected performance when we tried.
        x_t = T.dot(T.ones((height, 1)),
                    _linspace(-1.0, 1.0, width).dimshuffle('x', 0))
        y_t = T.dot(_linspace(-1.0, 1.0, height).dimshuffle(0, 'x'),
                    T.ones((1, width)))

        x_t_flat = x_t.reshape((1, -1))
        y_t_flat = y_t.reshape((1, -1))
        ones = T.ones_like(x_t_flat)
        grid = T.concatenate([x_t_flat, y_t_flat, ones], axis=0)
        return grid

class dropout_rotate_layer (rotate_layer):
    """
    This class is the typical dropout neural hidden layer and batch normalization layer. Called
    by the ``add_layer`` method in network class.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        num_neurons: number of neurons in the layer
        input_shape: ``(mini_batch_size, input_size)``
        batch_norm: If provided will be used, default is ``False``.
        rng: typically ``numpy.random``.
        borrow: ``theano`` borrow, typicall ``True``.
        dropout_rate: ``0.5`` is the default.

    Notes:
        Use ``dropout_rotate_layer.output`` and ``dropout_rotate_layer.output_shape`` from
        this class. ``L1`` and ``L2`` are also public and can also can be used for regularization.
        The class also has in public ``w``, ``b`` and ``alpha``
        which are also a list in ``params``, another property of this class.
    """
    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  rng = None,
                  dropout_rate = 0.5,
                  angle = None,
                  borrow = True,
                  verbose = 2):

        if verbose >= 3:
            print "... set up the dropout rotate layer"
        import pdb
            pdb.set_trace()
        if rng is None:
            rng = numpy.random
        super(dropout_rotate_layer, self).__init__ (
                                        input = input,
                                        input_shape = input_shape,
                                        id = id,
                                        borrow = borrow,
                                        verbose = verbose
                                        )
        if not dropout_rate == 0:
            self.output = _dropout(rng = rng,
                                params = self.output,
                                dropout_rate = dropout_rate)
        self.dropout_rate = dropout_rate
        if verbose >=3:
            print "... Dropped out"

if __name__ == '__main__':
    pass  <commit_msg>network.py edit<commit_after>from abstract import layer, _activate, _dropout
import numpy
import numpy.random as rng
import theano
import theano.tensor as T
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams

class rotate_layer (layer):
    """
    This is a rotate layer. This takes a layer and an angle (rotation normalized in [0,1]) as input
    and rotates the batch of images by the specified rotation parameter.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        input_shape: ``(mini_batch_size, input_size)``
        angle: value from [0,1]
        borrow: ``theano`` borrow, typicall ``True``.
        input_params: Supply params or initializations from a pre-trained system.
    """

    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  angle = None,
                  borrow = True,
                  verbose = 2 ):
        super(rotate_layer,self).__init__(id = id, type = 'rotate', verbose = verbose)
        srng = RandomStreams(rng.randint(1,2147462468), use_cuda=None)
        if verbose >= 3:
            print "... Creating rotate layer"

        if input_shape == 4:
            if verbose >= 3:
                print "... Creating the rotate layer"

            if angle is None:
                angle = srng.uniform(size = input_shape[0], low =limits[0], high =limits[1],
                                                dtype = theano.config.floatX )

            theta = numpy.zeros((input_shape[0],2,3),dtype='float32')
            theta[:,0,0] = numpy.cos(angle[:,0]*180)
            theta[:,0,1] = -numpy.sin(angle[:,0]*180)
            theta[:,1,0] = numpy.sin(angle[:,0]*180)
            theta[:,1,1] = numpy.cos(angle[:,0]*180)
            theta = theta.reshape((input_shape[0], 6))

            self.output = self._transform_affine(theta, input)
            import pdb
            pdb.set_trace()
            self.output_shape = input_shape
            self.angle = angle

    def _transform_affine(self, theta, input):
        num_batch, num_channels, height, width = input.shape
        theta = T.reshape(theta, (-1, 2, 3))

        # grid of (x_t, y_t, 1)
        out_height = T.cast(height, 'int64')
        out_width = T.cast(width, 'int64')
        grid = _meshgrid(out_height, out_width)

        # Transform A x (x_t, y_t, 1)^T -> (x_s, y_s)
        T_g = T.dot(theta, grid)
        x_s = T_g[:, 0]
        y_s = T_g[:, 1]
        x_s_flat = x_s.flatten()
        y_s_flat = y_s.flatten()

        # dimshuffle input to  (bs, height, width, channels)
        input_dim = input.dimshuffle(0, 2, 3, 1)
        input_transformed = _interpolate(
            input_dim, x_s_flat, y_s_flat,
            out_height, out_width)

        output = T.reshape(
            input_transformed, (num_batch, out_height, out_width, num_channels))
        output = output.dimshuffle(0, 3, 1, 2)  # dimshuffle to conv format
        return output

    def _interpolate(self, im, x, y, out_height, out_width):
        # *_f are floats
        num_batch, height, width, channels = im.shape
        height_f = T.cast(height, theano.config.floatX)
        width_f = T.cast(width, theano.config.floatX)

        # clip coordinates to [-1, 1]
        x = T.clip(x, -1, 1)
        y = T.clip(y, -1, 1)

        # scale coordinates from [-1, 1] to [0, width/height - 1]
        x = (x + 1) / 2 * (width_f - 1)
        y = (y + 1) / 2 * (height_f - 1)

        # obtain indices of the 2x2 pixel neighborhood surrounding the coordinates;
        # we need those in floatX for interpolation and in int64 for indexing. for
        # indexing, we need to take care they do not extend past the image.
        x0_f = T.floor(x)
        y0_f = T.floor(y)
        x1_f = x0_f + 1
        y1_f = y0_f + 1
        x0 = T.cast(x0_f, 'int64')
        y0 = T.cast(y0_f, 'int64')
        x1 = T.cast(T.minimum(x1_f, width_f - 1), 'int64')
        y1 = T.cast(T.minimum(y1_f, height_f - 1), 'int64')

        # The input is [num_batch, height, width, channels]. We do the lookup in
        # the flattened input, i.e [num_batch*height*width, channels]. We need
        # to offset all indices to match the flat version
        dim2 = width
        dim1 = width*height
        base = T.repeat(
            T.arange(num_batch, dtype='int64')*dim1, out_height*out_width)
        base_y0 = base + y0*dim2
        base_y1 = base + y1*dim2
        idx_a = base_y0 + x0
        idx_b = base_y1 + x0
        idx_c = base_y0 + x1
        idx_d = base_y1 + x1

        # use indices to lookup pixels for all samples
        im_flat = im.reshape((-1, channels))
        Ia = im_flat[idx_a]
        Ib = im_flat[idx_b]
        Ic = im_flat[idx_c]
        Id = im_flat[idx_d]

        # calculate interpolated values
        wa = ((x1_f-x) * (y1_f-y)).dimshuffle(0, 'x')
        wb = ((x1_f-x) * (y-y0_f)).dimshuffle(0, 'x')
        wc = ((x-x0_f) * (y1_f-y)).dimshuffle(0, 'x')
        wd = ((x-x0_f) * (y-y0_f)).dimshuffle(0, 'x')
        output = T.sum([wa*Ia, wb*Ib, wc*Ic, wd*Id], axis=0)
        return output

    def _linspace(self, start, stop, num):
        # Theano linspace. Behaves similar to np.linspace
        start = T.cast(start, theano.config.floatX)
        stop = T.cast(stop, theano.config.floatX)
        num = T.cast(num, theano.config.floatX)
        step = (stop-start)/(num-1)
        return T.arange(num, dtype=theano.config.floatX)*step+start

    def _meshgrid(self, height, width):
        # This function is the grid generator.
        # It is equivalent to the following numpy code:
        #  x_t, y_t = np.meshgrid(np.linspace(-1, 1, width),
        #                         np.linspace(-1, 1, height))
        #  ones = np.ones(np.prod(x_t.shape))
        #  grid = np.vstack([x_t.flatten(), y_t.flatten(), ones])
        # It is implemented in Theano instead to support symbolic grid sizes.
        # Note: If the image size is known at layer construction time, we could
        # compute the meshgrid offline in numpy instead of doing it dynamically
        # in Theano. However, it hardly affected performance when we tried.
        x_t = T.dot(T.ones((height, 1)),
                    _linspace(-1.0, 1.0, width).dimshuffle('x', 0))
        y_t = T.dot(_linspace(-1.0, 1.0, height).dimshuffle(0, 'x'),
                    T.ones((1, width)))

        x_t_flat = x_t.reshape((1, -1))
        y_t_flat = y_t.reshape((1, -1))
        ones = T.ones_like(x_t_flat)
        grid = T.concatenate([x_t_flat, y_t_flat, ones], axis=0)
        return grid

class dropout_rotate_layer (rotate_layer):
    """
    This class is the typical dropout neural hidden layer and batch normalization layer. Called
    by the ``add_layer`` method in network class.

    Args:
        input: An input ``theano.tensor`` variable. Even ``theano.shared`` will work as long as they
               are in the following shape ``mini_batch_size, height, width, channels``
        verbose: similar to the rest of the toolbox.
        num_neurons: number of neurons in the layer
        input_shape: ``(mini_batch_size, input_size)``
        batch_norm: If provided will be used, default is ``False``.
        rng: typically ``numpy.random``.
        borrow: ``theano`` borrow, typicall ``True``.
        dropout_rate: ``0.5`` is the default.

    Notes:
        Use ``dropout_rotate_layer.output`` and ``dropout_rotate_layer.output_shape`` from
        this class. ``L1`` and ``L2`` are also public and can also can be used for regularization.
        The class also has in public ``w``, ``b`` and ``alpha``
        which are also a list in ``params``, another property of this class.
    """
    def __init__ (self,
                  input,
                  input_shape,
                  id,
                  rng = None,
                  dropout_rate = 0.5,
                  angle = None,
                  borrow = True,
                  verbose = 2):

        if verbose >= 3:
            print "... set up the dropout rotate layer"
        import pdb
        pdb.set_trace()
        if rng is None:
            rng = numpy.random
        super(dropout_rotate_layer, self).__init__ (
                                        input = input,
                                        input_shape = input_shape,
                                        id = id,
                                        borrow = borrow,
                                        verbose = verbose
                                        )
        if not dropout_rate == 0:
            self.output = _dropout(rng = rng,
                                params = self.output,
                                dropout_rate = dropout_rate)
        self.dropout_rate = dropout_rate
        if verbose >=3:
            print "... Dropped out"

if __name__ == '__main__':
    pass  <|endoftext|>