<commit_before># -*- coding: utf-8  -*-
""" Script to create user files (user-config.py, user-fixes.py) """
__version__ = '$Id$'

import codecs, os, re, sys
import platform

def get_base_dir():
    """Return the directory in which user-specific information is stored.

    This is determined in the following order -
    1.  If the script was called with a -dir: argument, use the directory
        provided in this argument
    2.  If the user has a PYWIKIBOT2_DIR environment variable, use the value
        of it
    3.  Use (and if necessary create) a 'pywikibot' folder (Windows) or
        '.pywikibot' directory (Unix and similar) under the user's home
        directory.

    """
    # copied from config2.py, without the lines that check whether the
    # directory already contains a user-config.py file
    # this code duplication is nasty, should fix
    NAME = "pywikibot"
    for arg in sys.argv[1:]:
        if arg.startswith("-dir:"):
            base_dir = arg[5:]
            sys.argv.remove(arg)
            break
    else:
        if "PYWIKIBOT2_DIR" in os.environ:
            base_dir = os.environ["PYWIKIBOT2_DIR"]
        else:
            is_windows = sys.platform == 'win32'
            home = os.path.expanduser("~")
            if is_windows:
                _win_version = int(platform.version()[0])
                if _win_version == 5:
                    base_dir = os.path.join(home, "Application Data", NAME)
                elif _win_version == 6:
                    base_dir = os.path.join(home, "AppData\\Roaming", NAME)
            else:
                base_dir = os.path.join(home, "."+NAME)
            if not os.path.isdir(base_dir):
                os.makedirs(base_dir, mode=0700)
    if not os.path.isabs(base_dir):
        base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
    return base_dir

base_dir = get_base_dir()
console_encoding = sys.stdout.encoding
# the directory in which generate_user_files.py is located
pywikibot_dir = sys.path[0]

if console_encoding is None or sys.platform == 'cygwin':
    console_encoding = "iso-8859-1"

def listchoice(clist = [], message = None, default = None):

    if not message:
        message = "Select"

    if default:
        message += " (default: %s)" % default

    message += ": "

    for n, i in enumerate(clist):
        print ("%d: %s" % (n + 1, i))

    while True:
        choice = raw_input(message)

        if choice == '' and default:
            return default

        try:
            return clist[int(choice) - 1]
        except:
            print("Invalid response")
    return response

def change_base_dir():
    """Create a new user directory."""
    global base_dir
    while True:
        new_base = raw_input("New user directory? ")
        new_base = os.path.abspath(new_base)
        if os.path.exists(new_base):
            if os.path.isfile(new_base):
                print("ERROR: there is an existing file with that name.")
                continue
            # make sure user can read and write this directory
            if not os.access(new_base, os.R_OK|os.W_OK):
                print("ERROR: directory access restricted")
                continue
            print("OK: using existing directory")
            break
        else:
            try:
                os.mkdir(new_base, 0700)
            except Exception:
                print("ERROR: directory creation failed")
                continue
            print("OK: Created new directory.")
            break

    from textwrap import wrap
    msg = wrap("""WARNING: Your user files will be created in the directory
'%(new_base)s' you have chosen. To access these files, you will either have
to use the argument "-dir:%(new_base)s" every time you run the bot, or set
the environment variable "PYWIKIBOT2_DIR" equal to this directory name in
your operating system. See your operating system documentation for how to
set environment variables.""" % locals(), width=76)
    for line in msg:
        print line
    ok = raw_input("Is this OK? ([yes], [N]o) ")
    if ok in ["Y", "y"]:
        base_dir = new_base
        return True
    print "Aborting changes."
    return False

def file_exists(filename):
    if os.path.exists(filename):
        print("'%s' already exists." % filename)
        return True
    return False

def create_user_config():
    _fnc = os.path.join(base_dir, "user-config.py")
    if not file_exists(_fnc):
        known_families = re.findall(r'(.+)_family.py\b',
                                   '\n'.join(os.listdir(
                                       os.path.join(pywikibot_dir,
                                                    "pywikibot",
                                                    "families"))))
        fam = listchoice(known_families,
                         "Select family of sites we are working on",
                         default='wikipedia')
        mylang = raw_input(
"The language code of the site we're working on (default: 'en'): ") or 'en'
        username = raw_input("Username (%s %s): "
                             % (mylang, fam)) or 'UnnamedBot'
        username = unicode(username, console_encoding)
        while True:
            choice = raw_input(
"Which variant of user_config.py:\n[S]mall or [E]xtended (with further information)? "
                               ).upper()
            if choice in "SE":
                break

        #
        # I don't like this solution. Temporary for me.
        #
        # determine what directory this script (generate_user_files.py) lives in
        install = os.path.dirname(os.path.abspath(sys.argv[0]))
        # config2.py will be in the pywikibot/ directory
        f = codecs.open(os.path.join(install, "pywikibot", "config2.py"),
                        "r", "utf-8")
        cpy = f.read()
        f.close()

        res = re.findall("^(############## (?:LOGFILE|"
                                            "INTERWIKI|"
                                            "SOLVE_DISAMBIGUATION|"
                                            "IMAGE RELATED|"
                                            "TABLE CONVERSION BOT|"
                                            "WEBLINK CHECKER|"
                                            "DATABASE|"
                                            "SEARCH ENGINE|"
                                            "COPYRIGHT|"
                                            "FURTHER) SETTINGS .*?)^(?=#####|# =====)",
                         cpy, re.MULTILINE | re.DOTALL)
        config_text = '\n'.join(res)

        f = codecs.open(_fnc, "w", "utf-8")
        if choice == 'E':
            f.write("""# -*- coding: utf-8  -*-

# This is an automatically generated file. You can find more configuration
# parameters in 'config.py' file.

# The family of sites we are working on. wikipedia.py will import
# families/xxx_family.py so if you want to change this variable,
# you need to write such a file.
family = '%s'

# The language code of the site we're working on.
mylang = '%s'

# The dictionary usernames should contain a username for each site where you
# have a bot account.
usernames['%s']['%s'] = u'%s'

%s""" % (fam, mylang, fam, mylang, username, config_text))
        else:
            f.write("""# -*- coding: utf-8  -*-
family = '%s'
mylang = '%s'
usernames['%s']['%s'] = u'%s'
""" % (fam, mylang, fam, mylang, username))
        f.close()
        print("'%s' written." % _fnc)

def create_user_fixes():
    _fnf = os.path.join(base_dir, "user-fixes.py")
    if not file_exists(_fnf):
        f = codecs.open(_fnf, "w", "utf-8")
        f.write(r"""# -*- coding: utf-8  -*-

#
# This is only an example. Don't use it.
#

fixes['example'] = {
    'regex': True,
    'msg': {
        '_default':u'no summary specified',
    },
    'replacements': [
        (ur'\bword\b', u'two words'),
    ]
}

""")
        f.close()
        print("'%s' written." % _fnf)

if __name__ == "__main__":
    while True:
        print('\nYour default user directory is "%s"' % base_dir)
        ok = raw_input("How to proceed? ([K]eep [c]hange) ").upper().strip()
        if (not ok) or "KEEP".startswith(ok):
            break
        if "CHANGE".startswith(ok):
            if change_base_dir():
                break
    while True:
        if os.path.exists(os.path.join(base_dir, "user-config.py")):
            break
        do_copy = raw_input(
"Do you want to copy user files from an existing pywikipedia installation? "
                            ).upper().strip()
        if do_copy and "YES".startswith(do_copy):
            oldpath = raw_input("Path to existing wikipedia.py? ")
            if not os.path.exists(oldpath):
                print("ERROR: Not a valid path")
                continue
            if os.path.isfile(oldpath):
                # User probably typed /wikipedia.py at the end, so strip it
                oldpath = os.path.dirname(oldpath)
            if not os.path.isfile(os.path.join(oldpath, "user-config.py")):
                print("ERROR: no user_config.py found in that directory")
                continue
            newf = file(os.path.join(base_dir, "user-config.py"), "wb")
            oldf = file(os.path.join(oldpath, "user-config.py"), "rb")
            newf.write(oldf.read())
            newf.close(); oldf.close()
            if os.path.isfile(os.path.join(oldpath, "user-fixes.py")):
                newfix = file(os.path.join(base_dir, "user-fixes.py"), "wb")
                oldfix = file(os.path.join(oldpath, "user-fixes.py"), "rb")
                newfix.write(oldfix.read())
                newfix.close() ; oldfix.close()
        elif do_copy and "NO".startswith(do_copy):
            break
    if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
        a = raw_input("Create user-config.py file? ([y]es, [N]o) ")
        if a[:1] in ["Y", "y"]:
            create_user_config()
    else:
        print("NOTE: user-config.py already exists in the directory")
    if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")):
        a = raw_input("Create user-fixes.py file? ([y]es, [N]o) ")
        if a[:1] in ["Y", "y"]:
            create_user_fixes()
    else:
        print("NOTE: user-fixes.py already exists in the directory")
<commit_msg>+ MIT license<commit_after># -*- coding: utf-8  -*-
""" Script to create user files (user-config.py, user-fixes.py) """
#
# (C) Pywikipedia team, 2010-2012
#
__version__ = '$Id$'
#
# Distributed under the terms of the MIT license.
#

import codecs, os, re, sys
import platform

def get_base_dir():
    """Return the directory in which user-specific information is stored.

    This is determined in the following order -
    1.  If the script was called with a -dir: argument, use the directory
        provided in this argument
    2.  If the user has a PYWIKIBOT2_DIR environment variable, use the value
        of it
    3.  Use (and if necessary create) a 'pywikibot' folder (Windows) or
        '.pywikibot' directory (Unix and similar) under the user's home
        directory.

    """
    # copied from config2.py, without the lines that check whether the
    # directory already contains a user-config.py file
    # this code duplication is nasty, should fix
    NAME = "pywikibot"
    for arg in sys.argv[1:]:
        if arg.startswith("-dir:"):
            base_dir = arg[5:]
            sys.argv.remove(arg)
            break
    else:
        if "PYWIKIBOT2_DIR" in os.environ:
            base_dir = os.environ["PYWIKIBOT2_DIR"]
        else:
            is_windows = sys.platform == 'win32'
            home = os.path.expanduser("~")
            if is_windows:
                _win_version = int(platform.version()[0])
                if _win_version == 5:
                    base_dir = os.path.join(home, "Application Data", NAME)
                elif _win_version == 6:
                    base_dir = os.path.join(home, "AppData\\Roaming", NAME)
            else:
                base_dir = os.path.join(home, "."+NAME)
            if not os.path.isdir(base_dir):
                os.makedirs(base_dir, mode=0700)
    if not os.path.isabs(base_dir):
        base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
    return base_dir

base_dir = get_base_dir()
console_encoding = sys.stdout.encoding
# the directory in which generate_user_files.py is located
pywikibot_dir = sys.path[0]

if console_encoding is None or sys.platform == 'cygwin':
    console_encoding = "iso-8859-1"

def listchoice(clist = [], message = None, default = None):

    if not message:
        message = "Select"

    if default:
        message += " (default: %s)" % default

    message += ": "

    for n, i in enumerate(clist):
        print ("%d: %s" % (n + 1, i))

    while True:
        choice = raw_input(message)

        if choice == '' and default:
            return default

        try:
            return clist[int(choice) - 1]
        except:
            print("Invalid response")
    return response

def change_base_dir():
    """Create a new user directory."""
    global base_dir
    while True:
        new_base = raw_input("New user directory? ")
        new_base = os.path.abspath(new_base)
        if os.path.exists(new_base):
            if os.path.isfile(new_base):
                print("ERROR: there is an existing file with that name.")
                continue
            # make sure user can read and write this directory
            if not os.access(new_base, os.R_OK|os.W_OK):
                print("ERROR: directory access restricted")
                continue
            print("OK: using existing directory")
            break
        else:
            try:
                os.mkdir(new_base, 0700)
            except Exception:
                print("ERROR: directory creation failed")
                continue
            print("OK: Created new directory.")
            break

    from textwrap import wrap
    msg = wrap("""WARNING: Your user files will be created in the directory
'%(new_base)s' you have chosen. To access these files, you will either have
to use the argument "-dir:%(new_base)s" every time you run the bot, or set
the environment variable "PYWIKIBOT2_DIR" equal to this directory name in
your operating system. See your operating system documentation for how to
set environment variables.""" % locals(), width=76)
    for line in msg:
        print line
    ok = raw_input("Is this OK? ([yes], [N]o) ")
    if ok in ["Y", "y"]:
        base_dir = new_base
        return True
    print "Aborting changes."
    return False

def file_exists(filename):
    if os.path.exists(filename):
        print("'%s' already exists." % filename)
        return True
    return False

def create_user_config():
    _fnc = os.path.join(base_dir, "user-config.py")
    if not file_exists(_fnc):
        known_families = re.findall(r'(.+)_family.py\b',
                                   '\n'.join(os.listdir(
                                       os.path.join(pywikibot_dir,
                                                    "pywikibot",
                                                    "families"))))
        fam = listchoice(known_families,
                         "Select family of sites we are working on",
                         default='wikipedia')
        mylang = raw_input(
"The language code of the site we're working on (default: 'en'): ") or 'en'
        username = raw_input("Username (%s %s): "
                             % (mylang, fam)) or 'UnnamedBot'
        username = unicode(username, console_encoding)
        while True:
            choice = raw_input(
"Which variant of user_config.py:\n[S]mall or [E]xtended (with further information)? "
                               ).upper()
            if choice in "SE":
                break

        #
        # I don't like this solution. Temporary for me.
        #
        # determine what directory this script (generate_user_files.py) lives in
        install = os.path.dirname(os.path.abspath(sys.argv[0]))
        # config2.py will be in the pywikibot/ directory
        f = codecs.open(os.path.join(install, "pywikibot", "config2.py"),
                        "r", "utf-8")
        cpy = f.read()
        f.close()

        res = re.findall("^(############## (?:LOGFILE|"
                                            "INTERWIKI|"
                                            "SOLVE_DISAMBIGUATION|"
                                            "IMAGE RELATED|"
                                            "TABLE CONVERSION BOT|"
                                            "WEBLINK CHECKER|"
                                            "DATABASE|"
                                            "SEARCH ENGINE|"
                                            "COPYRIGHT|"
                                            "FURTHER) SETTINGS .*?)^(?=#####|# =====)",
                         cpy, re.MULTILINE | re.DOTALL)
        config_text = '\n'.join(res)

        f = codecs.open(_fnc, "w", "utf-8")
        if choice == 'E':
            f.write("""# -*- coding: utf-8  -*-

# This is an automatically generated file. You can find more configuration
# parameters in 'config.py' file.

# The family of sites we are working on. wikipedia.py will import
# families/xxx_family.py so if you want to change this variable,
# you need to write such a file.
family = '%s'

# The language code of the site we're working on.
mylang = '%s'

# The dictionary usernames should contain a username for each site where you
# have a bot account.
usernames['%s']['%s'] = u'%s'

%s""" % (fam, mylang, fam, mylang, username, config_text))
        else:
            f.write("""# -*- coding: utf-8  -*-
family = '%s'
mylang = '%s'
usernames['%s']['%s'] = u'%s'
""" % (fam, mylang, fam, mylang, username))
        f.close()
        print("'%s' written." % _fnc)

def create_user_fixes():
    _fnf = os.path.join(base_dir, "user-fixes.py")
    if not file_exists(_fnf):
        f = codecs.open(_fnf, "w", "utf-8")
        f.write(r"""# -*- coding: utf-8  -*-

#
# This is only an example. Don't use it.
#

fixes['example'] = {
    'regex': True,
    'msg': {
        '_default':u'no summary specified',
    },
    'replacements': [
        (ur'\bword\b', u'two words'),
    ]
}

""")
        f.close()
        print("'%s' written." % _fnf)

if __name__ == "__main__":
    while True:
        print('\nYour default user directory is "%s"' % base_dir)
        ok = raw_input("How to proceed? ([K]eep [c]hange) ").upper().strip()
        if (not ok) or "KEEP".startswith(ok):
            break
        if "CHANGE".startswith(ok):
            if change_base_dir():
                break
    while True:
        if os.path.exists(os.path.join(base_dir, "user-config.py")):
            break
        do_copy = raw_input(
"Do you want to copy user files from an existing pywikipedia installation? "
                            ).upper().strip()
        if do_copy and "YES".startswith(do_copy):
            oldpath = raw_input("Path to existing wikipedia.py? ")
            if not os.path.exists(oldpath):
                print("ERROR: Not a valid path")
                continue
            if os.path.isfile(oldpath):
                # User probably typed /wikipedia.py at the end, so strip it
                oldpath = os.path.dirname(oldpath)
            if not os.path.isfile(os.path.join(oldpath, "user-config.py")):
                print("ERROR: no user_config.py found in that directory")
                continue
            newf = file(os.path.join(base_dir, "user-config.py"), "wb")
            oldf = file(os.path.join(oldpath, "user-config.py"), "rb")
            newf.write(oldf.read())
            newf.close(); oldf.close()
            if os.path.isfile(os.path.join(oldpath, "user-fixes.py")):
                newfix = file(os.path.join(base_dir, "user-fixes.py"), "wb")
                oldfix = file(os.path.join(oldpath, "user-fixes.py"), "rb")
                newfix.write(oldfix.read())
                newfix.close() ; oldfix.close()
        elif do_copy and "NO".startswith(do_copy):
            break
    if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
        a = raw_input("Create user-config.py file? ([y]es, [N]o) ")
        if a[:1] in ["Y", "y"]:
            create_user_config()
    else:
        print("NOTE: user-config.py already exists in the directory")
    if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")):
        a = raw_input("Create user-fixes.py file? ([y]es, [N]o) ")
        if a[:1] in ["Y", "y"]:
            create_user_fixes()
    else:
        print("NOTE: user-fixes.py already exists in the directory")
<|endoftext|>


<commit_before>"""Convert a set of mosaics into a video or animated gif."""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as pl

import os
import click

from astropy import visualization
import fitsio
import imageio
import numpy as np

from . import KEPLER_CHANNEL_SHAPE

class InvalidFrameException(Exception):
    pass

class KeplerMosaicMovieFrame(object):

    def __init__(self, fits_filename):
        self.fits_filename = fits_filename

    def to_fig(self, rowrange, colrange, extension=1, cmap='Greys_r', cut=None, dpi=50):
        """Turns a fits file into a cropped and contrast-stretched matplotlib figure."""
        fts = fitsio.FITS(self.fits_filename)
        if (-np.isnan(fts[extension].read())).sum() == 0:
            raise InvalidFrameException()
        image = fts[extension].read()[rowrange[0]:rowrange[1], colrange[0]:colrange[1]]
        if cut is None:
            cut = np.percentile(image[-np.isnan(image)], [10, 99.5])
        image_scaled = visualization.scale_image(image, scale="log",
                                                 min_cut=cut[0], max_cut=cut[1]) #min_percent=0.5, max_percent=99.5)

        px_per_kepler_px = 20
        dimensions = [image.shape[0] * px_per_kepler_px, image.shape[1] * px_per_kepler_px]
        figsize = [dimensions[1]/dpi, dimensions[0]/dpi]
        dpi = 440 / float(figsize[0])
        fig = pl.figure(figsize=figsize, dpi=dpi)
        ax = fig.add_subplot(1, 1, 1, axisbg='green')
        ax.matshow(image_scaled, aspect='auto',
                   cmap=cmap, origin='lower',
                   interpolation='nearest')
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')
        #ax.set_axis_bgcolor('red')
        fig.subplots_adjust(left=0.0, right=1.0, top=1.0, bottom=0.0)
        fig.canvas.draw()
        return fig

class KeplerMosaicMovie(object):

    def __init__(self, mosaic_filenames,
                 rowrange=(0, KEPLER_CHANNEL_SHAPE[0]),
                 colrange=(0, KEPLER_CHANNEL_SHAPE[1])):
        self.mosaic_filenames = mosaic_filenames
        self.rowrange = rowrange
        self.colrange = colrange

    def get_frame(self, frame_number=0):
        return KeplerMosaicMovieFrame(self.mosaic_filenames[frame_number])

    def export_frames(self, extension=1, cut=None):
        for fn in click.progressbar(self.mosaic_filenames, label="Reading mosaics", show_pos=True):
            try:
                frame = KeplerMosaicMovieFrame(fn)
                fig = frame.to_fig(rowrange=self.rowrange, colrange=self.colrange, extension=extension, cut=cut)
                out_fn = "movie-frame-" + os.path.basename(fn) + ".png"
                fig.savefig(out_fn, cmap='Greys_r', facecolor='#333333')
                pl.close(fig)
            except InvalidFrameException:
                print("InvalidFrameException for {}".format(fn))

    def to_movie(self, output_fn, fps=15., dpi=50, cut=None, cmap='gray', extension=1):
        viz = []
        with click.progressbar(self.mosaic_filenames, label="Reading mosaics", show_pos=True) as bar:
            for fn in bar:
                try:
                    frame = KeplerMosaicMovieFrame(fn)
                    fig = frame.to_fig(rowrange=self.rowrange, colrange=self.colrange,
                                       dpi=dpi, cut=cut, cmap=cmap, extension=extension,)
                    img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
                    img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,))
                    pl.close(fig)  # Avoid memory leak!
                    viz.append(img)
                except InvalidFrameException:
                    print("InvalidFrameException for {}".format(fn))
                    # Save the output as a movie
        if output_fn.endswith('.gif'):
            kwargs = {'duration': 1. / fps}
        else:
            kwargs = {'fps': fps}
        imageio.mimsave(output_fn, viz, **kwargs)

    def save_movie(self, output_fn=None, start=None, stop=None, step=None,
                   fps=15., dpi=None, min_percent=1., max_percent=95.,
                   cmap='gray', time_format='ut', show_flags=False, raw=False,
                   ignore_bad_frames=True,):
        """Save an animation.

        Parameters
        ----------
        output_fn : str
            The filename of the output movie.  The type of the movie
            is determined from the filename (e.g. use '.gif' to save
            as an animated gif). The default is a GIF file with the same name
            as the input FITS file.

        start : int
            Number or time of the first frame to show.
            If `None` (default), the first frame will be the start of the data.

        stop : int
            Number or time of the last frame to show.
            If `None` (default), the last frame will be the end of the data.

        step : int
            Spacing between frames.  Default is to set the stepsize
            automatically such that the movie contains 100 frames between
            start and stop.

        fps : float (optional)
            Frames per second.  Default is 15.0.

        dpi : float (optional)
            Resolution of the output in dots per Kepler pixel.
            The default is to produce output which is 440px wide.

        min_percent : float, optional
            The percentile value used to determine the pixel value of
            minimum cut level.  The default is 1.0.

        max_percent : float, optional
            The percentile value used to determine the pixel value of
            maximum cut level.  The default is 95.0.

        cmap : str, optional
            The matplotlib color map name.  The default is 'gray',
            can also be e.g. 'gist_heat'.

        raw : boolean, optional
            If `True`, show the raw pixel counts rather than
            the calibrated flux. Default: `False`.

        show_flags : boolean, optional
            If `True`, annotate the quality flags if set.

        ignore_bad_frames : boolean, optional
             If `True`, any frames which cannot be rendered will be ignored
             without raising a ``BadKeplerFrame`` exception. Default: `True`.
        """
        if output_fn is None:
            output_fn = self.mosaic_filenames[0].split('/')[-1] + '.gif'
        # Determine cut levels for contrast stretching from a sample of pixels
        vmin, vmax = self.cut_levels(min_percent=min_percent,
                                     max_percent=max_percent,
                                     raw=raw)
        # Determine the first/last frame number and the step size
        frameno_start, frameno_stop = self._frameno_range(start, stop, time_format)
        if step is None:
            step = int((frameno_stop - frameno_start) / 100)
            if step < 1:
                step = 1
        # Create the movie frames
        print('Creating {0}'.format(output_fn))
        viz = []
        for frameno in click.progressbar(np.arange(frameno_start, frameno_stop + 1, step, dtype=int)):
            try:
                fig = self.create_figure(frameno=frameno, dpi=dpi,
                                         vmin=vmin, vmax=vmax, cmap=cmap,
                                         raw=raw, time_format=time_format,
                                         show_flags=show_flags)
                img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
                img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,))
                pl.close(fig)  # Avoids memory leak!
                viz.append(img)
            except BadKeplerFrame as e:
                log.debug(e)
                if not ignore_bad_frames:
                    raise e
        # Save the output as a movie
        if output_fn.endswith('.gif'):
            kwargs = {'duration': 1. / fps}
        else:
            kwargs = {'fps': fps}
        imageio.mimsave(output_fn, viz, **kwargs)

    def to_gif(self):
        pass

    def to_mp4(self):
        pass
<commit_msg>no context manager anymore<commit_after>"""Convert a set of mosaics into a video or animated gif."""
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as pl

import os
import click

from astropy import visualization
import fitsio
import imageio
import numpy as np

from . import KEPLER_CHANNEL_SHAPE

class InvalidFrameException(Exception):
    pass

class KeplerMosaicMovieFrame(object):

    def __init__(self, fits_filename):
        self.fits_filename = fits_filename

    def to_fig(self, rowrange, colrange, extension=1, cmap='Greys_r', cut=None, dpi=50):
        """Turns a fits file into a cropped and contrast-stretched matplotlib figure."""
        fts = fitsio.FITS(self.fits_filename)
        if (-np.isnan(fts[extension].read())).sum() == 0:
            raise InvalidFrameException()
        image = fts[extension].read()[rowrange[0]:rowrange[1], colrange[0]:colrange[1]]
        fts.close()
        if cut is None:
            cut = np.percentile(image[-np.isnan(image)], [10, 99.5])
        image_scaled = visualization.scale_image(image, scale="log",
                                                 min_cut=cut[0], max_cut=cut[1]) #min_percent=0.5, max_percent=99.5)

        px_per_kepler_px = 20
        dimensions = [image.shape[0] * px_per_kepler_px, image.shape[1] * px_per_kepler_px]
        figsize = [dimensions[1]/dpi, dimensions[0]/dpi]
        dpi = 440 / float(figsize[0])
        fig = pl.figure(figsize=figsize, dpi=dpi)
        ax = fig.add_subplot(1, 1, 1, axisbg='green')
        ax.matshow(image_scaled, aspect='auto',
                   cmap=cmap, origin='lower',
                   interpolation='nearest')
        ax.set_xticks([])
        ax.set_yticks([])
        ax.axis('off')
        #ax.set_axis_bgcolor('red')
        fig.subplots_adjust(left=0.0, right=1.0, top=1.0, bottom=0.0)
        fig.canvas.draw()
        return fig

class KeplerMosaicMovie(object):

    def __init__(self, mosaic_filenames,
                 rowrange=(0, KEPLER_CHANNEL_SHAPE[0]),
                 colrange=(0, KEPLER_CHANNEL_SHAPE[1])):
        self.mosaic_filenames = mosaic_filenames
        self.rowrange = rowrange
        self.colrange = colrange

    def get_frame(self, frame_number=0):
        return KeplerMosaicMovieFrame(self.mosaic_filenames[frame_number])

    def export_frames(self, extension=1, cut=None):
        for fn in click.progressbar(self.mosaic_filenames, label="Reading mosaics", show_pos=True):
            try:
                frame = KeplerMosaicMovieFrame(fn)
                fig = frame.to_fig(rowrange=self.rowrange, colrange=self.colrange, extension=extension, cut=cut)
                out_fn = "movie-frame-" + os.path.basename(fn) + ".png"
                fig.savefig(out_fn, cmap='Greys_r', facecolor='#333333')
                pl.close(fig)
            except InvalidFrameException:
                print("InvalidFrameException for {}".format(fn))

    def to_movie(self, output_fn, fps=15., dpi=50, cut=None, cmap='gray', extension=1):
        viz = []
        with click.progressbar(self.mosaic_filenames, label="Reading mosaics", show_pos=True) as bar:
            for fn in bar:
                try:
                    frame = KeplerMosaicMovieFrame(fn)
                    fig = frame.to_fig(rowrange=self.rowrange, colrange=self.colrange,
                                       dpi=dpi, cut=cut, cmap=cmap, extension=extension,)
                    img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
                    img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,))
                    pl.close(fig)  # Avoid memory leak!
                    viz.append(img)
                except InvalidFrameException:
                    print("InvalidFrameException for {}".format(fn))
                    # Save the output as a movie
        if output_fn.endswith('.gif'):
            kwargs = {'duration': 1. / fps}
        else:
            kwargs = {'fps': fps}
        imageio.mimsave(output_fn, viz, **kwargs)

    def save_movie(self, output_fn=None, start=None, stop=None, step=None,
                   fps=15., dpi=None, min_percent=1., max_percent=95.,
                   cmap='gray', time_format='ut', show_flags=False, raw=False,
                   ignore_bad_frames=True,):
        """Save an animation.

        Parameters
        ----------
        output_fn : str
            The filename of the output movie.  The type of the movie
            is determined from the filename (e.g. use '.gif' to save
            as an animated gif). The default is a GIF file with the same name
            as the input FITS file.

        start : int
            Number or time of the first frame to show.
            If `None` (default), the first frame will be the start of the data.

        stop : int
            Number or time of the last frame to show.
            If `None` (default), the last frame will be the end of the data.

        step : int
            Spacing between frames.  Default is to set the stepsize
            automatically such that the movie contains 100 frames between
            start and stop.

        fps : float (optional)
            Frames per second.  Default is 15.0.

        dpi : float (optional)
            Resolution of the output in dots per Kepler pixel.
            The default is to produce output which is 440px wide.

        min_percent : float, optional
            The percentile value used to determine the pixel value of
            minimum cut level.  The default is 1.0.

        max_percent : float, optional
            The percentile value used to determine the pixel value of
            maximum cut level.  The default is 95.0.

        cmap : str, optional
            The matplotlib color map name.  The default is 'gray',
            can also be e.g. 'gist_heat'.

        raw : boolean, optional
            If `True`, show the raw pixel counts rather than
            the calibrated flux. Default: `False`.

        show_flags : boolean, optional
            If `True`, annotate the quality flags if set.

        ignore_bad_frames : boolean, optional
             If `True`, any frames which cannot be rendered will be ignored
             without raising a ``BadKeplerFrame`` exception. Default: `True`.
        """
        if output_fn is None:
            output_fn = self.mosaic_filenames[0].split('/')[-1] + '.gif'
        # Determine cut levels for contrast stretching from a sample of pixels
        vmin, vmax = self.cut_levels(min_percent=min_percent,
                                     max_percent=max_percent,
                                     raw=raw)
        # Determine the first/last frame number and the step size
        frameno_start, frameno_stop = self._frameno_range(start, stop, time_format)
        if step is None:
            step = int((frameno_stop - frameno_start) / 100)
            if step < 1:
                step = 1
        # Create the movie frames
        print('Creating {0}'.format(output_fn))
        viz = []
        for frameno in click.progressbar(np.arange(frameno_start, frameno_stop + 1, step, dtype=int)):
            try:
                fig = self.create_figure(frameno=frameno, dpi=dpi,
                                         vmin=vmin, vmax=vmax, cmap=cmap,
                                         raw=raw, time_format=time_format,
                                         show_flags=show_flags)
                img = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
                img = img.reshape(fig.canvas.get_width_height()[::-1] + (3,))
                pl.close(fig)  # Avoids memory leak!
                viz.append(img)
            except BadKeplerFrame as e:
                log.debug(e)
                if not ignore_bad_frames:
                    raise e
        # Save the output as a movie
        if output_fn.endswith('.gif'):
            kwargs = {'duration': 1. / fps}
        else:
            kwargs = {'fps': fps}
        imageio.mimsave(output_fn, viz, **kwargs)

    def to_gif(self):
        pass

    def to_mp4(self):
        pass
<|endoftext|>


<commit_before>from kafka.client import KafkaClient
from kafka.consumer import SimpleConsumer
from influxdb import InfluxDBClient # TODO import dynamically. If version 0.9 we have different requirements!
import json
import argparse
import time
from collections import defaultdict

DB_VERSION_DEFAULT = 0.8
DB_VERSION_APICHANGE = 0.9

#	python kafka_influxdb.py --kafka_topic perfmonspring1 --influxdb_user root --influxdb_password root --influxdb_dbname mydb --influxdb_version 0.9 --kafka_host 10.1.3.234 --influxdb_host springfield02.local --buffer_size 2
# TODO remove ^
# TODO test if old version still works

class InfluxDBData09(object):
	def __init__(self, database, retention_policy):
		self.database = database
		if retention_policy != u'':
			self.retentionPolicy = retention_policy
		self.points = []

	def add_points(self, points):
		self.points = self.points + points

	def reset(self):
		self.points = []

class InfluxDBData(object):
	def __init__(self, name, columns):
		self.name = name
		self.columns = columns
		self.points = []

	def add_point(self, *point):
		self.points.append(list(point))

	def reset(self):
		self.points = []

def main(config):
	# Kafka settings
	kafka = KafkaClient("{0}:{1}".format(config.kafka_host, config.kafka_port))

	client = InfluxDBClient(config.influxdb_host,
				config.influxdb_port,
				config.influxdb_user,
				config.influxdb_password,
				config.influxdb_dbname)

	try:
		db_version = float(config.influxdb_version)
		version_0_9 = db_version >= DB_VERSION_APICHANGE
	except:
		version_0_9 = False

	# Consume messages
	consumer = SimpleConsumer(kafka, config.kafka_group, config.kafka_topic)

	if version_0_9:
		stats = InfluxDBData09(config.influxdb_dbname, config.influxdb_retention_policy)
	else:
		stats = InfluxDBData(config.influxdb_data_name, config.influxdb_columns)
	i = 0
	count_flush = 0
	count_kafka = 0
	count_datapoints = 0
	starttime = get_millis()
	print "Version 9: %s" % version_0_9
	for message in consumer:
		i = i + 1
		val = message.message.value
		if version_0_9:
			transformed = transform_to_0_9(val)
			stats.add_points(transformed)
			if config.statistics:
				count_datapoints = count_datapoints + len(transformed)

		else:
			stats.add_point(val)
		if i == config.buffer_size or config.buffer_size == 0:
			if version_0_9:
				data = stats.points
			else:
				data = [stats.__dict__]
			client.write_points(data)

			if config.statistics:
				count_kafka = count_kafka + i
				count_flush = count_flush + 1
				now = get_millis()
				time_elapsed_ms = (now - starttime)
				if config.verbose:
					print "Time elapsed: %d - %d = %d" % (now, starttime, time_elapsed_ms)
				kafka_per_ms = (float(count_kafka) / time_elapsed_ms)
				kafka_per_sec = kafka_per_ms * 1000
				if config.verbose:
					print "Count kafka: %d / %d = %d * 1000 = %d" % (count_kafka, time_elapsed_ms , kafka_per_ms, kafka_per_sec)
				print "Flush %d. Buffer size %d. Parsed %d kafka into %d datapoints. Speed: %d kafka/sec" % (count_flush, config.buffer_size, count_kafka, count_datapoints, kafka_per_sec)
			i = 0
			stats.reset()

	kafka.close()

def get_millis():
	return int(round(time.time() * 1000))

def transform_to_0_9(kafka_message): # TODO add error handling
	results = []
	for json_obj in json.loads(kafka_message):
		timestamp = int(json_obj['time'])
		tags = {}
		tags['host'] = json_obj['host']
		if json_obj['plugin_instance'] != u'':
			tags['plugin_instance'] = json_obj['plugin_instance']
		if json_obj['type_instance'] != u'':
			tags['type_instance'] = json_obj['type_instance']
		if json_obj['type'] != u'':
			tags['type'] = json_obj['type']
		for i in range (0, len(json_obj['values'])):
			# TODO check that range is defined correctly (borders)
			new_point = {"precision":"s"}
			new_point["name"] = json_obj['plugin']
			new_point["timestamp"] = timestamp
			new_point["tags"] = tags
			# TODO append i indexed dstype and dsvalue if not empty to tags and check that that really works
			new_point["fields"] = {"value" : json_obj['values'][i]}
			results.append(new_point)

	return results

def parse_args():
	parser = argparse.ArgumentParser(description='A Kafka consumer for InfluxDB',
					formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument('--kafka_host', type=str, default='localhost', required=False)
	parser.add_argument('--kafka_port', type=int, default=9092, required=False)
	parser.add_argument('--kafka_topic', type=str, default='test', required=False)
	parser.add_argument('--kafka_group', type=str, default='my_group', required=False)
	parser.add_argument('--influxdb_host', type=str, default='localhost', required=False)
	parser.add_argument('--influxdb_port', type=int, default=8086, required=False)
	parser.add_argument('--influxdb_user', type=str, default='root', required=False)
	parser.add_argument('--influxdb_password', type=str, default='root', required=False)
	parser.add_argument('--influxdb_dbname', type=str, default='kafka', required=False)
	parser.add_argument('--influxdb_data_name', type=str, default='statsd', required=False)
	parser.add_argument('--influxdb_columns', type=str, default=['counter'], required=False)
	parser.add_argument('--influxdb_version', type=str, default=DB_VERSION_DEFAULT, required=False)
	parser.add_argument('--buffer_size', type=int, default=1000, required=False)
	parser.add_argument('--influxdb_retention_policy', type=str, default='', required=False)
	parser.add_argument('--verbose', type=bool, default=False, required=False)
	parser.add_argument('--statistics', type=bool, default=False, required=False)
	return parser.parse_args()

if __name__ == '__main__':
	args = parse_args()
	main(args)
<commit_msg>Statistics output improved<commit_after>from kafka.client import KafkaClient
from kafka.consumer import SimpleConsumer
from influxdb import InfluxDBClient # TODO import dynamically. If version 0.9 we have different requirements!
import json
import argparse
import time
from collections import defaultdict

DB_VERSION_DEFAULT = 0.8
DB_VERSION_APICHANGE = 0.9

#	python kafka_influxdb.py --kafka_topic perfmonspring1 --influxdb_user root --influxdb_password root --influxdb_dbname mydb --influxdb_version 0.9 --kafka_host 10.1.3.234 --influxdb_host springfield02.local --buffer_size 2
# TODO remove ^
# TODO test if old version still works

class InfluxDBData09(object):
	def __init__(self, database, retention_policy):
		self.database = database
		if retention_policy != u'':
			self.retentionPolicy = retention_policy
		self.points = []

	def add_points(self, points):
		self.points = self.points + points

	def reset(self):
		self.points = []

class InfluxDBData(object):
	def __init__(self, name, columns):
		self.name = name
		self.columns = columns
		self.points = []

	def add_point(self, *point):
		self.points.append(list(point))

	def reset(self):
		self.points = []

def main(config):
	# Kafka settings
	kafka = KafkaClient("{0}:{1}".format(config.kafka_host, config.kafka_port))

	client = InfluxDBClient(config.influxdb_host,
				config.influxdb_port,
				config.influxdb_user,
				config.influxdb_password,
				config.influxdb_dbname)

	try:
		db_version = float(config.influxdb_version)
		version_0_9 = db_version >= DB_VERSION_APICHANGE
	except:
		version_0_9 = False

	# Consume messages
	consumer = SimpleConsumer(kafka, config.kafka_group, config.kafka_topic)

	if version_0_9:
		stats = InfluxDBData09(config.influxdb_dbname, config.influxdb_retention_policy)
	else:
		stats = InfluxDBData(config.influxdb_data_name, config.influxdb_columns)
	i = 0
	count_flush = 0
	count_kafka = 0
	count_datapoints = 0
	starttime = get_millis()
	print "Version 9: %s" % version_0_9
	for message in consumer:
		i = i + 1
		val = message.message.value
		if version_0_9:
			transformed = transform_to_0_9(val)
			stats.add_points(transformed)
			if config.statistics:
				count_datapoints = count_datapoints + len(transformed)

		else:
			stats.add_point(val)
		if i == config.buffer_size or config.buffer_size == 0:
			if version_0_9:
				data = stats.points
			else:
				data = [stats.__dict__]
			client.write_points(data)

			if config.statistics:
				count_kafka = count_kafka + i
				count_flush = count_flush + 1
				now = get_millis()
				time_elapsed_ms = (now - starttime)
				if config.verbose:
					print "Time elapsed: %d - %d = %d" % (now, starttime, time_elapsed_ms)
				kafka_per_ms = (float(count_kafka) / time_elapsed_ms)
				kafka_per_sec = kafka_per_ms * 1000
				if config.verbose:
					print "Count kafka: %d / %d = %d * 1000 = %d" % (count_kafka, time_elapsed_ms , kafka_per_ms, kafka_per_sec)
				print "Flush %d. Buffer size %d. Parsed %d kafka mes. into %d datapoints. Speed: %d kafka mes./sec" % (count_flush, config.buffer_size, count_kafka, count_datapoints, kafka_per_sec)
			i = 0
			stats.reset()

	kafka.close()

def get_millis():
	return int(round(time.time() * 1000))

def transform_to_0_9(kafka_message): # TODO add error handling
	results = []
	for json_obj in json.loads(kafka_message):
		timestamp = int(json_obj['time'])
		tags = {}
		tags['host'] = json_obj['host']
		if json_obj['plugin_instance'] != u'':
			tags['plugin_instance'] = json_obj['plugin_instance']
		if json_obj['type_instance'] != u'':
			tags['type_instance'] = json_obj['type_instance']
		if json_obj['type'] != u'':
			tags['type'] = json_obj['type']
		for i in range (0, len(json_obj['values'])):
			# TODO check that range is defined correctly (borders)
			new_point = {"precision":"s"}
			new_point["name"] = json_obj['plugin']
			new_point["timestamp"] = timestamp
			new_point["tags"] = tags
			# TODO append i indexed dstype and dsvalue if not empty to tags and check that that really works
			new_point["fields"] = {"value" : json_obj['values'][i]}
			results.append(new_point)

	return results

def parse_args():
	parser = argparse.ArgumentParser(description='A Kafka consumer for InfluxDB',
					formatter_class=argparse.ArgumentDefaultsHelpFormatter)
	parser.add_argument('--kafka_host', type=str, default='localhost', required=False)
	parser.add_argument('--kafka_port', type=int, default=9092, required=False)
	parser.add_argument('--kafka_topic', type=str, default='test', required=False)
	parser.add_argument('--kafka_group', type=str, default='my_group', required=False)
	parser.add_argument('--influxdb_host', type=str, default='localhost', required=False)
	parser.add_argument('--influxdb_port', type=int, default=8086, required=False)
	parser.add_argument('--influxdb_user', type=str, default='root', required=False)
	parser.add_argument('--influxdb_password', type=str, default='root', required=False)
	parser.add_argument('--influxdb_dbname', type=str, default='kafka', required=False)
	parser.add_argument('--influxdb_data_name', type=str, default='statsd', required=False)
	parser.add_argument('--influxdb_columns', type=str, default=['counter'], required=False)
	parser.add_argument('--influxdb_version', type=str, default=DB_VERSION_DEFAULT, required=False)
	parser.add_argument('--buffer_size', type=int, default=1000, required=False)
	parser.add_argument('--influxdb_retention_policy', type=str, default='', required=False)
	parser.add_argument('--verbose', type=bool, default=False, required=False)
	parser.add_argument('--statistics', type=bool, default=False, required=False)
	return parser.parse_args()

if __name__ == '__main__':
	args = parse_args()
	main(args)
<|endoftext|>


<commit_before>"""
Copyright 2009 Chris Tarttelin and Point2 Technologies

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list
of conditions and the following disclaimer in the documentation and/or other materials
provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those of the
authors and should not be interpreted as representing official policies, either expressed
or implied, of the FreeBSD Project.
"""

import json, time
from datetime import datetime
from common_models import *

class BaseField:
    def __init__(self, **kw):
        if not kw.has_key('path'):
            raise Exception('No Path supplied for json field')
        self.path = kw['path']
        self._default = kw.pop('default', None)

    def get_nested_value(self, data, nodes):
        node = nodes.pop(0)
        if len(nodes) > 0:
            find = self.get_nested_value(getattr(data, node),nodes)
        else:
            find = getattr(data, node)
        if ((find == None) or (find == {})):
            return self._default
        else:
            return find

    def _parse(self, json_data):
        nodes = self.path.split('.')
        return self.get_nested_value(json_data, nodes)

    def save(self,value):
        return value

class CharField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class IntField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class BoolField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class DateField(BaseField):
    def parse(self, json_data):
        microseconds_from_epoch = self._parse(json_data)
        return microseconds_from_epoch and datetime.utcfromtimestamp(microseconds_from_epoch / 1000.0) or None

    def save(self,value):
        return value and (long)((time.mktime(value.utctimetuple()) - time.timezone) * 1000.0 + value.microsecond / 1000.0) or None

class Collection(BaseField):
    def __init__(self, field_type, order_by=None, **kw):
        self.field_type = field_type
        self.order_by = order_by
        BaseField.__init__(self, **kw)

    def parse(self, json_data):
        results = []
        matches = self._parse(json_data)
        if not BaseField in self.field_type.__bases__:
            results = []
            for match in matches:
                if isinstance(match, Model):
                    results.append(self.field_type(json=match._json))
                else:
                    results.append(self.field_type(json=match))
        elif matches:
            results = matches
        if self.order_by:
            results.sort(lambda a,b : cmp(getattr(a, self.order_by), getattr(b, self.order_by)))
        return results

CollectionField = Collection

class ModelBase(type):
    def __init__(cls, name, bases, attrs):
        fields = [field_name for field_name in attrs.keys() if isinstance(attrs[field_name], BaseField)]
        for field_name in fields:
            setattr(cls, field_name, cls._get_path(field_name, attrs[field_name]))
            attrs[field_name]._name = field_name
        if attrs.has_key("finders"):
            setattr(cls, "objects", ModelManager(cls, attrs["finders"]))
        else:
            setattr(cls, "objects", ModelManager(cls, {}))
        if attrs.has_key("headers"):
            setattr(cls.objects, "headers", attrs["headers"])

    def _get_path(cls, field_name, field_impl):
        return property(fget=lambda cls: cls._parse_field(field_impl),fset=lambda cls, value : cls._set_field(field_impl, value) )

class Model:
    __metaclass__ = ModelBase

    def __init__(self,json_data=None,**kw):
        if kw.has_key('json'):
            self._json = AttrDict(kw['json'])
        else:
            try:
                self._json = AttrDict(json.loads(json_data or '{}'))
            except:
                raise ValidationError("Invalid JSON")
        self.validate_on_load()

    def validate_on_load(self):
        pass

    def _parse_field(self, field):
        return field.parse(self._json)

    def _set_field(self, field, value):
        value = field.save(value)
        nodes = field.path.split('.')
        self.set_nested_value(self._json,nodes, value)

    def set_nested_value(self, data, nodes, value):
        node = nodes.pop(0)
        if len(nodes) > 0:
            self.set_nested_value(getattr(data,node),nodes, value)
        else:
            setattr(data,node, value)

    def __unicode__(self):
        return json.dumps(self._json,separators=(',',':'))

    def __str__(self):
        return self.__unicode__()

class AttrDict(dict):
    def __init__(self, value=None):
        if value is None:
            pass
        elif isinstance(value,dict):
            for key in value:
                self.__setitem__(key,value[key])
        else:
            TypeError, 'expected Dict'

    def __setitem__(self,key,value):
        if isinstance(value,dict) and not isinstance(value, AttrDict):
            value = AttrDict(value)
        dict.__setitem__(self,key,value)

    place_holder = object()
    def __getitem__(self,key):
        found = self.get(key, AttrDict.place_holder)
        if found is AttrDict.place_holder:
            found = AttrDict()
            dict.__setitem__(self,key,found)
        return found

    __setattr__ = __setitem__
    __getattr__ = __getitem__
<commit_msg>Fixed naming error on variable<commit_after>"""
Copyright 2009 Chris Tarttelin and Point2 Technologies

Redistribution and use in source and binary forms, with or without modification, are
permitted provided that the following conditions are met:

Redistributions of source code must retain the above copyright notice, this list of
conditions and the following disclaimer.

Redistributions in binary form must reproduce the above copyright notice, this list
of conditions and the following disclaimer in the documentation and/or other materials
provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE FREEBSD PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED
WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FREEBSD PROJECT OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and documentation are those of the
authors and should not be interpreted as representing official policies, either expressed
or implied, of the FreeBSD Project.
"""

import json, time
from datetime import datetime
from common_models import *

class BaseField:
    def __init__(self, **kw):
        if not kw.has_key('path'):
            raise Exception('No Path supplied for json field')
        self.path = kw['path']
        self._default = kw.pop('default', None)

    def get_nested_value(self, data, nodes):
        node = nodes.pop(0)
        if len(nodes) > 0:
            find = self.get_nested_value(getattr(data, node),nodes)
        else:
            find = getattr(data, node)
        if ((find == None) or (find == {})):
            return self._default
        else:
            return find

    def _parse(self, json_data):
        nodes = self.path.split('.')
        return self.get_nested_value(json_data, nodes)

    def save(self,value):
        return value

class CharField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class IntField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class BoolField(BaseField):
    def parse(self, json_data):
        return self._parse(json_data)

class DateField(BaseField):
    def parse(self, json_data):
        milliseconds_from_epoch = self._parse(json_data)
        return milliseconds_from_epoch and datetime.utcfromtimestamp(milliseconds_from_epoch / 1000.0) or None

    def save(self,value):
        return value and (long)((time.mktime(value.utctimetuple()) - time.timezone) * 1000.0 + value.microsecond / 1000.0) or None

class Collection(BaseField):
    def __init__(self, field_type, order_by=None, **kw):
        self.field_type = field_type
        self.order_by = order_by
        BaseField.__init__(self, **kw)

    def parse(self, json_data):
        results = []
        matches = self._parse(json_data)
        if not BaseField in self.field_type.__bases__:
            results = []
            for match in matches:
                if isinstance(match, Model):
                    results.append(self.field_type(json=match._json))
                else:
                    results.append(self.field_type(json=match))
        elif matches:
            results = matches
        if self.order_by:
            results.sort(lambda a,b : cmp(getattr(a, self.order_by), getattr(b, self.order_by)))
        return results

CollectionField = Collection

class ModelBase(type):
    def __init__(cls, name, bases, attrs):
        fields = [field_name for field_name in attrs.keys() if isinstance(attrs[field_name], BaseField)]
        for field_name in fields:
            setattr(cls, field_name, cls._get_path(field_name, attrs[field_name]))
            attrs[field_name]._name = field_name
        if attrs.has_key("finders"):
            setattr(cls, "objects", ModelManager(cls, attrs["finders"]))
        else:
            setattr(cls, "objects", ModelManager(cls, {}))
        if attrs.has_key("headers"):
            setattr(cls.objects, "headers", attrs["headers"])

    def _get_path(cls, field_name, field_impl):
        return property(fget=lambda cls: cls._parse_field(field_impl),fset=lambda cls, value : cls._set_field(field_impl, value) )

class Model:
    __metaclass__ = ModelBase

    def __init__(self,json_data=None,**kw):
        if kw.has_key('json'):
            self._json = AttrDict(kw['json'])
        else:
            try:
                self._json = AttrDict(json.loads(json_data or '{}'))
            except:
                raise ValidationError("Invalid JSON")
        self.validate_on_load()

    def validate_on_load(self):
        pass

    def _parse_field(self, field):
        return field.parse(self._json)

    def _set_field(self, field, value):
        value = field.save(value)
        nodes = field.path.split('.')
        self.set_nested_value(self._json,nodes, value)

    def set_nested_value(self, data, nodes, value):
        node = nodes.pop(0)
        if len(nodes) > 0:
            self.set_nested_value(getattr(data,node),nodes, value)
        else:
            setattr(data,node, value)

    def __unicode__(self):
        return json.dumps(self._json,separators=(',',':'))

    def __str__(self):
        return self.__unicode__()

class AttrDict(dict):
    def __init__(self, value=None):
        if value is None:
            pass
        elif isinstance(value,dict):
            for key in value:
                self.__setitem__(key,value[key])
        else:
            TypeError, 'expected Dict'

    def __setitem__(self,key,value):
        if isinstance(value,dict) and not isinstance(value, AttrDict):
            value = AttrDict(value)
        dict.__setitem__(self,key,value)

    place_holder = object()
    def __getitem__(self,key):
        found = self.get(key, AttrDict.place_holder)
        if found is AttrDict.place_holder:
            found = AttrDict()
            dict.__setitem__(self,key,found)
        return found

    __setattr__ = __setitem__
    __getattr__ = __getitem__
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import

from doto.logger import log
from doto.connection import connection

import requests

class Domain(object):

    def __str__(self):
        return ("Domain:%s") % (self.id)

    def __repr__(self):
        return ("Domain:%s") % (self.id)

    def __init__(self,conn=None, **kwds):
        self._conn = conn
        self.__dict__.update(kwds)

    def _pprint_table(self, data):
        """
        pprint table: from stackoverflow:
        http://stackoverflow.com/a/8356620
        """

        table = []
        for d in data:
            table.append([unicode(v) for v in d.values()])

        header = d.keys()
        table.insert(0,header)

        col_width = [max(len(x) for x in col) for col in zip(*table)]
        for line in table:
            print("| " + " | ".join("{:{}}".format(x, col_width[i])
                                    for i, x in enumerate(line)) + " |")

    def destroy(self):
        """
        This method allows you to destroy a domain.

        https://api.digitalocean.com/domains/[domain_id]/destroy/?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        url = "/domains/%s/destroy" % (str(self.id))

        data = self._conn.request(url)

        log.info(data)

    def get_all_records(self,table=False):
        """
        This method returns all of your current domain records.

        https://api.digitalocean.com/domains/[domain_id]/records?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Getting Records")
        url = "/domains/%s/records" % (str(self.id))

        data = self._conn.request(url)

        log.debug(data)

        if table:
            self._pprint_table(data['records'])

        return data['records']

    def get_record(self,record_id=None):
        """
        This method returns the specified domain record.

        :type record_id: int
        :param record_id: ID of record you are trying to retrieve.

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Getting Record: %d" % (record_id))
        url = "/domains/%s/records/%d" % (str(self.id), record_id)

        data = self._conn.request(url)

        log.debug(data)

        return data['record']

    def edit_record(self,record_id=None, record_type=None, data=None,
                         name=None,priority=None,port=None,weight=None,
                         ):
        """
        This method edits an existing domain record.

        :type record_id: int
        :param record_id: ID of record you are trying to retrieve.

        :type record_type: String
        :param record_type: the type of record you would like to create. 'A', 'CNAME', 'NS', 'TXT', 'MX' or 'SRV'.

        :type data: String
        :param data: This is the value of the record.

        :type name: String
        :param name: Required for 'A', 'CNAME', 'TXT' and 'SRV' records otherwise optional

        :type priority: int
        :param priority: required for 'SRV' and 'MX' records otherwise optional

        :type port: int
        :param port: required for 'SRV' records otherwise optional

        :type weight: int
        :param weight: required for 'SRV' records. otherwise optional

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]/edit?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Editing Record: %d" (record_id))
        url = "/domains/%s/records/%d/edit" % (str(self.id), record_id)

        data = self._conn.request(url,record_type=record_type,
                             data=data,name=name,priority=priority,
                             port=port,weight=weight)

        log.debug(data)

        return data['record']

    def record_destroy(self, record_id=None):
        """
        This method deletes the specified domain record.

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]/destroy?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        url = "domains/%s/records/%d/destroy" % (str(self.id),record_id)

        data = self._conn.request(url)

        log.info(data)

<commit_msg>Update domain.py<commit_after>from __future__ import print_function, division, absolute_import

from doto.logger import log
from doto.connection import connection

import requests

class Domain(object):

    def __str__(self):
        return ("Domain:%s") % (self.id)

    def __repr__(self):
        return ("Domain:%s") % (self.id)

    def __init__(self,conn=None, **kwds):
        self._conn = conn
        self.__dict__.update(kwds)

    def _pprint_table(self, data):
        """
        pprint table: from stackoverflow:
        http://stackoverflow.com/a/8356620
        """

        table = []
        for d in data:
            table.append([unicode(v) for v in d.values()])

        header = d.keys()
        table.insert(0,header)

        col_width = [max(len(x) for x in col) for col in zip(*table)]
        for line in table:
            print("| " + " | ".join("{:{}}".format(x, col_width[i])
                                    for i, x in enumerate(line)) + " |")

    def destroy(self):
        """
        This method allows you to destroy a domain.

        https://api.digitalocean.com/domains/[domain_id]/destroy/?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        url = "/domains/%s/destroy" % (str(self.id))

        data = self._conn.request(url)

        log.info(data)

    def get_all_records(self,table=False):
        """
        This method returns all of your current domain records.

        https://api.digitalocean.com/domains/[domain_id]/records?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Getting Records")
        url = "/domains/%s/records" % (str(self.id))

        data = self._conn.request(url)

        log.debug(data)

        if table:
            self._pprint_table(data['records'])

        return data['records']

    def get_record(self,record_id=None):
        """
        This method returns the specified domain record.

        :type record_id: int
        :param record_id: ID of record you are trying to retrieve.

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Getting Record: %d" % (record_id))
        url = "/domains/%s/records/%d" % (str(self.id), record_id)

        data = self._conn.request(url)

        log.debug(data)

        return data['record']

    def edit_record(self,record_id=None, record_type=None, data=None,
                         name=None,priority=None,port=None,weight=None,
                         ):
        """
        This method edits an existing domain record.

        :type record_id: int
        :param record_id: ID of record you are trying to retrieve.

        :type record_type: String
        :param record_type: the type of record you would like to create. 'A', 'CNAME', 'NS', 'TXT', 'MX' or 'SRV'.

        :type data: String
        :param data: This is the value of the record.

        :type name: String
        :param name: Required for 'A', 'CNAME', 'TXT' and 'SRV' records otherwise optional

        :type priority: int
        :param priority: required for 'SRV' and 'MX' records otherwise optional

        :type port: int
        :param port: required for 'SRV' records otherwise optional

        :type weight: int
        :param weight: required for 'SRV' records. otherwise optional

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]/edit?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        log.info("Editing Record: %d" % (record_id))
        url = "/domains/%s/records/%d/edit" % (str(self.id), record_id)

        data = self._conn.request(url,record_type=record_type,
                             data=data,name=name,priority=priority,
                             port=port,weight=weight)

        log.debug(data)

        return data['record']

    def record_destroy(self, record_id=None):
        """
        This method deletes the specified domain record.

        https://api.digitalocean.com/domains/[domain_id]/records/[record_id]/destroy?
        client_id=[your_client_id]&api_key=[your_api_key]
        """

        url = "domains/%s/records/%d/destroy" % (str(self.id),record_id)

        data = self._conn.request(url)

        log.info(data)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Mar 22 17:52:49 2016

@author: Alex Kerr

Contains class definition(s) used to minimize the energy of Molecules
"""
import sys
EP = sys.float_info.epsilon

import numpy as np

class Minimizer:
    """A energy minimizer for Molecules

    Args:
        n (int): Max number of iterations in minimization run(s).
    Keywords:
        descent (str): Method in which the local energy minimum will be approached;
            must be key in descentDict
        search (str): Line search method; must be key in searchDict
        numgrad (bool): True if the gradients are to be calculated numerically;
            default is False
        eprec (float): Precision for energy changes when an iteration is made
            signalling convergence has occurred
        fprec (float): Precision for force magnitude signaling
            convergence has occurred.
        efreq (int): Iteration period in which information is printed to the user;
            called frequency despite being inverse frequency."""

    def __init__(self, n, descent="cg", search="backtrack",
                 numgrad=False, eprec=1e-5, fprec=1e-3,
                 efreq=1):
        self.n = n
        self.descent = descent
        self.search = search
        self.numgrad = numgrad
        self.eprec = eprec
        self.fprec = fprec
        self.efreq =  efreq

    def __call__(self, molecule):
        if self.numgrad:
            grad_routine = molecule.define_gradient_routine()
        else:
            grad_routine = molecule.define_gradient_routine()
        descentDict[self.descent](molecule, self.n, searchDict[self.search], molecule.define_energy_routine(),
                                  grad_routine, self.efreq,
                                  self.eprec*molecule.ff.eunits, self.fprec*molecule.ff.eunits/molecule.ff.lunits)

def steepest_descent(mol, n, search, calc_e, calc_grad, efreq, eprec, fprec):
    """Minimize the energy of the inputted molecule via the steepest descent approach."""

    energy = calc_e()
    gradient, maxForce, totalMag, = calc_grad()
    eList = [energy]
    print('energy:   $s' % energy)
    print('maxforce: $s' % maxForce)

    for step in range(1, n+1):

        #calculate the stepsize
        stepSize = search(mol, -gradient/totalMag, energy)

        #take the step
        mol.posList += stepSize*(-gradient/totalMag)

        #reset quantities
        energy = calc_e()
        gradient, maxForce, totalMag = calc_grad()

        eList.append(energy)
        #for every multiple of efreq, print the status
        if step % efreq == 0:
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)

        #break the iteration if our forces are small enough
        if maxForce < fprec:
            print('#########\nFinished!\n#########')
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)
            break

    return mol, eList

def conjugate_gradient(mol, n, search, calc_e, calc_grad, efreq, eprec, fprec):
    """Minimize the energy of the inputted molecule via the conjugate gradient approach."""

    energy = calc_e()
    gradient, maxForce, totalMag, = calc_grad()
    eList = [energy]
    print('energy:   $s' % energy)
    print('maxforce: $s' % maxForce)

    gamma = 0.0
    prevH = np.zeros([len(mol), 3])

    for step in range(1, n+1):

        #get the step direction
        h = -gradient + gamma*prevH
        normH = h/np.linalg.norm(np.hstack(h))

        #calculate the stepsize
        stepSize = search(mol, normH, energy)

        #take the step
        mol.posList += stepSize*(-gradient/totalMag)

        #reset quantities
        prevH = h
        prevGrad = gradient
        energy = calc_e()
        gradient, maxForce, totalMag = calc_grad()
        gamma = calculate_gamma(gradient, prevGrad)

        eList.append(energy)
        #for every multiple of efreq, print the status
        if step % efreq == 0:
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)

        #break the iteration if our forces are small enough
        if maxForce < fprec:
            print('#########\nFinished!\n#########')
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)
            break

    return mol, eList

def line_search_backtrack(mol, stepList, e, grad, calc_e, calc_grad):
    """Return the stepsize determined by the backtracking strategies of
    Armijo and Goldstein."""

    n = 25
    alpha = 1e-2
    tau = 0.5
    c = 0.5

    for j in range(n):

        m = np.dot(np.hstack(stepList),np.hstack(grad))
        t = -c*m
        mol.posList += alpha*stepList
        newE = calc_e()
        if e - newE >= alpha*t:
            return alpha
        else:
            mol.posList += -alpha*stepList
            e, grad = newE, calc_grad()
            alpha *= tau

    return EP**.5

descentDict = {"sd":steepest_descent, "cg":conjugate_gradient}
searchDict = {"backtrack":line_search_backtrack}

def calculate_gamma(grad, pgrad):
    """Return the 'gamma' factor in the conjugate gradient method."""
    grad = np.hstack(grad)
    pgrad = np.hstack(pgrad)
#    return (np.dot(grad-pGrad,grad))/(np.dot(pGrad,pGrad))
    return (np.dot(grad,grad))/(np.dot(pgrad,pgrad))<commit_msg>Adjust line search to terminateat machine precision<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Mar 22 17:52:49 2016

@author: Alex Kerr

Contains class definition(s) used to minimize the energy of Molecules
"""
import sys
EP = sys.float_info.epsilon
SQRTEP = EP**.5

import numpy as np

class Minimizer:
    """A energy minimizer for Molecules

    Args:
        n (int): Max number of iterations in minimization run(s).
    Keywords:
        descent (str): Method in which the local energy minimum will be approached;
            must be key in descentDict
        search (str): Line search method; must be key in searchDict
        numgrad (bool): True if the gradients are to be calculated numerically;
            default is False
        eprec (float): Precision for energy changes when an iteration is made
            signalling convergence has occurred
        fprec (float): Precision for force magnitude signaling
            convergence has occurred.
        efreq (int): Iteration period in which information is printed to the user;
            called frequency despite being inverse frequency."""

    def __init__(self, n, descent="cg", search="backtrack",
                 numgrad=False, eprec=1e-5, fprec=1e-3,
                 efreq=1):
        self.n = n
        self.descent = descent
        self.search = search
        self.numgrad = numgrad
        self.eprec = eprec
        self.fprec = fprec
        self.efreq =  efreq

    def __call__(self, molecule):
        if self.numgrad:
            grad_routine = molecule.define_gradient_routine()
        else:
            grad_routine = molecule.define_gradient_routine()
        descentDict[self.descent](molecule, self.n, searchDict[self.search], molecule.define_energy_routine(),
                                  grad_routine, self.efreq,
                                  self.eprec*molecule.ff.eunits, self.fprec*molecule.ff.eunits/molecule.ff.lunits)

def steepest_descent(mol, n, search, calc_e, calc_grad, efreq, eprec, fprec):
    """Minimize the energy of the inputted molecule via the steepest descent approach."""

    energy = calc_e()
    gradient, maxForce, totalMag, = calc_grad()
    eList = [energy]
    print('energy:   $s' % energy)
    print('maxforce: $s' % maxForce)

    for step in range(1, n+1):

        #calculate the stepsize
        stepSize = search(mol, -gradient/totalMag, energy)

        #take the step
        mol.posList += stepSize*(-gradient/totalMag)

        #reset quantities
        energy = calc_e()
        gradient, maxForce, totalMag = calc_grad()

        eList.append(energy)
        #for every multiple of efreq, print the status
        if step % efreq == 0:
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)

        #break the iteration if our forces are small enough
        if maxForce < fprec:
            print('#########\nFinished!\n#########')
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)
            break

    return mol, eList

def conjugate_gradient(mol, n, search, calc_e, calc_grad, efreq, eprec, fprec):
    """Minimize the energy of the inputted molecule via the conjugate gradient approach."""

    energy = calc_e()
    gradient, maxForce, totalMag, = calc_grad()
    eList = [energy]
    print('energy:   $s' % energy)
    print('maxforce: $s' % maxForce)

    gamma = 0.0
    prevH = np.zeros([len(mol), 3])

    for step in range(1, n+1):

        #get the step direction
        h = -gradient + gamma*prevH
        normH = h/np.linalg.norm(np.hstack(h))

        #calculate the stepsize
        stepSize = search(mol, normH, energy)

        #take the step
        mol.posList += stepSize*(-gradient/totalMag)

        #reset quantities
        prevH = h
        prevGrad = gradient
        energy = calc_e()
        gradient, maxForce, totalMag = calc_grad()
        gamma = calculate_gamma(gradient, prevGrad)

        eList.append(energy)
        #for every multiple of efreq, print the status
        if step % efreq == 0:
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)

        #break the iteration if our forces are small enough
        if maxForce < fprec:
            print('#########\nFinished!\n#########')
            print('step:     %s' % step)
            print('energy:   %s' % energy)
            print('maxforce: %s' % maxForce)
            break

    return mol, eList

def line_search_backtrack(mol, stepList, e, grad, calc_e, calc_grad):
    """Return the stepsize determined by the backtracking strategies of
    Armijo and Goldstein."""

    alpha = 1e-2
    tau = 0.5
    c = 0.5

    while alpha > SQRTEP:

        m = np.dot(np.hstack(stepList),np.hstack(grad))
        t = -c*m
        mol.posList += alpha*stepList
        newE = calc_e()
        if e - newE >= alpha*t:
            return alpha
        else:
            mol.posList += -alpha*stepList
            e, grad = newE, calc_grad()
            alpha *= tau

    return SQRTEP

descentDict = {"sd":steepest_descent, "cg":conjugate_gradient}
searchDict = {"backtrack":line_search_backtrack}

def calculate_gamma(grad, pgrad):
    """Return the 'gamma' factor in the conjugate gradient method."""
    grad = np.hstack(grad)
    pgrad = np.hstack(pgrad)
#    return (np.dot(grad-pGrad,grad))/(np.dot(pGrad,pGrad))
    return (np.dot(grad,grad))/(np.dot(pgrad,pgrad))<|endoftext|>