<commit_before># -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
Generate the rst files for the examples by iterating over the python
example files.

Files that generate images should start with 'plot'

"""
from __future__ import division, print_function, absolute_import
from time import time
import ast
import os
import re
import shutil
import traceback
import sys
import subprocess
import warnings
from textwrap import dedent
from . import glr_path_static
from .backreferences import write_backreferences, _thumbnail_div

# Try Python 2 first, otherwise load from Python 3
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    basestring
except NameError:
    basestring = str

try:
    # make sure that the Agg backend is set before importing any
    # matplotlib
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
except ImportError:
    # this script can be imported by nosetest to find tests to run: we should
    # not impose the matplotlib requirement in that case.
    pass

###############################################################################

class Tee(object):
    """A tee object to redirect streams to multiple outputs"""
    def __init__(self, file1, file2):
        self.file1 = file1
        self.file2 = file2

    def write(self, data):
        self.file1.write(data)
        self.file2.write(data)

    def flush(self):
        self.file1.flush()
        self.file2.flush()

###############################################################################
CODE_DOWNLOAD = """**Total running time of the script:**
({0:.0f} minutes {1:.3f} seconds)\n\n
\n**Download Python source code:** :download:`{2} <{2}>`\n"""

# The following strings are used when we have several pictures: we use
# an html div tag that our CSS uses to turn the lists into horizontal
# lists.
HLIST_HEADER = """
.. rst-class:: sphx-glr-horizontal

"""

HLIST_IMAGE_TEMPLATE = """
    *

      .. image:: /%s
            :scale: 47
"""

SINGLE_IMAGE = """
.. image:: /%s
    :align: center
"""

CODE_OUTPUT = """**Script output**:\n
.. rst-class:: sphx-glr-script-out

  ::

    {0}\n"""

def split_code_and_text_blocks(source_file):
    """Return list with source file separated into code and text blocks.

    Returns
    -------
    blocks : list of (label, content)
        List where each element is a tuple with the label ('text' or 'code'),
        and content string of block.
    """

    with open(source_file) as f:
        source_lines = f.read()

    doc_string = ast.get_docstring(ast.parse(source_lines))

    blocks = [('text', doc_string)]
    # Remove from source_lines the docstring
    end_doc = [m.end() for m in re.finditer('"""', source_lines)][1]
    source_lines = source_lines[end_doc:]

    pattern = re.compile('^#{20,}.*\s((?:^#.*\s)*)', flags=re.M)
    split_block = re.split(pattern, source_lines)
    for split_code_block in split_block:
        if split_code_block is None or split_code_block.strip() == '':
            continue
        elif split_code_block.startswith('#'):
            sub_pat = re.compile('^#', flags=re.M)
            coment_block = dedent(re.sub(sub_pat, '', split_code_block))
            blocks.append(('text', coment_block))
        else:
            blocks.append(('code', split_code_block))

    return blocks

def codestr2rst(codestr):
    """Return reStructuredText code block from code string"""
    code_directive = "\n.. code-block:: python\n\n"
    indented_block = '    ' + codestr.replace('\n', '\n    ')
    return code_directive + indented_block

def text2string(content):
    """Returns a string without the extra triple quotes"""
    try:
        return ast.literal_eval(content)+'\n'
    except:
        return content

def extract_intro(filename):
    """ Extract the first paragraph of module-level docstring. max:95 char"""

    first_text = split_code_and_text_blocks(filename)[0][-1]

    paragraphs = first_text.split('\n\n')
    if len(first_text) > 1:
        first_par = re.sub('\n', ' ', paragraphs[1])
        first_par = ((first_par[:95] + '...')
                     if len(first_par) > 95 else first_par)
    else:
        raise ValueError("Missing first paragraph."
                         "Please check the layout of your"
                         " example file:\n {}\n and make sure"
                         " it's correct.".format(filename))

    return first_par

def _plots_are_current(src_file, image_file):
    """Test existence of image file and later touch time to source script"""

    first_image_file = image_file.format(1)
    needs_replot = (not os.path.exists(first_image_file) or
              os.stat(first_image_file).st_mtime <= os.stat(src_file).st_mtime)
    return not needs_replot

def save_figures(image_path, fig_count):
    """Save all open matplotlib figures of the example code-block

    Parameters
    ----------
    image_path : str
        Path where plots are saved (format string which accepts figure number)
    fig_count : int
        Previous figure number count. Figure number add from this number

    Returns
    -------
    list of strings containing the full path to each figure
    """
    figure_list = []

    fig_managers = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
    for fig_mngr in fig_managers:
        # Set the fig_num figure as the current figure as we can't
        # save a figure that's not the current figure.
        fig = plt.figure(fig_mngr.num)
        kwargs = {}
        to_rgba = matplotlib.colors.colorConverter.to_rgba
        for attr in ['facecolor', 'edgecolor']:
            fig_attr = getattr(fig, 'get_' + attr)()
            default_attr = matplotlib.rcParams['figure.' + attr]
            if to_rgba(fig_attr) != to_rgba(default_attr):
                kwargs[attr] = fig_attr

        current_fig = image_path.format(fig_count + fig_mngr.num)
        fig.savefig(current_fig, **kwargs)
        figure_list.append(current_fig)
    return figure_list

def scale_image(in_fname, out_fname, max_width, max_height):
    """Scales an image with the same aspect ratio centered in an
       image with a given max_width and max_height
       if in_fname == out_fname the image can only be scaled down
    """
    # local import to avoid testing dependency on PIL:
    try:
        from PIL import Image
    except ImportError:
        import Image
    img = Image.open(in_fname)
    width_in, height_in = img.size
    scale_w = max_width / float(width_in)
    scale_h = max_height / float(height_in)

    if height_in * scale_w <= max_height:
        scale = scale_w
    else:
        scale = scale_h

    if scale >= 1.0 and in_fname == out_fname:
        return

    width_sc = int(round(scale * width_in))
    height_sc = int(round(scale * height_in))

    # resize the image
    img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)

    # insert centered
    thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
    pos_insert = ((max_width - width_sc) // 2, (max_height - height_sc) // 2)
    thumb.paste(img, pos_insert)

    thumb.save(out_fname)
    # Use optipng to perform lossless compression on the resized image if
    # software is installed
    if os.environ.get('SKLEARN_DOC_OPTIPNG', False):
        try:
            subprocess.call(["optipng", "-quiet", "-o", "9", out_fname])
        except Exception:
            warnings.warn('Install optipng to reduce the size of the \
                          generated images')

def save_thumbnail(image_path, base_image_name):
    """Save the thumbnail image"""
    first_image_file = image_path.format(1)
    thumb_dir = os.path.join(os.path.dirname(first_image_file), 'thumb')
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_file = os.path.join(thumb_dir,
                              'sphx_glr_%s_thumb.png' % base_image_name)

    if os.path.exists(first_image_file):
        scale_image(first_image_file, thumb_file, 400, 280)
    elif not os.path.exists(thumb_file):
        # create something to replace the thumbnail
        scale_image(os.path.join(glr_path_static(), 'no_image.png'),
                    thumb_file, 200, 140)

def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
    """Generate the gallery reStructuredText for an example directory"""
    if not os.path.exists(os.path.join(src_dir, 'README.txt')):
        print(80 * '_')
        print('Example directory %s does not have a README.txt file' %
              src_dir)
        print('Skipping this directory')
        print(80 * '_')
        return ""  # because string is an expected return type

    fhindex = open(os.path.join(src_dir, 'README.txt')).read()
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    sorted_listdir = [fname for fname in sorted(os.listdir(src_dir))
                                if fname.endswith('py')]
    for fname in sorted_listdir:
        generate_file_rst(fname, target_dir, src_dir)
        new_fname = os.path.join(src_dir, fname)
        intro = extract_intro(new_fname)
        write_backreferences(seen_backrefs, gallery_conf,
                             target_dir, fname, intro)

        fhindex += _thumbnail_div(target_dir, fname, intro)
        fhindex += """

.. toctree::
   :hidden:

   /%s/%s\n""" % (target_dir, fname[:-3])

    # clear at the end of the section
    fhindex += """.. raw:: html\n

\n\n"""

    return fhindex

def execute_script(code_block, example_globals, image_path, fig_count, src_file):
    """Executes the code block of the example file"""
    time_elapsed = 0
    stdout = ''

    # We need to execute the code
    print('plotting code blocks in %s' % src_file)

    plt.close('all')
    cwd = os.getcwd()
    # Redirect output to stdout and
    orig_stdout = sys.stdout

    try:
        # First cd in the original example dir, so that any file
        # created by the example get created in this directory
        os.chdir(os.path.dirname(src_file))
        my_buffer = StringIO()
        my_stdout = Tee(sys.stdout, my_buffer)
        sys.stdout = my_stdout

        t_start = time()
        exec(code_block, example_globals)
        time_elapsed = time() - t_start

        sys.stdout = orig_stdout

        my_stdout = my_buffer.getvalue().strip().expandtabs()
        if my_stdout:
            stdout = CODE_OUTPUT.format(my_stdout.replace('\n', '\n    '))
        os.chdir(cwd)
        figure_list = save_figures(image_path, fig_count)

        # Depending on whether we have one or more figures, we're using a
        # horizontal list or a single rst call to 'image'.
        if len(figure_list) == 1:
            figure_name = figure_list[0]
            image_list = SINGLE_IMAGE % figure_name.lstrip('/')
        else:
            image_list = HLIST_HEADER
            for figure_name in figure_list:
                image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')

    except Exception:
        image_list = '%s is not compiling:' % src_file
        print(80 * '_')
        print(image_list)
        traceback.print_exc()
        print(80 * '_')
    finally:
        os.chdir(cwd)
        sys.stdout = orig_stdout

    print(" - time elapsed : %.2g sec" % time_elapsed)
    code_output = "\n{0}\n\n{1}\n\n".format(image_list, stdout)

    return code_output, time_elapsed, fig_count + len(figure_list)

def generate_file_rst(fname, target_dir, src_dir):
    """ Generate the rst file for a given example."""

    src_file = os.path.join(src_dir, fname)
    example_file = os.path.join(target_dir, fname)
    shutil.copyfile(src_file, example_file)

    image_dir = os.path.join(target_dir, 'images')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)

    base_image_name = os.path.splitext(fname)[0]
    image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png'
    image_path = os.path.join(image_dir, image_fname)

    if _plots_are_current(src_file, image_path):
        return

    time_elapsed = 0
    script_blocks = split_code_and_text_blocks(example_file)

    ref_fname = example_file.replace(os.path.sep, '_')
    example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname)

    if not fname.startswith('plot'):
        convert_func = dict(code=codestr2rst, text=text2string)
        for blabel, bcontent in script_blocks:
            example_rst += convert_func[blabel](bcontent)+'\n'
    else:
        example_globals = {}
        fig_count = 0
        for blabel, bcontent in script_blocks:
            if blabel == 'code':
                code_output, rtime, fig_count = execute_script(bcontent,
                                                               example_globals,
                                                               image_path,
                                                               fig_count,
                                                               src_file)

                time_elapsed += rtime
                # Single example style
                if len(script_blocks) == 2:
                    example_rst += code_output
                    example_rst += codestr2rst(bcontent)+'\n'
                # Notebook style
                else:
                    example_rst += codestr2rst(bcontent)+'\n'
                    example_rst += code_output
            else:
                example_rst += text2string(bcontent)+'\n'

    save_thumbnail(image_path, base_image_name)

    time_m, time_s = divmod(time_elapsed, 60)
    with open(os.path.join(target_dir, base_image_name + '.rst'), 'w') as f:
        example_rst += CODE_DOWNLOAD.format(time_m, time_s, fname)
        f.write(example_rst)
<commit_msg>MAINT fix typo coment -> comment<commit_after># -*- coding: utf-8 -*-
# Author: Óscar Nájera
# License: 3-clause BSD
"""
Generate the rst files for the examples by iterating over the python
example files.

Files that generate images should start with 'plot'

"""
from __future__ import division, print_function, absolute_import
from time import time
import ast
import os
import re
import shutil
import traceback
import sys
import subprocess
import warnings
from textwrap import dedent
from . import glr_path_static
from .backreferences import write_backreferences, _thumbnail_div

# Try Python 2 first, otherwise load from Python 3
try:
    from StringIO import StringIO
except ImportError:
    from io import StringIO

try:
    basestring
except NameError:
    basestring = str

try:
    # make sure that the Agg backend is set before importing any
    # matplotlib
    import matplotlib
    matplotlib.use('Agg')
    import matplotlib.pyplot as plt
except ImportError:
    # this script can be imported by nosetest to find tests to run: we should
    # not impose the matplotlib requirement in that case.
    pass

###############################################################################

class Tee(object):
    """A tee object to redirect streams to multiple outputs"""
    def __init__(self, file1, file2):
        self.file1 = file1
        self.file2 = file2

    def write(self, data):
        self.file1.write(data)
        self.file2.write(data)

    def flush(self):
        self.file1.flush()
        self.file2.flush()

###############################################################################
CODE_DOWNLOAD = """**Total running time of the script:**
({0:.0f} minutes {1:.3f} seconds)\n\n
\n**Download Python source code:** :download:`{2} <{2}>`\n"""

# The following strings are used when we have several pictures: we use
# an html div tag that our CSS uses to turn the lists into horizontal
# lists.
HLIST_HEADER = """
.. rst-class:: sphx-glr-horizontal

"""

HLIST_IMAGE_TEMPLATE = """
    *

      .. image:: /%s
            :scale: 47
"""

SINGLE_IMAGE = """
.. image:: /%s
    :align: center
"""

CODE_OUTPUT = """**Script output**:\n
.. rst-class:: sphx-glr-script-out

  ::

    {0}\n"""

def split_code_and_text_blocks(source_file):
    """Return list with source file separated into code and text blocks.

    Returns
    -------
    blocks : list of (label, content)
        List where each element is a tuple with the label ('text' or 'code'),
        and content string of block.
    """

    with open(source_file) as f:
        source_lines = f.read()

    doc_string = ast.get_docstring(ast.parse(source_lines))

    blocks = [('text', doc_string)]
    # Remove from source_lines the docstring
    end_doc = [m.end() for m in re.finditer('"""', source_lines)][1]
    source_lines = source_lines[end_doc:]

    pattern = re.compile('^#{20,}.*\s((?:^#.*\s)*)', flags=re.M)
    split_block = re.split(pattern, source_lines)
    for split_code_block in split_block:
        if split_code_block is None or split_code_block.strip() == '':
            continue
        elif split_code_block.startswith('#'):
            sub_pat = re.compile('^#', flags=re.M)
            comment_block = dedent(re.sub(sub_pat, '', split_code_block))
            blocks.append(('text', comment_block))
        else:
            blocks.append(('code', split_code_block))

    return blocks

def codestr2rst(codestr):
    """Return reStructuredText code block from code string"""
    code_directive = "\n.. code-block:: python\n\n"
    indented_block = '    ' + codestr.replace('\n', '\n    ')
    return code_directive + indented_block

def text2string(content):
    """Returns a string without the extra triple quotes"""
    try:
        return ast.literal_eval(content)+'\n'
    except:
        return content

def extract_intro(filename):
    """ Extract the first paragraph of module-level docstring. max:95 char"""

    first_text = split_code_and_text_blocks(filename)[0][-1]

    paragraphs = first_text.split('\n\n')
    if len(first_text) > 1:
        first_par = re.sub('\n', ' ', paragraphs[1])
        first_par = ((first_par[:95] + '...')
                     if len(first_par) > 95 else first_par)
    else:
        raise ValueError("Missing first paragraph."
                         "Please check the layout of your"
                         " example file:\n {}\n and make sure"
                         " it's correct.".format(filename))

    return first_par

def _plots_are_current(src_file, image_file):
    """Test existence of image file and later touch time to source script"""

    first_image_file = image_file.format(1)
    needs_replot = (
        not os.path.exists(first_image_file) or
        os.stat(first_image_file).st_mtime <= os.stat(src_file).st_mtime)
    return not needs_replot

def save_figures(image_path, fig_count):
    """Save all open matplotlib figures of the example code-block

    Parameters
    ----------
    image_path : str
        Path where plots are saved (format string which accepts figure number)
    fig_count : int
        Previous figure number count. Figure number add from this number

    Returns
    -------
    list of strings containing the full path to each figure
    """
    figure_list = []

    fig_managers = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
    for fig_mngr in fig_managers:
        # Set the fig_num figure as the current figure as we can't
        # save a figure that's not the current figure.
        fig = plt.figure(fig_mngr.num)
        kwargs = {}
        to_rgba = matplotlib.colors.colorConverter.to_rgba
        for attr in ['facecolor', 'edgecolor']:
            fig_attr = getattr(fig, 'get_' + attr)()
            default_attr = matplotlib.rcParams['figure.' + attr]
            if to_rgba(fig_attr) != to_rgba(default_attr):
                kwargs[attr] = fig_attr

        current_fig = image_path.format(fig_count + fig_mngr.num)
        fig.savefig(current_fig, **kwargs)
        figure_list.append(current_fig)
    return figure_list

def scale_image(in_fname, out_fname, max_width, max_height):
    """Scales an image with the same aspect ratio centered in an
       image with a given max_width and max_height
       if in_fname == out_fname the image can only be scaled down
    """
    # local import to avoid testing dependency on PIL:
    try:
        from PIL import Image
    except ImportError:
        import Image
    img = Image.open(in_fname)
    width_in, height_in = img.size
    scale_w = max_width / float(width_in)
    scale_h = max_height / float(height_in)

    if height_in * scale_w <= max_height:
        scale = scale_w
    else:
        scale = scale_h

    if scale >= 1.0 and in_fname == out_fname:
        return

    width_sc = int(round(scale * width_in))
    height_sc = int(round(scale * height_in))

    # resize the image
    img.thumbnail((width_sc, height_sc), Image.ANTIALIAS)

    # insert centered
    thumb = Image.new('RGB', (max_width, max_height), (255, 255, 255))
    pos_insert = ((max_width - width_sc) // 2, (max_height - height_sc) // 2)
    thumb.paste(img, pos_insert)

    thumb.save(out_fname)
    # Use optipng to perform lossless compression on the resized image if
    # software is installed
    if os.environ.get('SKLEARN_DOC_OPTIPNG', False):
        try:
            subprocess.call(["optipng", "-quiet", "-o", "9", out_fname])
        except Exception:
            warnings.warn('Install optipng to reduce the size of the \
                          generated images')

def save_thumbnail(image_path, base_image_name):
    """Save the thumbnail image"""
    first_image_file = image_path.format(1)
    thumb_dir = os.path.join(os.path.dirname(first_image_file), 'thumb')
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_file = os.path.join(thumb_dir,
                              'sphx_glr_%s_thumb.png' % base_image_name)

    if os.path.exists(first_image_file):
        scale_image(first_image_file, thumb_file, 400, 280)
    elif not os.path.exists(thumb_file):
        # create something to replace the thumbnail
        scale_image(os.path.join(glr_path_static(), 'no_image.png'),
                    thumb_file, 200, 140)

def generate_dir_rst(src_dir, target_dir, gallery_conf, seen_backrefs):
    """Generate the gallery reStructuredText for an example directory"""
    if not os.path.exists(os.path.join(src_dir, 'README.txt')):
        print(80 * '_')
        print('Example directory %s does not have a README.txt file' %
              src_dir)
        print('Skipping this directory')
        print(80 * '_')
        return ""  # because string is an expected return type

    fhindex = open(os.path.join(src_dir, 'README.txt')).read()
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    sorted_listdir = [fname for fname in sorted(os.listdir(src_dir))
                      if fname.endswith('py')]
    for fname in sorted_listdir:
        generate_file_rst(fname, target_dir, src_dir)
        new_fname = os.path.join(src_dir, fname)
        intro = extract_intro(new_fname)
        write_backreferences(seen_backrefs, gallery_conf,
                             target_dir, fname, intro)

        fhindex += _thumbnail_div(target_dir, fname, intro)
        fhindex += """

.. toctree::
   :hidden:

   /%s/%s\n""" % (target_dir, fname[:-3])

    # clear at the end of the section
    fhindex += """.. raw:: html\n

\n\n"""

    return fhindex

def execute_script(code_block, example_globals, image_path, fig_count,
                   src_file):
    """Executes the code block of the example file"""
    time_elapsed = 0
    stdout = ''

    # We need to execute the code
    print('plotting code blocks in %s' % src_file)

    plt.close('all')
    cwd = os.getcwd()
    # Redirect output to stdout and
    orig_stdout = sys.stdout

    try:
        # First cd in the original example dir, so that any file
        # created by the example get created in this directory
        os.chdir(os.path.dirname(src_file))
        my_buffer = StringIO()
        my_stdout = Tee(sys.stdout, my_buffer)
        sys.stdout = my_stdout

        t_start = time()
        exec(code_block, example_globals)
        time_elapsed = time() - t_start

        sys.stdout = orig_stdout

        my_stdout = my_buffer.getvalue().strip().expandtabs()
        if my_stdout:
            stdout = CODE_OUTPUT.format(my_stdout.replace('\n', '\n    '))
        os.chdir(cwd)
        figure_list = save_figures(image_path, fig_count)

        # Depending on whether we have one or more figures, we're using a
        # horizontal list or a single rst call to 'image'.
        if len(figure_list) == 1:
            figure_name = figure_list[0]
            image_list = SINGLE_IMAGE % figure_name.lstrip('/')
        else:
            image_list = HLIST_HEADER
            for figure_name in figure_list:
                image_list += HLIST_IMAGE_TEMPLATE % figure_name.lstrip('/')

    except Exception:
        image_list = '%s is not compiling:' % src_file
        print(80 * '_')
        print(image_list)
        traceback.print_exc()
        print(80 * '_')
    finally:
        os.chdir(cwd)
        sys.stdout = orig_stdout

    print(" - time elapsed : %.2g sec" % time_elapsed)
    code_output = "\n{0}\n\n{1}\n\n".format(image_list, stdout)

    return code_output, time_elapsed, fig_count + len(figure_list)

def generate_file_rst(fname, target_dir, src_dir):
    """ Generate the rst file for a given example."""

    src_file = os.path.join(src_dir, fname)
    example_file = os.path.join(target_dir, fname)
    shutil.copyfile(src_file, example_file)

    image_dir = os.path.join(target_dir, 'images')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)

    base_image_name = os.path.splitext(fname)[0]
    image_fname = 'sphx_glr_' + base_image_name + '_{0:03}.png'
    image_path = os.path.join(image_dir, image_fname)

    if _plots_are_current(src_file, image_path):
        return

    time_elapsed = 0
    script_blocks = split_code_and_text_blocks(example_file)

    ref_fname = example_file.replace(os.path.sep, '_')
    example_rst = """\n\n.. _sphx_glr_{0}:\n\n""".format(ref_fname)

    if not fname.startswith('plot'):
        convert_func = dict(code=codestr2rst, text=text2string)
        for blabel, bcontent in script_blocks:
            example_rst += convert_func[blabel](bcontent)+'\n'
    else:
        example_globals = {}
        fig_count = 0
        for blabel, bcontent in script_blocks:
            if blabel == 'code':
                code_output, rtime, fig_count = execute_script(bcontent,
                                                               example_globals,
                                                               image_path,
                                                               fig_count,
                                                               src_file)

                time_elapsed += rtime
                # Single example style
                if len(script_blocks) == 2:
                    example_rst += code_output
                    example_rst += codestr2rst(bcontent)+'\n'
                # Notebook style
                else:
                    example_rst += codestr2rst(bcontent)+'\n'
                    example_rst += code_output
            else:
                example_rst += text2string(bcontent)+'\n'

    save_thumbnail(image_path, base_image_name)

    time_m, time_s = divmod(time_elapsed, 60)
    with open(os.path.join(target_dir, base_image_name + '.rst'), 'w') as f:
        example_rst += CODE_DOWNLOAD.format(time_m, time_s, fname)
        f.write(example_rst)
<|endoftext|>


<commit_before># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

from .agents import Teacher

import random

class FixedDataTeacher(Teacher):
    """A teacher agent for all teachers involved in tasks with fixed data.

    This class provides the following functionality for its subclasses:

    - Resets a teacher
    - Provides an observe method
    - Computes and retrieves the next episdoe index for a teacher
    - TODO Provides a threadpool option for loading data (especially useful for
      large examples, e.g. images)

    """
    def __init__(self, opt, shared=None):
        super().__init__(opt, shared)

        self.datatype = opt['datatype']
        self.random = self.datatype == 'train'
        # for ordered data in batch mode (especially, for validation and
        # testing), each teacher in the batch gets a start index and a step
        # size so they all process disparate sets of the data
        self.step_size = opt.get('batchsize', 1)
        self.data_offset = opt.get('batchindex', 0)

    def __len__(self):
        return len(self.examples)

    def reset(self):
        """Reset the dialog so that it is at the start of the epoch,
        and all metrics are reset.
        """
        super().reset()
        self.metrics.clear()
        self.lastY = None
        self.episode_idx = self.data_offset - self.step_size
        self.episode_done = True
        self.epochDone = False
        if (self.episode_idx + self.step_size > len(self) and not self.random):
            self.epochDone = True

    def observe(self, observation):
        """Process observation for metrics."""
        if self.lastY is not None:
            self.metrics.update(observation, self.lastY)
            self.lastY = None
        return observation

    def next_episode_idx(self, num_eps=None):
        if not num_eps:
            num_eps = len(self)
        epoch_done = False
        if self.random:
            episode_idx = random.randrange(num_eps)
        else:
            episode_idx = (self.episode_idx + self.step_size) % num_eps
            if episode_idx + self.step_size > num_eps:
                epoch_done = True
        return episode_idx, epoch_done
<commit_msg>Update fixed_data_teacher.py<commit_after># Copyright (c) 2017-present, Facebook, Inc.
# All rights reserved.
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

from .agents import Teacher

import random

class FixedDataTeacher(Teacher):
    """A teacher agent for all teachers involved in tasks with fixed data.

    This class provides the following functionality for its subclasses:

    - Resets a teacher
    - Provides an observe method
    - Computes and retrieves the next episode index for a teacher
    - TODO Provides a threadpool option for loading data (especially useful for
      large examples, e.g. images)

    """
    def __init__(self, opt, shared=None):
        super().__init__(opt, shared)

        self.datatype = opt['datatype']
        self.random = self.datatype == 'train'
        # for ordered data in batch mode (especially, for validation and
        # testing), each teacher in the batch gets a start index and a step
        # size so they all process disparate sets of the data
        self.step_size = opt.get('batchsize', 1)
        self.data_offset = opt.get('batchindex', 0)

    def __len__(self):
        return len(self.examples)

    def reset(self):
        """Reset the dialog so that it is at the start of the epoch,
        and all metrics are reset.
        """
        super().reset()
        self.metrics.clear()
        self.lastY = None
        self.episode_idx = self.data_offset - self.step_size
        self.episode_done = True
        self.epochDone = False
        if (self.episode_idx + self.step_size > len(self) and not self.random):
            self.epochDone = True

    def observe(self, observation):
        """Process observation for metrics."""
        if self.lastY is not None:
            self.metrics.update(observation, self.lastY)
            self.lastY = None
        return observation

    def next_episode_idx(self, num_eps=None):
        if not num_eps:
            num_eps = len(self)
        epoch_done = False
        if self.random:
            episode_idx = random.randrange(num_eps)
        else:
            episode_idx = (self.episode_idx + self.step_size) % num_eps
            if episode_idx + self.step_size > num_eps:
                epoch_done = True
        return episode_idx, epoch_done
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import logging
import time
import uuid

# Sleep constants
THREAD_SHORT_SLEEP = 0.1
THREAD_MEDIUM_SLEEP = 0.3
# ThrottlingException might happen if we poll too frequently
THREAD_MTURK_POLLING_SLEEP = 10

logger = None
logging_enabled = True
debug = True
log_level = logging.ERROR

if logging_enabled:
    logging.basicConfig(
        filename=str(time.time()) + '.log',
        filemode='w',
        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
        datefmt='%H:%M:%S',
        level=logging.DEBUG
    )
    logger = logging.getLogger('mturk')

def set_log_level(new_level):
    global log_level
    log_level = new_level

def set_is_debug(is_debug):
    global debug
    debug = is_debug

def disable_logging():
    global logging_enabled
    logging_enabled = False

def print_and_log(level, message, should_print=False):
    if (logging_enabled and level >= log_level) or debug:
        logger.log(level, message)
    if should_print or debug:  # Always print message in debug mode
        print(message)

def generate_event_id(worker_id):
    """Return a unique id to use for identifying a packet for a worker"""
    return '{}_{}'.format(worker_id, uuid.uuid4())
<commit_msg>[MTurk] [Easy] Should not print debug by default (#1452)<commit_after>#!/usr/bin/env python3

# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import logging
import time
import uuid

# Sleep constants
THREAD_SHORT_SLEEP = 0.1
THREAD_MEDIUM_SLEEP = 0.3
# ThrottlingException might happen if we poll too frequently
THREAD_MTURK_POLLING_SLEEP = 10

logger = None
logging_enabled = True
debug = False
log_level = logging.ERROR

if logging_enabled:
    logging.basicConfig(
        filename=str(time.time()) + '.log',
        filemode='w',
        format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
        datefmt='%H:%M:%S',
        level=logging.DEBUG
    )
    logger = logging.getLogger('mturk')

def set_log_level(new_level):
    global log_level
    log_level = new_level

def set_is_debug(is_debug):
    global debug
    debug = is_debug

def disable_logging():
    global logging_enabled
    logging_enabled = False

def print_and_log(level, message, should_print=False):
    if (logging_enabled and level >= log_level) or debug:
        logger.log(level, message)
    if should_print or debug:  # Always print message in debug mode
        print(message)

def generate_event_id(worker_id):
    """Return a unique id to use for identifying a packet for a worker"""
    return '{}_{}'.format(worker_id, uuid.uuid4())
<|endoftext|>


<commit_before>from collections import OrderedDict, namedtuple
import traceback
import datetime
import asyncio
import logging
import json
import uuid
import os

from discord.ext import commands
import discord
import aiohttp
import asyncpg

from utils.utils import pluralize, TIME_BRIEF, TIME_LONG, tb_args, db_decode

Response = namedtuple('Response', 'status data')

IGNORE_EXCS = (discord.Forbidden,  # people keep disallowing send_messages
               commands.CommandNotFound,
               commands.MissingRequiredArgument,
               commands.NoPrivateMessage)

async def command_prefix(bot, message):
    """Custom prefix function for guild-specific prefixes."""
    default = bot.default_prefix
    if message.guild is None:
        return default
    async with bot.db_pool.acquire() as con:
        settings = await con.fetchrow('''
            SELECT * FROM prefixes WHERE guild_id = $1
            ''', str(message.guild.id))
    if settings is None:
        return default
    prefix, allow_default = settings['prefix'], settings['allow_default']
    valid = [db_decode(prefix)]
    if allow_default:
        if isinstance(default, (tuple, list)):
            valid.extend(default)
        else:
            valid.append(default)
    return valid

class LagBot(commands.Bot):
    def __init__(self, *args, config_file, debug=False, **kwargs):
        self._debug = debug
        self.config_file = config_file
        self.config_lock = asyncio.Lock(loop=self.loop)
        self.loop.run_until_complete(self.load_config())
        self.game = self.config.get('game')
        status = discord.Status.dnd if self._debug else discord.Status.online
        game = self.game if self.game is None else discord.Game(name=self.game)
        super().__init__(*args, game=game, status=status, **kwargs)
        self._before_invoke = self._before_invoke_
        self._after_invoke = self._after_invoke_
        self.default_prefix = self.command_prefix
        self.resumes = 0
        if self._debug:
            self.command_prefix = '?!'
        else:
            self.command_prefix = command_prefix
        useragent = 'Discord Bot'
        source = self.config.get('source')
        if source is not None:
            useragent += ' ' + source
        self.http_ = aiohttp.ClientSession(loop=self.loop, headers={'User-Agent': useragent})
        self.db_pool = self.loop.run_until_complete(
            asyncpg.create_pool(database='lagbot', command_timeout=10, loop=self.loop))

    def _config_to_file(self):
        app_path, config_file = os.path.split(self.config_file)
        temp = os.path.join(app_path, f'{uuid.uuid4()}-{config_file}.tmp')
        with open(temp, 'w', encoding='utf-8') as fp:
            json.dump(self.config.copy(), fp, indent=4, ensure_ascii=True)
        os.replace(temp, self.config_file)

    def _config_from_file(self):
        try:
            with open(self.config_file, 'r', encoding='utf-8') as fp:
                self.config = json.load(fp, object_pairs_hook=OrderedDict)
        except FileNotFoundError:
            self.config = {}

    async def save_config(self):
        with await self.config_lock:
            await self.loop.run_in_executor(None, self._config_to_file)

    async def load_config(self):
        with await self.config_lock:
            await self.loop.run_in_executor(None, self._config_from_file)

    async def logout(self):
        await self.http_.close()
        await self.db_pool.close()
        await super().logout()

    def logout_(self):
        """Mainly for use in signal handler, may have other uses."""
        self.loop.create_task(self.logout())

    def run(self, *args, **kwargs):
        super().run(self.config['bot_token'], *args, **kwargs)
        try:
            return self.exit_status
        except AttributeError:
            return 0

    async def on_ready(self):
        if hasattr(self, 'start_time'):
            logging.info('Ready again.')
            self.resumes += 1
            await self.set_game(self.game)
            return
        self.start_time = datetime.datetime.utcnow()
        self.app = await self.application_info()
        self.owner_id = self.app.owner.id
        if self._debug:
            logging.info('Ready.')

    async def on_resumed(self):
        self.resumes += 1
        await self.set_game(self.game)

    async def on_message(self, msg):
        if msg.author.bot:
            return
        if self._debug:
            debug_channel = self.config.get('debug_channel')
            if debug_channel is None or msg.channel.id != int(debug_channel):
                return
        await self.process_commands(msg)

    async def _before_invoke_(self, ctx):
        if getattr(ctx.command, '_db', False):
            ctx.con = await self.db_pool.acquire()

    async def _after_invoke_(self, ctx):
        try:
            await self.db_pool.release(ctx.con)
        except AttributeError:
            pass

    async def on_command_error(self, ctx, exc):
        """Emulate default on_command_error and add guild + channel info."""
        original = getattr(exc, 'original', exc)
        if hasattr(ctx.command, 'on_error') or getattr(exc, 'handled', False) or isinstance(original, IGNORE_EXCS):
            return
        logging.warning(f'Ignoring exception in command {ctx.command}')
        msg = f'{ctx.message.content}\nin {"guild" if ctx.guild else "DM"}'
        outer_path = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
        tb = ''.join(traceback.format_exception(*tb_args(original))).replace(outer_path, '...')
        logging.error('\n'.join((msg, tb)))

        if not self._debug and isinstance(exc, commands.CommandInvokeError):
            try:
                await self.app.owner.send(f'{msg}\n```py\n{tb}\n```'.format(msg, tb))
            except:
                pass

    async def wait_for(self, *args, ignore_timeout=False, **kwargs):
        """Override default wait_for to allow ignoring TimeoutError."""
        if ignore_timeout:
            try:
                return await super().wait_for(*args, **kwargs)
            except asyncio.TimeoutError:
                return None
        else:
            return await super().wait_for(*args, **kwargs)

    async def _request(self, url, type_='json', *, timeout=10, method='GET', **kwargs):
        if type_ not in {'json', 'read', 'text'}:
            return
        if kwargs.get('data') and method == 'GET':
            method = 'POST'
        async with self.http_.request(method, url, timeout=timeout, **kwargs) as resp:
            data = None
            try:
                data = await getattr(resp, type_)()
            except:
                logging.exception(f'Failed getting type {type_} from "{url}".')
            return Response(resp.status, data)

    async def request(self, *args, ignore_timeout=True, **kwargs):
        """Utility request function.

        type_ is the method to get data from response
        """
        if ignore_timeout:
            try:
                return await self._request(*args, **kwargs)
            except asyncio.TimeoutError:
                return Response(None, None)
        else:
            return await self._request(*args, **kwargs)

    async def set_game(self, game=None):
        self.game = game
        status = None
        for guild in self.guilds:
            if isinstance(guild.me, discord.Member):
                status = guild.me.status
                break
        if game is None:
            game = 'Resumes:'
            if self.resumes == 0:
                await self.change_presence(status=status)
                return
        await self.change_presence(game=discord.Game(name=f'{game} {self.resumes or ""}'), status=status)

    def get_uptime(self, brief=False):
        now = datetime.datetime.utcnow()
        delta = now - self.start_time
        hours, remainder = divmod(int(delta.total_seconds()), 3600)
        minutes, seconds = divmod(remainder, 60)
        days, hours = divmod(hours, 24)

        if brief:
            fmt = TIME_BRIEF
            joiner = ' '
        else:
            fmt = TIME_LONG
            joiner = ', '

        for ind, time in enumerate((days, hours, minutes, seconds, None)):
            if time:
                fmt = fmt[ind:]
                break
            elif time is None:
                fmt = [fmt[3]]

        return pluralize(joiner.join(fmt).format(d=days, h=hours, m=minutes, s=seconds))
<commit_msg>fix loop not being defined for config_lock<commit_after>from collections import OrderedDict, namedtuple
import traceback
import datetime
import asyncio
import logging
import json
import uuid
import os

from discord.ext import commands
import discord
import aiohttp
import asyncpg

from utils.utils import pluralize, TIME_BRIEF, TIME_LONG, tb_args, db_decode

Response = namedtuple('Response', 'status data')

IGNORE_EXCS = (discord.Forbidden,  # people keep disallowing send_messages
               commands.CommandNotFound,
               commands.MissingRequiredArgument,
               commands.NoPrivateMessage)

async def command_prefix(bot, message):
    """Custom prefix function for guild-specific prefixes."""
    default = bot.default_prefix
    if message.guild is None:
        return default
    async with bot.db_pool.acquire() as con:
        settings = await con.fetchrow('''
            SELECT * FROM prefixes WHERE guild_id = $1
            ''', str(message.guild.id))
    if settings is None:
        return default
    prefix, allow_default = settings['prefix'], settings['allow_default']
    valid = [db_decode(prefix)]
    if allow_default:
        if isinstance(default, (tuple, list)):
            valid.extend(default)
        else:
            valid.append(default)
    return valid

class LagBot(commands.Bot):
    def __init__(self, *args, config_file, debug=False, **kwargs):
        self._debug = debug
        self.loop = kwargs.get('loop', asyncio.get_event_loop())
        self.config_file = config_file
        self.config_lock = asyncio.Lock(loop=self.loop)
        self.loop.run_until_complete(self.load_config())
        self.game = self.config.get('game')
        status = discord.Status.dnd if self._debug else discord.Status.online
        game = self.game if self.game is None else discord.Game(name=self.game)
        super().__init__(*args, game=game, status=status, loop=self.loop, **kwargs)
        self._before_invoke = self._before_invoke_
        self._after_invoke = self._after_invoke_
        self.default_prefix = self.command_prefix
        self.resumes = 0
        if self._debug:
            self.command_prefix = '?!'
        else:
            self.command_prefix = command_prefix
        useragent = 'Discord Bot'
        source = self.config.get('source')
        if source is not None:
            useragent += ' ' + source
        self.http_ = aiohttp.ClientSession(loop=self.loop, headers={'User-Agent': useragent})
        self.db_pool = self.loop.run_until_complete(
            asyncpg.create_pool(database='lagbot', command_timeout=10, loop=self.loop))

    def _config_to_file(self):
        app_path, config_file = os.path.split(self.config_file)
        temp = os.path.join(app_path, f'{uuid.uuid4()}-{config_file}.tmp')
        with open(temp, 'w', encoding='utf-8') as fp:
            json.dump(self.config.copy(), fp, indent=4, ensure_ascii=True)
        os.replace(temp, self.config_file)

    def _config_from_file(self):
        try:
            with open(self.config_file, 'r', encoding='utf-8') as fp:
                self.config = json.load(fp, object_pairs_hook=OrderedDict)
        except FileNotFoundError:
            self.config = {}

    async def save_config(self):
        with await self.config_lock:
            await self.loop.run_in_executor(None, self._config_to_file)

    async def load_config(self):
        with await self.config_lock:
            await self.loop.run_in_executor(None, self._config_from_file)

    async def logout(self):
        await self.http_.close()
        await self.db_pool.close()
        await super().logout()

    def logout_(self):
        """Mainly for use in signal handler, may have other uses."""
        self.loop.create_task(self.logout())

    def run(self, *args, **kwargs):
        super().run(self.config['bot_token'], *args, **kwargs)
        try:
            return self.exit_status
        except AttributeError:
            return 0

    async def on_ready(self):
        if hasattr(self, 'start_time'):
            logging.info('Ready again.')
            self.resumes += 1
            await self.set_game(self.game)
            return
        self.start_time = datetime.datetime.utcnow()
        self.app = await self.application_info()
        self.owner_id = self.app.owner.id
        if self._debug:
            logging.info('Ready.')

    async def on_resumed(self):
        self.resumes += 1
        await self.set_game(self.game)

    async def on_message(self, msg):
        if msg.author.bot:
            return
        if self._debug:
            debug_channel = self.config.get('debug_channel')
            if debug_channel is None or msg.channel.id != int(debug_channel):
                return
        await self.process_commands(msg)

    async def _before_invoke_(self, ctx):
        if getattr(ctx.command, '_db', False):
            ctx.con = await self.db_pool.acquire()

    async def _after_invoke_(self, ctx):
        try:
            await self.db_pool.release(ctx.con)
        except AttributeError:
            pass

    async def on_command_error(self, ctx, exc):
        """Emulate default on_command_error and add guild + channel info."""
        original = getattr(exc, 'original', exc)
        if hasattr(ctx.command, 'on_error') or getattr(exc, 'handled', False) or isinstance(original, IGNORE_EXCS):
            return
        logging.warning(f'Ignoring exception in command {ctx.command}')
        msg = f'{ctx.message.content}\nin {"guild" if ctx.guild else "DM"}'
        outer_path = os.path.split(os.path.split(os.path.abspath(__file__))[0])[0]
        tb = ''.join(traceback.format_exception(*tb_args(original))).replace(outer_path, '...')
        logging.error('\n'.join((msg, tb)))

        if not self._debug and isinstance(exc, commands.CommandInvokeError):
            try:
                await self.app.owner.send(f'{msg}\n```py\n{tb}\n```'.format(msg, tb))
            except:
                pass

    async def wait_for(self, *args, ignore_timeout=False, **kwargs):
        """Override default wait_for to allow ignoring TimeoutError."""
        if ignore_timeout:
            try:
                return await super().wait_for(*args, **kwargs)
            except asyncio.TimeoutError:
                return None
        else:
            return await super().wait_for(*args, **kwargs)

    async def _request(self, url, type_='json', *, timeout=10, method='GET', **kwargs):
        if type_ not in {'json', 'read', 'text'}:
            return
        if kwargs.get('data') and method == 'GET':
            method = 'POST'
        async with self.http_.request(method, url, timeout=timeout, **kwargs) as resp:
            data = None
            try:
                data = await getattr(resp, type_)()
            except:
                logging.exception(f'Failed getting type {type_} from "{url}".')
            return Response(resp.status, data)

    async def request(self, *args, ignore_timeout=True, **kwargs):
        """Utility request function.

        type_ is the method to get data from response
        """
        if ignore_timeout:
            try:
                return await self._request(*args, **kwargs)
            except asyncio.TimeoutError:
                return Response(None, None)
        else:
            return await self._request(*args, **kwargs)

    async def set_game(self, game=None):
        self.game = game
        status = None
        for guild in self.guilds:
            if isinstance(guild.me, discord.Member):
                status = guild.me.status
                break
        if game is None:
            game = 'Resumes:'
            if self.resumes == 0:
                await self.change_presence(status=status)
                return
        await self.change_presence(game=discord.Game(name=f'{game} {self.resumes or ""}'), status=status)

    def get_uptime(self, brief=False):
        now = datetime.datetime.utcnow()
        delta = now - self.start_time
        hours, remainder = divmod(int(delta.total_seconds()), 3600)
        minutes, seconds = divmod(remainder, 60)
        days, hours = divmod(hours, 24)

        if brief:
            fmt = TIME_BRIEF
            joiner = ' '
        else:
            fmt = TIME_LONG
            joiner = ', '

        for ind, time in enumerate((days, hours, minutes, seconds, None)):
            if time:
                fmt = fmt[ind:]
                break
            elif time is None:
                fmt = [fmt[3]]

        return pluralize(joiner.join(fmt).format(d=days, h=hours, m=minutes, s=seconds))
<|endoftext|>


<commit_before>''' Home of the configuration defaults.

'''

import collections
import pprint
import logging

logger = logging.getLogger(__name__)

def pp_config (config):
    ''' Pretty print the config, this should be part of the
    default logging to the debug logs.

    Args:
        - config (dict) : The config data structure
    '''

    logger.debug(pprint.pformat(config, indent=2))
    return

def recursive_update (template, userdata):
    ''' Recursively update the template with userdata.
    If we don't do this the value updates for nested collections
    would get simply overwritten rathen than updated.

    Args:
        template (dict) : The templated dict
        userdata (dict) : User supplied config dict structure

    Returns:
        Updated template stucture.
    '''

    for k, v in userdata.items():
        if isinstance (v, collections.Mapping):
            template[k] = recursive_update(template.get(k, {}), v)
        else:
            template[k] = v

    return template

def update_config (config, rundir):
    ''' Update the config datastructure with defaults. This is the one centralized
    location where the default live.

    Args:
         - config (dict) : The config dictionary

    Returns:
         - A standardized config dict if a config was passed, else None.
    '''

    if not config :
        return None

    config_base = { "sites" : [],
                    "globals" : {
                        "lazyErrors" : False,
                        "usageTracking" : True,
                        "strategy" : "simple"
                    },
                    "controller" : {
                        "mode" : "auto"
                    }
    }

    sites = config["sites"] ; del config["sites"]

    config_base.update(config)

    config_sites = []
    for site in sites:
        site_base = {
            "auth" : {},
            "execution" : {
                "scriptDir" : rundir,
                "block" : {
                    "nodes" : 1,
                    "taskBlocks" : 1,
                    "minBlocks"  : 0,
                    "initBlocks" : 0,
                    "maxBlocks"  : 10,
                    "parallelism" : 0.75,
                    "walltime"   : "00:20:00",
                    "options" : {
                    }
                }
            }
        }

        recursive_update(site_base, site)
        config_sites.extend([site_base])

    config_base["sites"] = config_sites

    pp_config(config_base)

    return config_base
<commit_msg>Fixing issue with partial config updates #74.<commit_after>''' Home of the configuration defaults.

'''

import collections
import pprint
import logging

logger = logging.getLogger(__name__)

def pp_config (config):
    ''' Pretty print the config, this should be part of the
    default logging to the debug logs.

    Args:
        - config (dict) : The config data structure
    '''

    logger.debug(pprint.pformat(config, indent=2))
    return

def recursive_update (template, userdata):
    ''' Recursively update the template with userdata.
    If we don't do this the value updates for nested collections
    would get simply overwritten rathen than updated.

    Args:
        template (dict) : The templated dict
        userdata (dict) : User supplied config dict structure

    Returns:
        Updated template stucture.
    '''

    for k, v in userdata.items():
        if isinstance (v, collections.Mapping):
            template[k] = recursive_update(template.get(k, {}), v)
        else:
            template[k] = v

    return template

def update_config (config, rundir):
    ''' Update the config datastructure with defaults. This is the one centralized
    location where the default live.

    Args:
         - config (dict) : The config dictionary

    Returns:
         - A standardized config dict if a config was passed, else None.
    '''

    if not config :
        return None

    config_base = { "sites" : [],
                    "globals" : {
                        "lazyErrors" : False,
                        "usageTracking" : True,
                        "strategy" : "simple"
                    },
                    "controller" : {
                        "mode" : "auto"
                    }
    }

    sites = config["sites"] ; del config["sites"]

    recursive_update(config_base, config)

    config_sites = []
    for site in sites:
        site_base = {
            "auth" : {},
            "execution" : {
                "scriptDir" : rundir,
                "block" : {
                    "nodes" : 1,
                    "taskBlocks" : 1,
                    "minBlocks"  : 0,
                    "initBlocks" : 0,
                    "maxBlocks"  : 10,
                    "parallelism" : 0.75,
                    "walltime"   : "00:20:00",
                    "options" : {
                    }
                }
            }
        }

        recursive_update(site_base, site)
        config_sites.extend([site_base])

    config_base["sites"] = config_sites

    pp_config(config_base)

    return config_base
<|endoftext|>


<commit_before>##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Partner Sales ABC',
    'version': "13.0.1.0.0",
    'website': 'www.adhoc.com.ar',
    'author': 'ADHOC SA',
    'license': 'AGPL-3',
    'category': 'Tools',
    'depends': [
        'base',
        'mail'
    ],
    'data': [
        'views/partner_view.xml',
    ],
    'test': [],
    'installable': False,
    'auto_install': False,
    'application': True,
}
<commit_msg>[MIG] partner_sales_abc: Migration to 15.0<commit_after>##############################################################################
#
#    Copyright (C) 2015  ADHOC SA  (http://www.adhoc.com.ar)
#    All Rights Reserved.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
    'name': 'Partner Sales ABC',
    'version': "15.0.1.0.0",
    'website': 'www.adhoc.com.ar',
    'author': 'ADHOC SA',
    'license': 'AGPL-3',
    'category': 'Tools',
    'depends': [
        'contacts',
    ],
    'data': [
        'views/partner_view.xml',
    ],
    'test': [],
    'installable': True,
    'auto_install': False,
    'application': True,
}
<|endoftext|>


<commit_before>import sys

from six import b as b_
from .recursive import ForwardRequestException, RecursionLoop

class StatusPersist(object):

    def __init__(self, app, status, url):
        self.app = app
        self.status = status
        self.url = url

    def __call__(self, environ, start_response):
        def keep_status_start_response(status, headers, exc_info=None):
            return start_response(self.status, headers, exc_info)
        parts = self.url.split('?')
        environ['PATH_INFO'] = parts[0]
        if len(parts) > 1:
            environ['QUERY_STRING'] = parts[1]
        else:
            environ['QUERY_STRING'] = ''

        try:
            return self.app(environ, keep_status_start_response)
        except RecursionLoop as e:
            environ['wsgi.errors'].write(
                'Recursion error getting error page: %s\n' % e
            )
            keep_status_start_response(
                '500 Server Error',
                [('Content-type', 'text/plain')],
                sys.exc_info()
            )
            return [b_(
                'Error: %s.  (Error page could not be fetched)' % self.status
            )]

class ErrorDocumentMiddleware(object):
    '''
    Intersects HTTP response status code, looks it up in the error map defined
    in the Pecan app config.py, and routes to the controller assigned to that
    status.
    '''
    def __init__(self, app, error_map):
        self.app = app
        self.error_map = error_map

    def __call__(self, environ, start_response):

        def replacement_start_response(status, headers, exc_info=None):
            '''
            Overrides the default response if the status is defined in the
            Pecan app error map configuration.
            '''
            try:
                status_code = int(status.split(' ')[0])
            except (ValueError, TypeError):  # pragma: nocover
                raise Exception((
                    'ErrorDocumentMiddleware received an invalid '
                    'status %s' % status
                ))

            if status_code in self.error_map:
                def factory(app):
                    return StatusPersist(
                        app,
                        status,
                        self.error_map[status_code]
                    )
                raise ForwardRequestException(factory=factory)
            return start_response(status, headers, exc_info)

        app_iter = self.app(environ, replacement_start_response)
        return app_iter
<commit_msg>Fix a Py3 compatability issue with WebTest > 1.3.1 < 2.0<commit_after>import sys

from six import b as b_
from .recursive import ForwardRequestException, RecursionLoop

class StatusPersist(object):

    def __init__(self, app, status, url):
        self.app = app
        self.status = status
        self.url = url

    def __call__(self, environ, start_response):
        def keep_status_start_response(status, headers, exc_info=None):
            return start_response(self.status, headers, exc_info)
        parts = self.url.split('?')
        environ['PATH_INFO'] = parts[0]
        if len(parts) > 1:
            environ['QUERY_STRING'] = parts[1]
        else:
            environ['QUERY_STRING'] = ''

        try:
            return self.app(environ, keep_status_start_response)
        except RecursionLoop as e:
            environ['wsgi.errors'].errors.write(
                'Recursion error getting error page: %s\n' % e
            )
            keep_status_start_response(
                '500 Server Error',
                [('Content-type', 'text/plain')],
                sys.exc_info()
            )
            return [b_(
                'Error: %s.  (Error page could not be fetched)' % self.status
            )]

class ErrorDocumentMiddleware(object):
    '''
    Intersects HTTP response status code, looks it up in the error map defined
    in the Pecan app config.py, and routes to the controller assigned to that
    status.
    '''
    def __init__(self, app, error_map):
        self.app = app
        self.error_map = error_map

    def __call__(self, environ, start_response):

        def replacement_start_response(status, headers, exc_info=None):
            '''
            Overrides the default response if the status is defined in the
            Pecan app error map configuration.
            '''
            try:
                status_code = int(status.split(' ')[0])
            except (ValueError, TypeError):  # pragma: nocover
                raise Exception((
                    'ErrorDocumentMiddleware received an invalid '
                    'status %s' % status
                ))

            if status_code in self.error_map:
                def factory(app):
                    return StatusPersist(
                        app,
                        status,
                        self.error_map[status_code]
                    )
                raise ForwardRequestException(factory=factory)
            return start_response(status, headers, exc_info)

        app_iter = self.app(environ, replacement_start_response)
        return app_iter
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This script will query for upcoming rocket launches and post results to Slack
"""

import sys
import json
import datetime
import requests

# Set the webhook_url to the one provided by Slack when you create
# the webhook at https://my.slack.com/services/new/incoming-webhook/
SLACK_WEBHOOK = "<WEBHOOK URL HERE>"

# Dev
#NEXT_LAUNCH = "https://lldev.thespacedevs.com/2.1.0/"
# Prod
NEXT_LAUNCH = "https://llv.thespacedevs.com/2.1.0/"
# Grab next 5 launches
# We'd be living in the future if there are >5 in a day
NEXT_LAUNCH += "launch/upcoming/?"

# Omit the launches that occurred in the past 24h
# since this script should run daily-ish

NEXT_LAUNCH += "hide_recent_previous=true"

def generatepayload():
    """
    Parse out the data and create a JSON payload for Slack
    """
    r = requests.get(NEXT_LAUNCH)

    # Return None if the request was unsuccessful
    if r.status_code != requests.codes.ok:
        return None

    # Get JSON output
    data = r.json()
    launch_data = data["results"]

    # Count the number of launches today
    launchCount = len(launch_data)
    for launch in launch_data:
        if datetime.datetime.utcnow() <= \
                datetime.datetime.strptime(launch["net"], "%Y-%m-%dT%H:%M:%SZ") <= \
                datetime.datetime.utcnow() + datetime.timedelta(hours=24):
            continue
        launchCount -= 1
    if launchCount == 0:
        sys.exit()
    datetime.timedelta(hours=0)

    # Create payload skeleton
    if launchCount > 1:
        payload = {"text": str(launchCount) + " launches scheduled in next 24h", "attachments": []}
    else:
        payload = {"text": str(launchCount) + " launch scheduled in next 24h", "attachments": []}

    counter = 0
    for launch_data in data["results"]:
        if counter < launchCount:
            # Extract primary launch agency
            agency = launch_data["launch_service_provider"]

            # Check if there is mission info available
            mission = "No Mission Info Available"
            for mission in launch_data["mission"]:
                mission = launch_data["mission"]["description"]
                mission_type = launch_data["mission"]["type"]

            image = ""
            image = launch_data["image"]

            liftoff_ts = ""
            footer = ""
            if launch_data["net"] != 0:
                footer = "Expected Launch Time"
                liftoff_ts = (datetime.datetime.strptime(
                    launch_data["net"], "%Y-%m-%dT%H:%M:%SZ") - \
                            datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)

            window_start = datetime.datetime.strptime(launch_data["window_start"], "%Y-%m-%dT%H:%M:%SZ")
            window_end = datetime.datetime.strptime(launch_data["window_end"], "%Y-%m-%dT%H:%M:%SZ")
            # generate the launch info attachment
            payload["attachments"].append({
                "fallback": "Information",
                "color": "good",
                "author_name": agency["name"],
                "author_link": agency["url"],
                "author_icon": "https://cdn4.iconfinder.com/data/icons/whsr-january-flaticon-set/512/rocket.png",
                "title": launch_data["name"],
                "text": mission,
                "image_url": image,
                "fields": [
                    {
                        "title": "Launch Location",
                        "value": launch_data["pad"]["location"]["name"],
                        "short": True
                    },
                    {
                        "title": "Launch Window",
                        "value": str(window_end - window_start),
                        "short": True
                    },
                    {
                        "title": "Mission Type",
                        "value": mission_type,
                        "short": True
                    }
                ],
                "footer": footer,
                "footer_icon": "",
                "ts": liftoff_ts,
                "actions": []
            })

            # Attach video buttons
            # check if there is a video stream available
            if launch_data["webcast_live"]:
                for item in launch_data["webcast_live"]:
                    video = item
                    payload["attachments"][counter]["actions"].append({
                        "type": "button",
                        "name": "LaunchStream",
                        "text": "Video Stream ",
                        "style": "primary",
                        "url": video
                    })
            counter += 1

    return json.dumps(payload)

def postdata(message_payload):
    """
    Send json message payload to Slack webhook
    """
    requests.post(
        SLACK_WEBHOOK,
        data=message_payload,
        headers={"Content-Type": "application/json"}
    )

message = generatepayload()
postdata(message)
<commit_msg>Update URLs, fix typo in prod URL<commit_after>#!/usr/bin/env python

"""
This script will query for upcoming rocket launches and post results to Slack
"""

import sys
import json
import datetime
import requests

# Set the webhook_url to the one provided by Slack when you create
# the webhook at https://my.slack.com/services/new/incoming-webhook/
SLACK_WEBHOOK = "<WEBHOOK URL HERE>"

# Dev
#NEXT_LAUNCH = "https://lldev.thespacedevs.com"
# Prod
NEXT_LAUNCH = "http://ll.thespacedevs.com"

# API Version
NEXT_LAUNCH += "/2.1.0/"

# Grab next 5 launches
# We'd be living in the future if there are >5 in a day
NEXT_LAUNCH += "launch/upcoming/?"

# Omit the launches that occurred in the past 24h
# since this script should run daily-ish
NEXT_LAUNCH += "hide_recent_previous=true"

def generatepayload():
    """
    Parse out the data and create a JSON payload for Slack
    """
    r = requests.get(NEXT_LAUNCH)

    # Return None if the request was unsuccessful
    if r.status_code != requests.codes.ok:
        return None

    # Get JSON output
    data = r.json()
    launch_data = data["results"]

    # Count the number of launches today
    launchCount = len(launch_data)
    for launch in launch_data:
        if datetime.datetime.utcnow() <= \
                datetime.datetime.strptime(launch["net"], "%Y-%m-%dT%H:%M:%SZ") <= \
                datetime.datetime.utcnow() + datetime.timedelta(hours=24):
            continue
        launchCount -= 1
    if launchCount == 0:
        sys.exit()
    datetime.timedelta(hours=0)

    # Create payload skeleton
    if launchCount > 1:
        payload = {"text": str(launchCount) + " launches scheduled in next 24h", "attachments": []}
    else:
        payload = {"text": str(launchCount) + " launch scheduled in next 24h", "attachments": []}

    counter = 0
    for launch_data in data["results"]:
        if counter < launchCount:
            # Extract primary launch agency
            agency = launch_data["launch_service_provider"]

            # Check if there is mission info available
            mission = "No Mission Info Available"
            for mission in launch_data["mission"]:
                mission = launch_data["mission"]["description"]
                mission_type = launch_data["mission"]["type"]

            image = ""
            image = launch_data["image"]

            liftoff_ts = ""
            footer = ""
            if launch_data["net"] != 0:
                footer = "Expected Launch Time"
                liftoff_ts = (datetime.datetime.strptime(
                    launch_data["net"], "%Y-%m-%dT%H:%M:%SZ") - \
                            datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)

            window_start = datetime.datetime.strptime(launch_data["window_start"], "%Y-%m-%dT%H:%M:%SZ")
            window_end = datetime.datetime.strptime(launch_data["window_end"], "%Y-%m-%dT%H:%M:%SZ")
            # generate the launch info attachment
            payload["attachments"].append({
                "fallback": "Information",
                "color": "good",
                "author_name": agency["name"],
                "author_link": agency["url"],
                "author_icon": "https://cdn4.iconfinder.com/data/icons/whsr-january-flaticon-set/512/rocket.png",
                "title": launch_data["name"],
                "text": mission,
                "image_url": image,
                "fields": [
                    {
                        "title": "Launch Location",
                        "value": launch_data["pad"]["location"]["name"],
                        "short": True
                    },
                    {
                        "title": "Launch Window",
                        "value": str(window_end - window_start),
                        "short": True
                    },
                    {
                        "title": "Mission Type",
                        "value": mission_type,
                        "short": True
                    }
                ],
                "footer": footer,
                "footer_icon": "",
                "ts": liftoff_ts,
                "actions": []
            })

            # Attach video buttons
            # check if there is a video stream available
            if launch_data["webcast_live"]:
                for item in launch_data["webcast_live"]:
                    video = item
                    payload["attachments"][counter]["actions"].append({
                        "type": "button",
                        "name": "LaunchStream",
                        "text": "Video Stream ",
                        "style": "primary",
                        "url": video
                    })
            counter += 1

    return json.dumps(payload)

def postdata(message_payload):
    """
    Send json message payload to Slack webhook
    """
    requests.post(
        SLACK_WEBHOOK,
        data=message_payload,
        headers={"Content-Type": "application/json"}
    )

message = generatepayload()
postdata(message)
<|endoftext|>


<commit_before>import boto, simplejson, urllib2, requests
from random import randint
from tumblpy import *

published_url = "https://s3.amazonaws.com/sunglints_oy_ya__/published.txt"
blog_url = 'sunglint.tumblr.com'
post_tags = 'sunglint,NASA,Earth'

# AWS creds
aws_access_key_id = os.environ['aws_access_key_id'])
aws_secret_access_key = os.environ['aws_secret_access_key'])
bucket_name = os.environ['bucket_name'])

# Tumblr creds
app_key = os.environ['app_key'])
app_secret = os.environ['app_secret'])
oauth_token = os.environ['oauth_token'])
oauth_token_secret = os.environ['oauth_token_secret'])

class ScienceData:

	def get_dataset(self):
		# fetch the dataset, it's in a public google spreadsheet:
		response = requests.get('https://docs.google.com/spreadsheet/ccc?key=0AtHxCskz4p33dEhBZ3Jpc1VoTmIzQ0V3ODJ5eVdzcUE&output=csv')
		assert response.status_code == 200, 'Wrong status code'
		all_rows = [r.split(',') for r in response.content.split("\n")]
		return all_rows

	def total_image_count(self):
		all_rows = self.get_dataset()
		total_image_count = sum(1 for img in all_rows if img[all_rows[0].index('Online Source')].upper().find('.JPG') > -1) # some rows contain non-images so checking for that
		return total_image_count

class BlogPosts:

	def get_previous(self):
		# returns the list previously published images from public url on s3
		response = urllib2.urlopen(published_url)
		published = [p.rstrip() for p in response.readlines()]

		# are we at the end? have they all been published?
		if len(published) == ScienceData().total_image_count:
			published = [] # every post has been published, starting over from scratch

		return published

	def get_most_recent(self):
		return "most recent time"

class Image:

	# pick an image at random that hasn't been picked before
	def random(self):
		published = BlogPosts().get_previous()
		all_rows = ScienceData().get_dataset()
		column_titles = all_rows[0]
		total_image_count = ScienceData().total_image_count()
		random_row = all_rows[randint(1,len(all_rows)-1)]
		random_image = dict(zip(column_titles, random_row)) # make a nice json version - http://stackoverflow.com/questions/209840/map-two-lists-into-a-dictionary-in-python
		url = random_image['Online Source']
		if (url in published) | ((url.upper().find('.JPG') < 0) & (url.upper().find('.JPEG') < 0)): # this is already published or it's not an image
			return self.random() # game over try again
		return random_image

	# post to tumblr
	def post_to_tumblr(self, random_image):

		print random_image
		t = Tumblpy(app_key = app_key,
		        app_secret = app_secret,
		        oauth_token = oauth_token,
		        oauth_token_secret=oauth_token_secret)
		title = random_image['Title'].title()
		description = random_image['Abstract'].title()
		credit = random_image['NASA Center']
		link = random_image['NTRS Link']
		date = random_image['Publication Year']
		img_url = random_image['Online Source']
		acc_no = random_image['Accession Number']
		caption = """

{0}

{1}

					{2}

					{3}

					{4}
via ntrs.nasa.gov
				"""
		caption = caption.format(title,date,description,credit,acc_no,link).strip()
		img = urllib2.urlopen(img_url)
		post = t.post('post', blog_url=blog_url, params={'type':'photo', 'caption': str(caption), 'data': img, 'link':str(img_url), 'tags':str(post_tags)})
		return post['id'] if post['id'] else False

	def log_as_published(self,image_url):
		# update the published log
		published = BlogPosts().get_previous()
		published.append(image_url)
		s3 = boto.connect_s3(aws_access_key_id,aws_secret_access_key)
		bucket = s3.create_bucket(bucket_name)
		key_name = published_url.split('/').pop()
		bucket.delete_key(key_name)
		key = bucket.new_key(key_name)
		key.set_contents_from_string("\n".join(published))
		key.set_acl('public-read')

try:
	img = Image()
	random_image = img.random()
	img.post_to_tumblr(random_image)
	img.log_as_published(random_image['Online Source'])
except:
	pass

<commit_msg>setup creds for Heroku<commit_after>import boto, simplejson, urllib2, requests
from random import randint
from tumblpy import *

published_url = "https://s3.amazonaws.com/sunglints_oy_ya__/published.txt"
blog_url = 'sunglint.tumblr.com'
post_tags = 'sunglint,NASA,Earth'

# AWS creds
aws_access_key_id = os.environ['aws_access_key_id']
aws_secret_access_key = os.environ['aws_secret_access_key']
bucket_name = os.environ['bucket_name']

# Tumblr creds
app_key = os.environ['app_key']
app_secret = os.environ['app_secret']
oauth_token = os.environ['oauth_token']
oauth_token_secret = os.environ['oauth_token_secret']

class ScienceData:

	def get_dataset(self):
		# fetch the dataset, it's in a public google spreadsheet:
		response = requests.get('https://docs.google.com/spreadsheet/ccc?key=0AtHxCskz4p33dEhBZ3Jpc1VoTmIzQ0V3ODJ5eVdzcUE&output=csv')
		assert response.status_code == 200, 'Wrong status code'
		all_rows = [r.split(',') for r in response.content.split("\n")]
		return all_rows

	def total_image_count(self):
		all_rows = self.get_dataset()
		total_image_count = sum(1 for img in all_rows if img[all_rows[0].index('Online Source')].upper().find('.JPG') > -1) # some rows contain non-images so checking for that
		return total_image_count

class BlogPosts:

	def get_previous(self):
		# returns the list previously published images from public url on s3
		response = urllib2.urlopen(published_url)
		published = [p.rstrip() for p in response.readlines()]

		# are we at the end? have they all been published?
		if len(published) == ScienceData().total_image_count:
			published = [] # every post has been published, starting over from scratch

		return published

	def get_most_recent(self):
		return "most recent time"

class Image:

	# pick an image at random that hasn't been picked before
	def random(self):
		published = BlogPosts().get_previous()
		all_rows = ScienceData().get_dataset()
		column_titles = all_rows[0]
		total_image_count = ScienceData().total_image_count()
		random_row = all_rows[randint(1,len(all_rows)-1)]
		random_image = dict(zip(column_titles, random_row)) # make a nice json version - http://stackoverflow.com/questions/209840/map-two-lists-into-a-dictionary-in-python
		url = random_image['Online Source']
		if (url in published) | ((url.upper().find('.JPG') < 0) & (url.upper().find('.JPEG') < 0)): # this is already published or it's not an image
			return self.random() # game over try again
		return random_image

	# post to tumblr
	def post_to_tumblr(self, random_image):

		print random_image
		t = Tumblpy(app_key = app_key,
		        app_secret = app_secret,
		        oauth_token = oauth_token,
		        oauth_token_secret=oauth_token_secret)
		title = random_image['Title'].title()
		description = random_image['Abstract'].title()
		credit = random_image['NASA Center']
		link = random_image['NTRS Link']
		date = random_image['Publication Year']
		img_url = random_image['Online Source']
		acc_no = random_image['Accession Number']
		caption = """

{0}

{1}

					{2}

					{3}

					{4}
via ntrs.nasa.gov
				"""
		caption = caption.format(title,date,description,credit,acc_no,link).strip()
		img = urllib2.urlopen(img_url)
		post = t.post('post', blog_url=blog_url, params={'type':'photo', 'caption': str(caption), 'data': img, 'link':str(img_url), 'tags':str(post_tags)})
		return post['id'] if post['id'] else False

	def log_as_published(self,image_url):
		# update the published log
		published = BlogPosts().get_previous()
		published.append(image_url)
		s3 = boto.connect_s3(aws_access_key_id,aws_secret_access_key)
		bucket = s3.create_bucket(bucket_name)
		key_name = published_url.split('/').pop()
		bucket.delete_key(key_name)
		key = bucket.new_key(key_name)
		key.set_contents_from_string("\n".join(published))
		key.set_acl('public-read')

try:
	img = Image()
	random_image = img.random()
	img.post_to_tumblr(random_image)
	img.log_as_published(random_image['Online Source'])
except:
	pass

<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This script will query for upcoming rocket launches and post results to Slack
"""

import os
import sys
import json
import datetime
import requests

# Set the webhook_url to the one provided by Slack when you create
# the webhook at https://my.slack.com/services/new/incoming-webhook/
print(f'Running now {datetime.datetime.now()}')
SLACK_WEBHOOK = os.environ.get('SLACK_WEBHOOK')

# Dev
# NEXT_LAUNCH = "https://lldev.thespacedevs.com"
# Prod
NEXT_LAUNCH = "http://ll.thespacedevs.com"

# API Version
NEXT_LAUNCH += "/2.2.0/"

# Grab next 5 launches
# We'd be living in the future if there are >5 in a day
NEXT_LAUNCH += "launch/upcoming/?"

# Omit the launches that occurred in the past 24h
# since this script should run daily-ish
NEXT_LAUNCH += "hide_recent_previous=true"

def generatepayload():
    """
    Parse out the data and create a JSON payload for Slack
    """
    r = requests.get(NEXT_LAUNCH)

    # Return None if the request was unsuccessful
    if r.status_code != requests.codes.ok:
        return None

    # Get JSON output
    data = r.json()
    launch_data = data["results"]

    # Count the number of launches today
    launchCount = len(launch_data)
    for launch in launch_data:
        if datetime.datetime.utcnow() <= \
                datetime.datetime.strptime(launch["net"], "%Y-%m-%dT%H:%M:%SZ") <= \
                datetime.datetime.utcnow() + datetime.timedelta(hours=24):
            continue
        launchCount -= 1
    if launchCount == 0:
        sys.exit()
    datetime.timedelta(hours=0)

    # Create payload skeleton
    if launchCount > 1:
        payload = {"text": str(launchCount) + " launches scheduled in next 24h", "attachments": []}
    else:
        payload = {"text": str(launchCount) + " launch scheduled in next 24h", "attachments": []}

    counter = 0
    for launch_data in data["results"]:
        if counter < launchCount:
            # Extract primary launch agency
            agency = launch_data["launch_service_provider"]

            # Check if there is mission info available
            mission = "No Mission Info Available"
            for mission in launch_data["mission"]:
                mission = launch_data["mission"]["description"]
                mission_type = launch_data["mission"]["type"]

            image = ""
            image = launch_data["image"]

            liftoff_ts = ""
            footer = ""
            if launch_data["net"] != 0:
                footer = "Expected Launch Time"
                liftoff_ts = (datetime.datetime.strptime(
                    launch_data["net"], "%Y-%m-%dT%H:%M:%SZ") -
                            datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)

            window_start = datetime.datetime.strptime(
                    launch_data["window_start"], "%Y-%m-%dT%H:%M:%SZ")
            window_end = datetime.datetime.strptime(
                    launch_data["window_end"], "%Y-%m-%dT%H:%M:%SZ")
            # generate the launch info attachment
            payload["attachments"].append({
                "fallback": "Information",
                "color": "good",
                "author_name": agency["name"],
                "author_link": agency["url"],
                "author_icon": "https://cdn4.iconfinder.com/data/icons/whsr-january-flaticon-set/512/rocket.png",
                "title": launch_data["name"],
                "text": mission,
                "image_url": image,
                "fields": [
                    {
                        "title": "Launch Location",
                        "value": launch_data["pad"]["location"]["name"],
                        "short": True
                    },
                    {
                        "title": "Launch Window",
                        "value": str(window_end - window_start),
                        "short": True
                    },
                    {
                        "title": "Mission Type",
                        "value": mission_type,
                        "short": True
                    }
                ],
                "footer": footer,
                "footer_icon": "",
                "ts": liftoff_ts,
                "actions": []
            })

            # Attach video buttons
            # check if there is a video stream available
            if launch_data["webcast_live"]:
                for item in launch_data["webcast_live"]:
                    video = item
                    payload["attachments"][counter]["actions"].append({
                        "type": "button",
                        "name": "LaunchStream",
                        "text": "Video Stream ",
                        "style": "primary",
                        "url": video
                    })
            counter += 1

    return json.dumps(payload)

def postdata(message_payload):
    """
    Send json message payload to Slack webhook
    """
    requests.post(
        SLACK_WEBHOOK,
        data=message_payload,
        timeout=60,
        headers={"Content-Type": "application/json"}
    )

message = generatepayload()
postdata(message)
<commit_msg>Split API call and payload gen<commit_after>#!/usr/bin/env python

"""
This script will query for upcoming rocket launches and post results to Slack
"""

import os
import sys
import json
import datetime
import requests
from requests import codes

# Set the webhook_url to the one provided by Slack when you create
# the webhook at https://my.slack.com/services/new/incoming-webhook/
print(f'Running now {datetime.datetime.now()}')
SLACK_WEBHOOK = os.environ.get('SLACK_WEBHOOK')

# Dev
# NEXT_LAUNCH = "https://lldev.thespacedevs.com"
# Prod
NEXT_LAUNCH = "http://ll.thespacedevs.com"

# API Version
NEXT_LAUNCH += "/2.2.0/"

# Grab next launches iterate over a single page of API results
# We'd be living in the future if there are >5 in a day
# (N.B. There were 5 launches on 3 Aug 2022)
NEXT_LAUNCH += "launch/upcoming/?"

# Omit the launches that occurred in the past 24h
# since this script should run daily-ish
NEXT_LAUNCH += "hide_recent_previous=true"

def getLaunches():
    """
    Get info from the Launch Library API
    """
    r = requests.get(NEXT_LAUNCH)

    # Return None if the request was unsuccessful
    if r.status_code != codes.ok:  # pylint: disable=E1101
        return None

    # Get JSON output
    data = r.json()
    launch_data = data["results"]

    # Count the number of launches today
    launchCount = len(launch_data)
    for launch in launch_data:
        if datetime.datetime.utcnow() <= \
                datetime.datetime.strptime(launch["net"], "%Y-%m-%dT%H:%M:%SZ") <= \
                datetime.datetime.utcnow() + datetime.timedelta(hours=24):
            continue
        launchCount -= 1
    if launchCount == 0:
        print("No launches in the next 24 hours")
        sys.exit()
    return launch_data, launchCount

def generatepayload(launches, count):
    """
    Parse out the data and create a JSON payload for Slack
    """
    launch_data = launches
    launchCount = count
    datetime.timedelta(hours=0)

    # Create payload skeleton
    if launchCount > 1:
        payload = {"text": str(launchCount) + " launches scheduled in next 24h", "attachments": []}
    else:
        payload = {"text": str(launchCount) + " launch scheduled in next 24h", "attachments": []}

    counter = 0
    for launch_data in launches:
        if counter < launchCount:
            # Extract primary launch agency
            agency = launch_data["launch_service_provider"]

            # Check if there is mission info available
            mission = "No Mission Info Available"
            for mission in launch_data["mission"]:
                mission = launch_data["mission"]["description"]
                mission_type = launch_data["mission"]["type"]

            image = ""
            image = launch_data["image"]

            liftoff_ts = ""
            footer = ""
            if launch_data["net"] != 0:
                footer = "Expected Launch Time"
                liftoff_ts = (datetime.datetime.strptime(
                    launch_data["net"], "%Y-%m-%dT%H:%M:%SZ") -
                            datetime.datetime(1970, 1, 1)) / datetime.timedelta(seconds=1)

            window_start = datetime.datetime.strptime(
                    launch_data["window_start"], "%Y-%m-%dT%H:%M:%SZ")
            window_end = datetime.datetime.strptime(
                    launch_data["window_end"], "%Y-%m-%dT%H:%M:%SZ")
            if str(window_end - window_start) == "0:00:00":
                launch_window = "Instantaneous"
            else:
                launch_window = str(window_end - window_start)
            # generate the launch info attachment
            payload["attachments"].append({
                "fallback": "Information",
                "color": "good",
                "author_name": agency["name"],
                "author_link": agency["url"],
                "author_icon": "https://cdn4.iconfinder.com/data/icons/whsr-january-flaticon-set/512/rocket.png",
                "title": launch_data["name"],
                "text": mission,
                "image_url": image,
                "fields": [
                    {
                        "title": "Launch Location",
                        "value": launch_data["pad"]["location"]["name"],
                        "short": True
                    },
                    {
                        "title": "Launch Window",
                        "value": launch_window,
                        "short": True
                    },
                    {
                        "title": "Mission Type",
                        "value": mission_type,
                        "short": True
                    }
                ],
                "footer": footer,
                "footer_icon": "",
                "ts": liftoff_ts,
                "actions": []
            })

            # Attach video buttons
            # check if there is a video stream available
            if launch_data["webcast_live"]:
                for item in launch_data["webcast_live"]:
                    video = item
                    payload["attachments"][counter]["actions"].append({
                        "type": "button",
                        "name": "LaunchStream",
                        "text": "Video Stream ",
                        "style": "primary",
                        "url": video
                    })
            counter += 1

    return json.dumps(payload)

def postdata(message_payload):
    """
    Send json message payload to Slack webhook
    """
    requests.post(
        SLACK_WEBHOOK,
        data=message_payload,
        timeout=60,
        headers={"Content-Type": "application/json"}
    )

def run():
    """
    Do it to it
    Get the launches and post the Slack message
    """
    data, count = getLaunches()
    message = generatepayload(data, count)
    postdata(message)

if __name__ == "__main__":
    run()
<|endoftext|>