<commit_before>import os
import sys

import arcpy
import pythonaddins

# enable local imports
local_path = os.path.dirname(__file__)
sys.path.insert(0, local_path)

# import local settings
import config
import utils

# get the paths for our toolboxes
imported_toolbox = os.path.join(local_path, "toolbox", "imported.tbx")
genegis_toolbox = os.path.join(local_path, "toolbox", "genegis.pyt")

#
# data management
#
class ImportData(object):
    """Implementation for genegis_import.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, "ClassifiedImport")

class ExportGenAlEx(object):
    """Implementation for genegis_export_genalex_codominant.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, "ExportGenAlEx")

class ExportKML(object):
    """Implementation for genegis_export_kml.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        # TODO: update this to custom code (issue #22)
        utils.toolDialog("Conversion Tools", "LayerToKML")

class ExportSRGD(object):
    """Implementation for genegis_export_kml.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False

    def onClick(self):
        #with open("c:\\log\\genegis-export-srgd.csv", 'w') as f:
        #    f.write("onclick Called! trying to get layer combo value...\n")
        fc = config.selected_layer
        if fc is None:
            msg = "Nothing selected. Please enter a valid layer into the geneGIS selection box."
            title = "Export SRGD: ComboBox value"
            #f.write("writing message box...\n")
            pythonaddins.MessageBox(msg, title)
        else:
            #f.write("have a valid layer selected, %s\n" % fc)
            output_path = pythonaddins.SaveDialog("Output SRGD file name", "SRGD_%s.csv" % fc)
            #f.write("got output_path = `%s`\n" % output_path)
            utils.writeToSRGD(fc, output_path)

# summarization tools
#

class ExtractValuesToPoints(object):
    """Implementation for genegis_extract_values_to_points.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, 'extractRasterByPoints')

class SummarizeEncounters(object):
    """Implementation for genegis_summarize.tool (Tool)."""

    def __init__(self, display=True):
        self.enabled = True
        # XXX: If needed, this tool could be swapped to a combobox class, then
        # the 'shape' and 'cursor' can be set based on the selection type.
        self.shape = "RECTANGLE"
        self.cursor = 3 # the 'crosshair'
        self._extent = None
        # display controls whether this tool does its reporting,
        # or just passes on its results silently.
        self.display = display

    def onRectangle(self, rectangle_geometry):
        polygon_extent = utils.extentPolygon(rectangle_geometry)
        # the current _selected_ layer in ArcMap
        layer = utils.selectedLayer()
        if layer is None:
            return None

        # store the results in memory, no need to bring spindles into this
        output_feature = 'in_memory/primary_selection_points'
        utils.intersectFeatures(layer.name, polygon_extent, output_feature)

        # get the stats for our inviduals
        indiv_stats = utils.selectIndividuals(output_feature, self.display)
        """
        so we'd probably want to identify the specific columns of interest (haplotypes?),
        perhaps using a drop-down menu, and then use the select tool to generate our areas of
        interest, and shoot back out some summary statistics based on those observations.
        """

        results = {'indiv_stats': indiv_stats, 'output_feature': output_feature}

        # push results to a shared variable
        config.primary_results = results
        return results

class CompareEncounters(object):
    """Implementation for genegis_compare.tool (Tool)"""
    def __init__(self, display=True):
        self.enabled = True
        self.shape = "Rectangle"
        self.cursor = 3 # the 'crosshair'
        self._extent = None
        self.display = display

    def onRectangle(self, rectangle_geometry):
        polygon_extent = utils.extentPolygon(rectangle_geometry)
        layer = utils.selectedLayer()
        if layer is None:
            return None

        output_feature = 'in_memory/compare_selection_points'
        utils.intersectFeatures(layer.name, polygon_extent, output_feature)

        res2 = utils.selectIndividuals(output_feature, False)

        # now, get the results from the summarize encounters tool
        # XXX: ss = SummarizeEncounters()
        # XXX: ss.onRectangle() # doesn't work; won't get the geom

        if self.display:
            if config.primary_results is not None:
                res = config.primary_results['indiv_stats']

                common_indiv = res['unique'].intersection(res2['unique'])
                # compare the two sets of results
                msg = ("First Set:  {0} samples, "
                       "{1} unique individuals\n"
                       "Second Set: {2} samples, "
                       "{3} unique individuals\n\n"
                       "Common to both: {4}".format(
                           len(res['indiv']), len(res['unique']),
                           len(res2['indiv']), len(res2['unique']),
                           len(common_indiv)))
                title = "Comparison Results"
                pythonaddins.MessageBox(msg, title)
            else:
                pythonaddins.Messagebox("please select first set", "selection missing")

class LayerCombo(object):
    """Implementation for genegis_layer_combo.combobox (Combobox)"""
    def __init__(self):
        self.layers = []
        self.items = ['genetic_points']
        self.editable = True
        self.enabled = True
        self.dropdownWidth = "WWWWWWW"
        self.width = "WWWWWWW"
        self.value = None

    def onSelChange(self, selection):
        if selection:
            config.selected_layer = utils.getLayerByName(selection)
            # FIXME: check how much memory the object will soak up
            # prior to loading
            config.selected_object = None
        pass

    def onFocus(self, focused):
        # update the layer list _only_ on focus events, preventing this from
        # being triggered on the addin startup.
        if focused:
            self.layers = utils.currentLayers()
            if len(self.layers) > 0:
                self.items = [l.name for l in self.layers]
    def onEnter(self):
        pass
    def refresh(self):
        pass
<commit_msg>automatically include the configured layer to the combo box when it's recently changed; partial fix for #27.<commit_after>import os
import sys
import time

import arcpy
import pythonaddins

# enable local imports
local_path = os.path.dirname(__file__)
sys.path.insert(0, local_path)

# import local settings
import config
import utils

# get the paths for our toolboxes
imported_toolbox = os.path.join(local_path, "toolbox", "imported.tbx")
genegis_toolbox = os.path.join(local_path, "toolbox", "genegis.pyt")

#
# data management
#
class ImportData(object):
    """Implementation for genegis_import.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, "ClassifiedImport")

class ExportGenAlEx(object):
    """Implementation for genegis_export_genalex_codominant.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, "ExportGenAlEx")

class ExportKML(object):
    """Implementation for genegis_export_kml.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        # TODO: update this to custom code (issue #22)
        utils.toolDialog("Conversion Tools", "LayerToKML")

class ExportSRGD(object):
    """Implementation for genegis_export_kml.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False

    def onClick(self):
        #with open("c:\\log\\genegis-export-srgd.csv", 'w') as f:
        #    f.write("onclick Called! trying to get layer combo value...\n")
        fc = config.selected_layer
        if fc is None:
            msg = "Nothing selected. Please enter a valid layer into the geneGIS selection box."
            title = "Export SRGD: ComboBox value"
            #f.write("writing message box...\n")
            pythonaddins.MessageBox(msg, title)
        else:
            #f.write("have a valid layer selected, %s\n" % fc)
            output_path = pythonaddins.SaveDialog("Output SRGD file name", "SRGD_%s.csv" % fc)
            #f.write("got output_path = `%s`\n" % output_path)
            utils.writeToSRGD(fc, output_path)

# summarization tools
#

class ExtractValuesToPoints(object):
    """Implementation for genegis_extract_values_to_points.button (Button)"""
    def __init__(self):
        self.enabled = True
        self.checked = False
    def onClick(self):
        utils.toolDialog(genegis_toolbox, 'extractRasterByPoints')

class SummarizeEncounters(object):
    """Implementation for genegis_summarize.tool (Tool)."""

    def __init__(self, display=True):
        self.enabled = True
        # XXX: If needed, this tool could be swapped to a combobox class, then
        # the 'shape' and 'cursor' can be set based on the selection type.
        self.shape = "RECTANGLE"
        self.cursor = 3 # the 'crosshair'
        self._extent = None
        # display controls whether this tool does its reporting,
        # or just passes on its results silently.
        self.display = display

    def onRectangle(self, rectangle_geometry):
        polygon_extent = utils.extentPolygon(rectangle_geometry)
        # the current _selected_ layer in ArcMap
        layer = utils.selectedLayer()
        if layer is None:
            return None

        # store the results in memory, no need to bring spindles into this
        output_feature = 'in_memory/primary_selection_points'
        utils.intersectFeatures(layer.name, polygon_extent, output_feature)

        # get the stats for our inviduals
        indiv_stats = utils.selectIndividuals(output_feature, self.display)
        """
        so we'd probably want to identify the specific columns of interest (haplotypes?),
        perhaps using a drop-down menu, and then use the select tool to generate our areas of
        interest, and shoot back out some summary statistics based on those observations.
        """

        results = {'indiv_stats': indiv_stats, 'output_feature': output_feature}

        # push results to a shared variable
        config.primary_results = results
        return results

class CompareEncounters(object):
    """Implementation for genegis_compare.tool (Tool)"""
    def __init__(self, display=True):
        self.enabled = True
        self.shape = "Rectangle"
        self.cursor = 3 # the 'crosshair'
        self._extent = None
        self.display = display

    def onRectangle(self, rectangle_geometry):
        polygon_extent = utils.extentPolygon(rectangle_geometry)
        layer = utils.selectedLayer()
        if layer is None:
            return None

        output_feature = 'in_memory/compare_selection_points'
        utils.intersectFeatures(layer.name, polygon_extent, output_feature)

        res2 = utils.selectIndividuals(output_feature, False)

        # now, get the results from the summarize encounters tool
        # XXX: ss = SummarizeEncounters()
        # XXX: ss.onRectangle() # doesn't work; won't get the geom

        if self.display:
            if config.primary_results is not None:
                res = config.primary_results['indiv_stats']

                common_indiv = res['unique'].intersection(res2['unique'])
                # compare the two sets of results
                msg = ("First Set:  {0} samples, "
                       "{1} unique individuals\n"
                       "Second Set: {2} samples, "
                       "{3} unique individuals\n\n"
                       "Common to both: {4}".format(
                           len(res['indiv']), len(res['unique']),
                           len(res2['indiv']), len(res2['unique']),
                           len(common_indiv)))
                title = "Comparison Results"
                pythonaddins.MessageBox(msg, title)
            else:
                pythonaddins.Messagebox("please select first set", "selection missing")

class LayerCombo(object):
    """Implementation for genegis_layer_combo.combobox (Combobox)"""
    def __init__(self):
        self.layers = []
        self.items = ['genetic_points']
        self.editable = True
        self.enabled = True
        self.dropdownWidth = "WWWWWWW"
        self.width = "WWWWWWW"
        self.value = self.getFeatureClassFromDisk(timeout=3600)

    def getFeatureClassFromDisk(self, timeout=None):
        # this is a hack -- if the class has been updated recently, refresh the list
        # diff_in_sec = time.time() - os.path.getmtime()
        # if diff_in_sec < 300: do the thang
        if os.path.exists(config.fc_path_file):
            diff_in_sec = time.time() - os.path.getmtime(config.fc_path_file)
            if timeout is None or diff_in_sec <= timeout:
                with open(config.fc_path_file) as f:
                    layer = utils.loadLayer(f.read())
        return layer

    def onSelChange(self, selection):
        if selection:
            config.selected_layer = utils.getLayerByName(selection)
            # FIXME: check how much memory the object will soak up
            # prior to loading
            config.selected_object = None
        pass

    def onFocus(self, focused):
        # update the layer list _only_ on focus events, preventing this from
        # being triggered on the addin startup.
        if focused:
            self.layers = utils.currentLayers()
            if len(self.layers) > 0:
                self.items = [l.name for l in self.layers]
    def onEnter(self):
        pass
    def refresh(self):
        pass
<|endoftext|>


<commit_before>"""
Example generation from python files.

Generate the rst files for the examples by iterating over the python
example files. Files that generate images should start with 'plot'.

To generate your own examples, just add ``'mpltools.sphinx.plot2rst'``` to the
list of ``extensions``in your Sphinx configuration file.

This code was adapted from scikits-image, which took it from scikits-learn.

Options
-------
The ``plot2rst`` extension accepts the following options:

    plot2rst_paths : length-2 tuple
        Paths to (python plot, generated rst) files, i.e. (source, destination).
        Note that both paths are relative to Sphinx 'source' path.

    plot2rst_rcparams : dict
        Matplotlib configuration parameters. See
        http://matplotlib.sourceforge.net/users/customizing.html for details.
"""
import os
import shutil
import traceback
import glob

import matplotlib
matplotlib.use('Agg')

import token, tokenize

plot_rst_template = """
.. _example_%(short_filename)s:

%(docstring)s

%(image_list)s

**Python source code:** :download:`%(src_name)s <%(src_name)s>`
(generated using ``mpltools`` |version|)

.. literalinclude:: %(src_name)s
    :lines: %(end_row)s-

"""

toctree_template = """
.. toctree::
   :hidden:

   %s

"""

CLEAR_SECTION = """
.. raw:: html

    <div style="clear: both"></div>

"""

MULTI_IMAGE_HEADER = """
.. rst-class:: multi-image

"""

MULTI_IMAGE_TEMPLATE = """

      .. image:: images/%s
          :align: center

"""

SINGLE_IMAGE_TEMPLATE = """
.. image:: images/%s
    :align: center

"""

GALLERY_HEADER = """

Examples
========

.. _examples-index:

"""

def setup(app):
    app.connect('builder-inited', generate_rst_gallery)

    app.add_config_value('plot2rst_paths',
                         ('../examples', 'auto_examples'), True)
    app.add_config_value('plot2rst_rcparams', {}, True)

def generate_rst_gallery(app):
    """Add list of examples and gallery to Sphinx app."""
    cfg = app.builder.config

    doc_src = os.path.abspath(app.builder.srcdir) # path/to/doc/source

    plot_path, rst_path = cfg.plot2rst_paths
    rst_dir = os.path.join(doc_src, rst_path)
    example_dir = os.path.join(doc_src, plot_path)

    if not os.path.exists(example_dir):
        print "No example directory found at", example_dir
        return
    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    # we create an index.rst with all examples
    gallery_index = file(os.path.join(rst_dir, 'index'+cfg.source_suffix), 'w')
    gallery_index.write(GALLERY_HEADER)

    # Here we don't use an os.walk, but we recurse only twice: flat is
    # better than nested.
    write_gallery(gallery_index, example_dir, rst_dir, cfg)
    for d in sorted(os.listdir(example_dir)):
        example_sub = os.path.join(example_dir, d)
        if os.path.isdir(example_sub):
            rst_sub = os.path.join(rst_dir, d)
            if not os.path.exists(rst_sub):
                os.makedirs(rst_sub)
            write_gallery(gallery_index, example_sub, rst_sub, cfg, depth=1)
    gallery_index.flush()

def write_gallery(gallery_index, src_dir, rst_dir, cfg, depth=0):
    """Generate the rst files for an example directory, i.e. gallery.

    Write rst files from python examples and add example links to gallery.

    Parameters
    ----------
    gallery_index : file
        Index file for plot gallery.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    index_name = 'index' + cfg.source_suffix
    if not os.path.exists(os.path.join(src_dir, index_name)):
        print src_dir
        print 80*'_'
        print ('Example directory %s does not have a %s file'
                        % (src_dir, index_name))
        print 'Skipping this directory'
        print 80*'_'
        return

    gallery_description = file(os.path.join(src_dir, index_name)).read()
    gallery_index.write('\n\n%s\n\n' % gallery_description)

    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    def sort_key(a):
        # Elements that are not plots should be last.
        if not valid_plot_script(a):
            return 'zz' + a
        return a

    examples = [fname for fname in sorted(os.listdir(src_dir), key=sort_key)
                      if fname.endswith('py')]
    ex_names = [ex[:-3] for ex in examples] # strip '.py' extension
    if depth == 0:
        sub_dir = ''
    else:
        sub_dir_list = src_dir.split('/')[-depth:]
        sub_dir = '/'.join(sub_dir_list) + '/'
    gallery_index.write(toctree_template % (sub_dir + '\n   '.join(ex_names)))

    write = gallery_index.write
    for src_name in examples:
        rst_file_from_example(src_name, src_dir, rst_dir, cfg)
        thumb = os.path.join(sub_dir, 'images/thumb', src_name[:-3] + '.png')
        gallery_index.write('.. figure:: %s\n' % thumb)

        link_name = sub_dir + src_name
        link_name = link_name.replace(os.path.sep, '_')
        if link_name.startswith('._'):
            link_name = link_name[2:]

        write('   :figclass: gallery\n')
        write('   :target: ./%s.html\n\n' % (sub_dir + src_name[:-3]))
        write('   :ref:`example_%s`\n\n' % (link_name))
    write(CLEAR_SECTION) # clear at the end of the section

def valid_plot_script(src_name):
    return src_name.startswith('plot') and src_name.endswith('.py')

def rst_file_from_example(src_name, src_dir, rst_dir, cfg):
    """Write rst file from a given python example.

    Parameters
    ----------
    src_name : str
        Name of example file.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    base_image_name = os.path.splitext(src_name)[0]
    image_fmt_str = '%s_%%s.png' % base_image_name

    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'

    info = dict(src_name=src_name)
    info['short_filename'] = last_dir + src_name
    src_path = os.path.join(src_dir, src_name)
    example_file = os.path.join(rst_dir, src_name)
    shutil.copyfile(src_path, example_file)

    image_dir = os.path.join(rst_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_path = os.path.join(thumb_dir, src_name[:-3] + '.png')
    image_path = os.path.join(image_dir, image_fmt_str)
    figure_list = save_plot(src_path, image_path, thumb_path, cfg)

    if not os.path.exists(thumb_path):
        # create something to stand in for the thumbnail
        # TODO: figure out a way that doesn't require storing an image.
        #shutil.copy('source/auto_examples/images/blank_image.png', thumb_path)
        pass

    docstring, short_desc, end_row = extract_module_docstring(example_file)
    info['docstring'] = docstring
    info['end_row'] = end_row

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.

    if len(figure_list) == 1:
        figure_name = figure_list[0]
        image_list = SINGLE_IMAGE_TEMPLATE % figure_name.lstrip('/')
    else:
        image_list = MULTI_IMAGE_HEADER
        for figure_name in figure_list:
            image_list += MULTI_IMAGE_TEMPLATE % figure_name.lstrip('/')
    info['image_list'] = image_list

    basename, py_ext = os.path.splitext(src_name)
    f = open(os.path.join(rst_dir, basename + cfg.source_suffix),'w')
    f.write(plot_rst_template % info)
    f.flush()

def save_plot(src_path, image_path, thumb_path, cfg):
    """Save plots as images.

    Parameters
    ----------
    src_path : str
        Path to example file.
    image_path : str
        Path where plots are saved (format string with single string argument).
    thumbpath : str
        Path where thumbnails of plots are saved.
    cfg : config object
        Sphinx config object created by Sphinx.

    Returns
    -------
    figure_list : list
        List of figure names saved by the example.
    """
    figure_list = []

    src_dir, src_name = os.path.split(src_path)
    if not src_name.startswith('plot'):
        return figure_list

    image_dir, image_fmt_str = os.path.split(image_path)
    first_image_file = image_path % 1

    needs_replot = (not os.path.exists(first_image_file) or
                    mod_time(first_image_file) <= mod_time(src_path))
    if needs_replot:
        print 'plotting %s' % src_name
        import matplotlib.pyplot as plt
        plt.rcdefaults()
        plt.rcParams.update(cfg.plot2rst_rcparams)
        plt.close('all')
        cwd = os.getcwd()
        try:
            # Plot example in source directory.
            os.chdir(src_dir)
            execfile(src_name, {'pl' : plt})
            os.chdir(cwd)

            fig_mngr = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
            # Save every figure by looping over all open figures.
            for fig_num in (m.num for m in fig_mngr):
                # Set the fig_num figure as the current figure as we can't
                # save a figure that's not the current figure.
                plt.figure(fig_num)
                plt.savefig(image_path % fig_num)
                figure_list.append(image_fmt_str % fig_num)
        except:
            print 80*'_'
            print '%s is not compiling:' % src_name
            traceback.print_exc()
            print 80*'_'
        finally:
            os.chdir(cwd)
    else:
        figure_list = [f[len(image_dir):]
                        for f in glob.glob(image_path % '[1-9]')]

    # generate thumb file
    from matplotlib import image
    if os.path.exists(first_image_file):
        image.thumbnail(first_image_file, thumb_path, 0.2)

    return figure_list

def mod_time(file_path):
    return os.stat(file_path).st_mtime

def extract_module_docstring(src_name):
    """Return module-level docstring.

    Parameters
    ----------
    src_name : str
        Name of python example.

    Returns
    -------
    docstring : str
        Module docstring.
    first_par : str
        First paragraph of docstring.
    end_first_par : int
        Line where first

    """
    lines = file(src_name).readlines()

    docstring = ''
    first_par = ''
    tokens = tokenize.generate_tokens(lines.__iter__().next)
    for tok_type, tok_content, _, (erow, _), _ in tokens:
        tok_type = token.tok_name[tok_type]
        if tok_type in ('NEWLINE', 'COMMENT', 'NL', 'INDENT', 'DEDENT'):
            continue
        elif tok_type == 'STRING':
            docstring = eval(tok_content)
            # Extract the first paragraph of docstring:
            lines = docstring.split('\n')
            clean_docstring = '\n'.join(line.rstrip() for line in lines)
            paragraphs = clean_docstring.split('\n\n')
            if len(paragraphs) > 0:
                first_par = paragraphs[0]
        break
    end_first_par = erow + 1
    return docstring, first_par, end_first_par

<commit_msg>Refactor: Simplify syntax for tokenizing.<commit_after>"""
Example generation from python files.

Generate the rst files for the examples by iterating over the python
example files. Files that generate images should start with 'plot'.

To generate your own examples, just add ``'mpltools.sphinx.plot2rst'``` to the
list of ``extensions``in your Sphinx configuration file.

This code was adapted from scikits-image, which took it from scikits-learn.

Options
-------
The ``plot2rst`` extension accepts the following options:

    plot2rst_paths : length-2 tuple
        Paths to (python plot, generated rst) files, i.e. (source, destination).
        Note that both paths are relative to Sphinx 'source' path.

    plot2rst_rcparams : dict
        Matplotlib configuration parameters. See
        http://matplotlib.sourceforge.net/users/customizing.html for details.
"""
import os
import shutil
import traceback
import glob

import matplotlib
matplotlib.use('Agg')

import token, tokenize

plot_rst_template = """
.. _example_%(short_filename)s:

%(docstring)s

%(image_list)s

**Python source code:** :download:`%(src_name)s <%(src_name)s>`
(generated using ``mpltools`` |version|)

.. literalinclude:: %(src_name)s
    :lines: %(end_row)s-

"""

toctree_template = """
.. toctree::
   :hidden:

   %s

"""

CLEAR_SECTION = """
.. raw:: html

"""

MULTI_IMAGE_HEADER = """
.. rst-class:: multi-image

"""

MULTI_IMAGE_TEMPLATE = """

      .. image:: images/%s
          :align: center

"""

SINGLE_IMAGE_TEMPLATE = """
.. image:: images/%s
    :align: center

"""

GALLERY_HEADER = """

Examples
========

.. _examples-index:

"""

def setup(app):
    app.connect('builder-inited', generate_rst_gallery)

    app.add_config_value('plot2rst_paths',
                         ('../examples', 'auto_examples'), True)
    app.add_config_value('plot2rst_rcparams', {}, True)

def generate_rst_gallery(app):
    """Add list of examples and gallery to Sphinx app."""
    cfg = app.builder.config

    doc_src = os.path.abspath(app.builder.srcdir) # path/to/doc/source

    plot_path, rst_path = cfg.plot2rst_paths
    rst_dir = os.path.join(doc_src, rst_path)
    example_dir = os.path.join(doc_src, plot_path)

    if not os.path.exists(example_dir):
        print "No example directory found at", example_dir
        return
    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    # we create an index.rst with all examples
    gallery_index = file(os.path.join(rst_dir, 'index'+cfg.source_suffix), 'w')
    gallery_index.write(GALLERY_HEADER)

    # Here we don't use an os.walk, but we recurse only twice: flat is
    # better than nested.
    write_gallery(gallery_index, example_dir, rst_dir, cfg)
    for d in sorted(os.listdir(example_dir)):
        example_sub = os.path.join(example_dir, d)
        if os.path.isdir(example_sub):
            rst_sub = os.path.join(rst_dir, d)
            if not os.path.exists(rst_sub):
                os.makedirs(rst_sub)
            write_gallery(gallery_index, example_sub, rst_sub, cfg, depth=1)
    gallery_index.flush()

def write_gallery(gallery_index, src_dir, rst_dir, cfg, depth=0):
    """Generate the rst files for an example directory, i.e. gallery.

    Write rst files from python examples and add example links to gallery.

    Parameters
    ----------
    gallery_index : file
        Index file for plot gallery.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    index_name = 'index' + cfg.source_suffix
    if not os.path.exists(os.path.join(src_dir, index_name)):
        print src_dir
        print 80*'_'
        print ('Example directory %s does not have a %s file'
                        % (src_dir, index_name))
        print 'Skipping this directory'
        print 80*'_'
        return

    gallery_description = file(os.path.join(src_dir, index_name)).read()
    gallery_index.write('\n\n%s\n\n' % gallery_description)

    if not os.path.exists(rst_dir):
        os.makedirs(rst_dir)

    def sort_key(a):
        # Elements that are not plots should be last.
        if not valid_plot_script(a):
            return 'zz' + a
        return a

    examples = [fname for fname in sorted(os.listdir(src_dir), key=sort_key)
                      if fname.endswith('py')]
    ex_names = [ex[:-3] for ex in examples] # strip '.py' extension
    if depth == 0:
        sub_dir = ''
    else:
        sub_dir_list = src_dir.split('/')[-depth:]
        sub_dir = '/'.join(sub_dir_list) + '/'
    gallery_index.write(toctree_template % (sub_dir + '\n   '.join(ex_names)))

    write = gallery_index.write
    for src_name in examples:
        rst_file_from_example(src_name, src_dir, rst_dir, cfg)
        thumb = os.path.join(sub_dir, 'images/thumb', src_name[:-3] + '.png')
        gallery_index.write('.. figure:: %s\n' % thumb)

        link_name = sub_dir + src_name
        link_name = link_name.replace(os.path.sep, '_')
        if link_name.startswith('._'):
            link_name = link_name[2:]

        write('   :figclass: gallery\n')
        write('   :target: ./%s.html\n\n' % (sub_dir + src_name[:-3]))
        write('   :ref:`example_%s`\n\n' % (link_name))
    write(CLEAR_SECTION) # clear at the end of the section

def valid_plot_script(src_name):
    return src_name.startswith('plot') and src_name.endswith('.py')

def rst_file_from_example(src_name, src_dir, rst_dir, cfg):
    """Write rst file from a given python example.

    Parameters
    ----------
    src_name : str
        Name of example file.
    src_dir : 'str'
        Source directory for python examples.
    rst_dir : 'str'
        Destination directory for rst files generated from python examples.
    cfg : config object
        Sphinx config object created by Sphinx.
    """
    base_image_name = os.path.splitext(src_name)[0]
    image_fmt_str = '%s_%%s.png' % base_image_name

    last_dir = os.path.split(src_dir)[-1]
    # to avoid leading . in file names, and wrong names in links
    if last_dir == '.' or last_dir == 'examples':
        last_dir = ''
    else:
        last_dir += '_'

    info = dict(src_name=src_name)
    info['short_filename'] = last_dir + src_name
    src_path = os.path.join(src_dir, src_name)
    example_file = os.path.join(rst_dir, src_name)
    shutil.copyfile(src_path, example_file)

    image_dir = os.path.join(rst_dir, 'images')
    thumb_dir = os.path.join(image_dir, 'thumb')
    if not os.path.exists(image_dir):
        os.makedirs(image_dir)
    if not os.path.exists(thumb_dir):
        os.makedirs(thumb_dir)

    thumb_path = os.path.join(thumb_dir, src_name[:-3] + '.png')
    image_path = os.path.join(image_dir, image_fmt_str)
    figure_list = save_plot(src_path, image_path, thumb_path, cfg)

    if not os.path.exists(thumb_path):
        # create something to stand in for the thumbnail
        # TODO: figure out a way that doesn't require storing an image.
        #shutil.copy('source/auto_examples/images/blank_image.png', thumb_path)
        pass

    docstring, short_desc, end_row = extract_module_docstring(example_file)
    info['docstring'] = docstring
    info['end_row'] = end_row

    # Depending on whether we have one or more figures, we're using a
    # horizontal list or a single rst call to 'image'.

    if len(figure_list) == 1:
        figure_name = figure_list[0]
        image_list = SINGLE_IMAGE_TEMPLATE % figure_name.lstrip('/')
    else:
        image_list = MULTI_IMAGE_HEADER
        for figure_name in figure_list:
            image_list += MULTI_IMAGE_TEMPLATE % figure_name.lstrip('/')
    info['image_list'] = image_list

    basename, py_ext = os.path.splitext(src_name)
    f = open(os.path.join(rst_dir, basename + cfg.source_suffix),'w')
    f.write(plot_rst_template % info)
    f.flush()

def save_plot(src_path, image_path, thumb_path, cfg):
    """Save plots as images.

    Parameters
    ----------
    src_path : str
        Path to example file.
    image_path : str
        Path where plots are saved (format string with single string argument).
    thumbpath : str
        Path where thumbnails of plots are saved.
    cfg : config object
        Sphinx config object created by Sphinx.

    Returns
    -------
    figure_list : list
        List of figure names saved by the example.
    """
    figure_list = []

    src_dir, src_name = os.path.split(src_path)
    if not src_name.startswith('plot'):
        return figure_list

    image_dir, image_fmt_str = os.path.split(image_path)
    first_image_file = image_path % 1

    needs_replot = (not os.path.exists(first_image_file) or
                    mod_time(first_image_file) <= mod_time(src_path))
    if needs_replot:
        print 'plotting %s' % src_name
        import matplotlib.pyplot as plt
        plt.rcdefaults()
        plt.rcParams.update(cfg.plot2rst_rcparams)
        plt.close('all')
        cwd = os.getcwd()
        try:
            # Plot example in source directory.
            os.chdir(src_dir)
            execfile(src_name, {'pl' : plt})
            os.chdir(cwd)

            fig_mngr = matplotlib._pylab_helpers.Gcf.get_all_fig_managers()
            # Save every figure by looping over all open figures.
            for fig_num in (m.num for m in fig_mngr):
                # Set the fig_num figure as the current figure as we can't
                # save a figure that's not the current figure.
                plt.figure(fig_num)
                plt.savefig(image_path % fig_num)
                figure_list.append(image_fmt_str % fig_num)
        except:
            print 80*'_'
            print '%s is not compiling:' % src_name
            traceback.print_exc()
            print 80*'_'
        finally:
            os.chdir(cwd)
    else:
        figure_list = [f[len(image_dir):]
                        for f in glob.glob(image_path % '[1-9]')]

    # generate thumb file
    from matplotlib import image
    if os.path.exists(first_image_file):
        image.thumbnail(first_image_file, thumb_path, 0.2)

    return figure_list

def mod_time(file_path):
    return os.stat(file_path).st_mtime

def extract_module_docstring(src_name):
    """Return module-level docstring.

    Parameters
    ----------
    src_name : str
        Name of python example.

    Returns
    -------
    docstring : str
        Module docstring.
    first_par : str
        First paragraph of docstring.
    end_first_par : int
        Line where first

    """
    py = open(src_name)

    docstring = ''
    first_par = ''
    tokens = tokenize.generate_tokens(py.readline)
    for tok_type, tok_content, _, (erow, _), _ in tokens:
        tok_type = token.tok_name[tok_type]
        if tok_type in ('NEWLINE', 'COMMENT', 'NL', 'INDENT', 'DEDENT'):
            continue
        elif tok_type == 'STRING':
            docstring = eval(tok_content)
            # Extract the first paragraph of docstring:
            lines = docstring.split('\n')
            clean_docstring = '\n'.join(line.rstrip() for line in lines)
            paragraphs = clean_docstring.split('\n\n')
            if len(paragraphs) > 0:
                first_par = paragraphs[0]
        break
    end_first_par = erow + 1
    return docstring, first_par, end_first_par

<|endoftext|>


<commit_before>from conans import ConanFile
from conans import tools
import os
import re
import sys

class BoostConan(ConanFile):
	name = "Boost"
	version = "1.58.0"
	license = "Boost Software License, http://www.boost.org/LICENSE_1_0.txt"
	url = "http://www.boost.org"
	settings = "os", "arch", "compiler", "build_type"
	FOLDER_NAME = "boost_%s" % version.replace(".", "_")
	options = {"shared": [True, False], "header_only": [True, False]}
	default_options = "shared=False", "header_only=False"
	counter_config = 0

	def config(self):
		self.output.info("")
		self.output.info("---------- config ----------")
		self.output.info("")

		# If header only, the compiler, etc, does not affect the package!
		self.counter_config += 1
		# config is called twice, one before receive the upper dependencies and another before
		if self.options.header_only and self.counter_config==2:
			self.output.info("HEADER ONLY")
			self.settings.clear()
			self.options.remove("shared")
		if not self.options.header_only and self.settings.os == "Windows" and self.options.shared and "MT" in str(self.settings.compiler.runtime):
			self.options.shared = False

	def source(self):
		self.output.info("")
		self.output.info("---------- source ----------")
		self.output.info("")

		zip_name = "%s.zip" % self.FOLDER_NAME if sys.platform == "win32" else "%s.tar.gz" % self.FOLDER_NAME
		url = "http://sourceforge.net/projects/boost/files/boost/%s/%s/download" % (self.version, zip_name)
		self.output.info("Downloading %s..." % url)
		tools.download(url, zip_name, retry=3, retry_wait=10)
		tools.unzip(zip_name, ".")
		os.unlink(zip_name)

	def build(self):
		self.output.info("")
		self.output.info("---------- build ----------")
		self.output.info("")
		self.output.info("os        : " + str(self.settings.os))
		self.output.info("arch      : " + str(self.settings.arch))
		self.output.info("build_type: " + str(self.settings.build_type))

		if self.settings.compiler == "Visual Studio":
			self.output.info("runtime   : " + str(self.settings.compiler.runtime))

		if self.options.header_only:
			return

		if self.settings.os == "Windows":
			command = "bootstrap"
		else:
			command = "./bootstrap.sh"

		try:
			with tools.chdir(self.FOLDER_NAME):
				self.run(command)
		except:
			with tools.chdir(self.FOLDER_NAME):
				if self.settings.os == "Windows":
					self.run("type bootstrap.log")
				else:
					self.run("cat bootstrap.log")
			raise

		flags = []

		if self.settings.compiler == "Visual Studio":
			flags.append("toolset=msvc-%s.0" % self.settings.compiler.version)

		if self.settings.compiler == "gcc" and self.settings.arch == "armv7hf":
			self.output.info("Patch project-config.jam for ARMv7hf")

			with tools.chdir(self.FOLDER_NAME):
				f = open("project-config.jam", 'r+')
				content = f.read()
				# TODO finetune regular expression
				content_new = re.sub(" *    using gcc ;*", r"    using gcc : arm : arm-linux-gnueabihf-g++ ;", content, flags = re.M)
				# self.output.info("content new: " + content_new)
				f.seek(0)
				f.write(content_new)
				f.truncate()
				f.close()

			flags.append("toolset=gcc-arm")

		flags.append("link=%s" % ("static" if not self.options.shared else "shared"))

		if self.settings.compiler == "Visual Studio" and self.settings.compiler.runtime:
			flags.append("runtime-link=%s" % ("static" if "MT" in str(self.settings.compiler.runtime) else "shared"))

		flags.append("variant=%s" % str(self.settings.build_type).lower())

		if self.settings.arch != "armv7hf":
			flags.append("address-model=%s" % ("32" if self.settings.arch == "x86" else "64"))

		b2_flags = " ".join(flags)

		if self.settings.os == "Windows":
			command = "b2"
		else:
			command = "./b2"

		full_command = command + " " + b2_flags + " -j " + str(tools.cpu_count()) + " -s NO_BZIP2=1 --abbreviate-paths --without-python"

		self.output.warn(full_command)

		with tools.chdir(self.FOLDER_NAME):
			if self.settings.arch == "armv7hf":
				try:
					self.run(full_command)
				except:
					# TODO fix failing build
					self.output.warn("we knew that this would happen")
			else:
				self.run(full_command)

	def package(self):
		self.output.info("")
		self.output.info("---------- package ----------")
		self.output.info("")

		self.copy(pattern="*", dst="include/boost", src="%s/boost" % self.FOLDER_NAME)
		self.copy(pattern="*.a", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.so", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.so.*", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.dylib*", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.lib", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.dll", dst="bin", src="%s/stage/lib" % self.FOLDER_NAME)

	def package_info(self):
		self.output.info("")
		self.output.info("---------- package_info ----------")
		self.output.info("")

		if not self.options.header_only:
			if self.options.shared:
				self.cpp_info.defines.append("BOOST_ALL_DYN_LINK")
			else:
				self.cpp_info.defines.append("BOOST_USE_STATIC_LIBS")

			libs = ("atomic chrono container coroutine date_time exception filesystem "
					"graph iostreams locale log_setup log math_c99 math_c99f math_c99l math_tr1 "
					"math_tr1f math_tr1l prg_exec_monitor program_options random regex serialization "
					"signals system test_exec_monitor thread timer unit_test_framework wave "
					"wserialization").split()

		if self.settings.os != "Windows":
				self.cpp_info.libs.extend(["boost_%s" % lib for lib in libs])
		elif self.settings.os == "Windows":
			win_libs = []
			# http://www.boost.org/doc/libs/1_55_0/more/getting_started/windows.html
			visual_version = int(str(self.settings.compiler.version)) * 10
			runtime = "mt"# str(self.settings.compiler.runtime).lower()

			abi_tags = []
			if self.settings.compiler.runtime in ("MTd", "MT"):
				abi_tags.append("s")

			if self.settings.build_type == "Debug":
				abi_tags.append("gd")

			abi_tags = ("-%s" % "".join(abi_tags)) if abi_tags else ""

			version = "_".join(self.version.split(".")[0:2])
			suffix = "vc%d-%s%s-%s" % (visual_version, runtime, abi_tags, version)
			prefix = "lib" if not self.options.shared else ""

			win_libs.extend(["%sboost_%s-%s" % (prefix, lib, suffix) for lib in libs if lib not in ["exception", "test_exec_monitor"]])
			win_libs.extend(["libboost_exception-%s" % suffix, "libboost_test_exec_monitor-%s" % suffix])

			#self.output.warn("EXPORTED BOOST LIBRARIES: %s" % win_libs)
			self.cpp_info.libs.extend(win_libs)
			self.cpp_info.defines.extend(["BOOST_ALL_NO_LIB"]) # DISABLES AUTO LINKING! NO SMART AND MAGIC DECISIONS THANKS!
<commit_msg>extract compiler from CXX environment variable and disable libs not supported for ARM build of Boost Conan package recipe<commit_after>from conans import ConanFile
from conans import tools
import os
import re
import sys

class BoostConan(ConanFile):
	name = "Boost"
	version = "1.58.0"
	license = "Boost Software License, http://www.boost.org/LICENSE_1_0.txt"
	url = "http://www.boost.org"
	settings = {"os": ["Windows", "Linux"], "compiler": ["Visual Studio", "gcc"], "arch": ["x86", "x86_64", "armv7hf"], "build_type": ["Debug", "Release"]}
	FOLDER_NAME = "boost_%s" % version.replace(".", "_")
	options = {"shared": [True, False], "header_only": [True, False]}
	default_options = "shared=False", "header_only=False"
	counter_config = 0

	def config(self):
		self.output.info("")
		self.output.info("---------- config ----------")
		self.output.info("")

		# If header only, the compiler, etc, does not affect the package!
		self.counter_config += 1
		# config is called twice, one before receive the upper dependencies and another before
		if self.options.header_only and self.counter_config==2:
			self.output.info("HEADER ONLY")
			self.settings.clear()
			self.options.remove("shared")
		if not self.options.header_only and self.settings.os == "Windows" and self.options.shared and "MT" in str(self.settings.compiler.runtime):
			self.options.shared = False

	def source(self):
		self.output.info("")
		self.output.info("---------- source ----------")
		self.output.info("")

		zip_name = "%s.zip" % self.FOLDER_NAME if sys.platform == "win32" else "%s.tar.gz" % self.FOLDER_NAME
		url = "http://sourceforge.net/projects/boost/files/boost/%s/%s/download" % (self.version, zip_name)
		self.output.info("Downloading %s..." % url)
		tools.download(url, zip_name, retry=3, retry_wait=10)
		tools.unzip(zip_name, ".")
		os.unlink(zip_name)

	def build(self):
		self.output.info("")
		self.output.info("---------- build ----------")
		self.output.info("")
		self.output.info("os        : " + str(self.settings.os))
		self.output.info("arch      : " + str(self.settings.arch))
		self.output.info("build_type: " + str(self.settings.build_type))

		if self.settings.compiler == "Visual Studio":
			self.output.info("runtime   : " + str(self.settings.compiler.runtime))

		if self.options.header_only:
			return

		if self.settings.os == "Windows":
			command = "bootstrap"
		else:
			command = "./bootstrap.sh"

		try:
			with tools.chdir(self.FOLDER_NAME):
				self.run(command)
		except:
			with tools.chdir(self.FOLDER_NAME):
				if self.settings.os == "Windows":
					self.run("type bootstrap.log")
				else:
					self.run("cat bootstrap.log")
			raise

		flags = []

		if self.settings.compiler == "Visual Studio":
			flags.append("toolset=msvc-%s.0" % self.settings.compiler.version)
		elif self.settings.arch == "armv7hf":
			if not os.environ.has_key("CXX"):
				raise Exception("failed to extract compiler from environment variable \"CXX\" (variable is not set)")

			result = re.search("(.*g\+\+)$", os.environ.get("CXX"), re.M|re.I)

			if not result:
				raise Exception("Failed to extract compiler from environment variable \"CXX\". Variable value \"" + os.environ.get("CXX") + "\" does not end with \"g++\", e.g. \"arm-linux-gnueabihf-g++\".")

			# set the compiler directly by editing the project-config.jam instead of specifying it in a user-config.jam file in combination with the BOOST_BUILD_PATH environment variable since a possibly existing user-config.jam in the HOME directory would take precedence over BOOST_BUILD_PATH
			with tools.chdir(self.FOLDER_NAME):
				with open("project-config.jam", "r+") as f:
					content = f.read()
					content = re.sub(r"([ \t]*)using gcc[ \t]*;*", r"\1using gcc : arm : " + result.group(1) + " ;", content)
					f.seek(0)
					f.write(content)
					f.truncate()

			flags.append("toolset=gcc-arm")

		flags.append("link=%s" % ("static" if not self.options.shared else "shared"))

		if self.settings.compiler == "Visual Studio" and self.settings.compiler.runtime:
			flags.append("runtime-link=%s" % ("static" if "MT" in str(self.settings.compiler.runtime) else "shared"))

		flags.append("variant=%s" % str(self.settings.build_type).lower())

		if self.settings.arch != "armv7hf":
			flags.append("address-model=%s" % ("32" if self.settings.arch == "x86" else "64"))

		b2_flags = " ".join(flags)

		if self.settings.os == "Windows":
			command = "b2"
		else:
			command = "./b2"

		command += " " + b2_flags + " -j " + str(tools.cpu_count()) + " -s NO_BZIP2=1 --abbreviate-paths --without-python"

		if self.settings.arch == "armv7hf":
			command += " --without-context --without-coroutine"

		self.output.warn(command)

		with tools.chdir(self.FOLDER_NAME):
			self.run(command)

	def package(self):
		self.output.info("")
		self.output.info("---------- package ----------")
		self.output.info("")

		self.copy(pattern="*", dst="include/boost", src="%s/boost" % self.FOLDER_NAME)
		self.copy(pattern="*.a", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.so", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.so.*", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.dylib*", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.lib", dst="lib", src="%s/stage/lib" % self.FOLDER_NAME)
		self.copy(pattern="*.dll", dst="bin", src="%s/stage/lib" % self.FOLDER_NAME)

	def package_info(self):
		self.output.info("")
		self.output.info("---------- package_info ----------")
		self.output.info("")

		if not self.options.header_only:
			if self.options.shared:
				self.cpp_info.defines.append("BOOST_ALL_DYN_LINK")
			else:
				self.cpp_info.defines.append("BOOST_USE_STATIC_LIBS")

			libs = ("atomic chrono container context coroutine date_time exception filesystem "
					"graph iostreams locale log_setup log math_c99 math_c99f math_c99l math_tr1 "
					"math_tr1f math_tr1l prg_exec_monitor program_options random regex serialization "
					"signals system test_exec_monitor thread timer unit_test_framework wave "
					"wserialization").split()

		if self.settings.os != "Windows":
				if self.settings.arch == "armv7hf":
					libs.remove("context")
					libs.remove("coroutine")

				self.cpp_info.libs.extend(["boost_%s" % lib for lib in libs])
		elif self.settings.os == "Windows":
			win_libs = []
			# http://www.boost.org/doc/libs/1_55_0/more/getting_started/windows.html
			visual_version = int(str(self.settings.compiler.version)) * 10
			runtime = "mt"# str(self.settings.compiler.runtime).lower()

			abi_tags = []
			if self.settings.compiler.runtime in ("MTd", "MT"):
				abi_tags.append("s")

			if self.settings.build_type == "Debug":
				abi_tags.append("gd")

			abi_tags = ("-%s" % "".join(abi_tags)) if abi_tags else ""

			version = "_".join(self.version.split(".")[0:2])
			suffix = "vc%d-%s%s-%s" % (visual_version, runtime, abi_tags, version)
			prefix = "lib" if not self.options.shared else ""

			win_libs.extend(["%sboost_%s-%s" % (prefix, lib, suffix) for lib in libs if lib not in ["exception", "test_exec_monitor"]])
			win_libs.extend(["libboost_exception-%s" % suffix, "libboost_test_exec_monitor-%s" % suffix])

			#self.output.warn("EXPORTED BOOST LIBRARIES: %s" % win_libs)
			self.cpp_info.libs.extend(win_libs)
			self.cpp_info.defines.extend(["BOOST_ALL_NO_LIB"]) # DISABLES AUTO LINKING! NO SMART AND MAGIC DECISIONS THANKS!
<|endoftext|>


<commit_before>"""
byceps.blueprints.admin.ticketing.checkin.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2021 Jochen Kupperschmidt
:License: Revised BSD (see `LICENSE` file for details)
"""

from datetime import date

from flask import abort, g, request

from .....services.party import service as party_service
from .....services.ticketing import (
    exceptions as ticket_exceptions,
    ticket_service,
    ticket_user_checkin_service,
)
from .....signals import ticketing as ticketing_signals
from .....util.framework.blueprint import create_blueprint
from .....util.framework.flash import flash_error, flash_notice, flash_success
from .....util.framework.templating import templated
from .....util.views import permission_required, respond_no_content

from ...ticketing.authorization import TicketingPermission
from ...user import service as user_blueprint_service

blueprint = create_blueprint('ticketing_checkin', __name__)

MINIMUM_AGE_IN_YEARS = 18

@blueprint.route('/for_party/<party_id>')
@permission_required(TicketingPermission.checkin)
@templated
def index(party_id):
    """Provide form to find tickets, then check them in."""
    party = _get_party_or_404(party_id)

    search_term = request.args.get('search_term', default='').strip()

    limit = 10

    if search_term:
        latest_dob_for_checkin = _get_latest_date_of_birth_for_checkin()
        tickets = _search_tickets(party.id, search_term, limit)
        users = _search_users(search_term, limit)

        tickets += list(_get_tickets_for_users(party.id, users))
    else:
        latest_dob_for_checkin = None
        tickets = None
        users = None

    return {
        'party': party,
        'latest_dob_for_checkin': latest_dob_for_checkin,
        'search_term': search_term,
        'tickets': tickets,
        'users': users,
    }

def _get_latest_date_of_birth_for_checkin():
    today = date.today()
    return today.replace(year=today.year - MINIMUM_AGE_IN_YEARS)

def _search_tickets(party_id, search_term, limit):
    page = 1
    per_page = limit

    tickets_pagination = ticket_service.get_tickets_with_details_for_party_paginated(
        party_id, page, per_page, search_term=search_term
    )

    return tickets_pagination.items

def _search_users(search_term, limit):
    page = 1
    per_page = limit

    users_pagination = user_blueprint_service.get_users_paginated(
        page, per_page, search_term=search_term
    )

    # Exclude deleted users.
    users_pagination.items = [
        user for user in users_pagination.items if not user.deleted
    ]

    return users_pagination.items

def _get_tickets_for_users(party_id, users):
    for user in users:
        yield from ticket_service.find_tickets_related_to_user_for_party(
            user.id, party_id
        )

@blueprint.route(
    '/for_party/<party_id>/tickets/<uuid:ticket_id>/check_in_user',
    methods=['POST'],
)
@permission_required(TicketingPermission.checkin)
@respond_no_content
def check_in_user(party_id, ticket_id):
    """Check the user in."""
    party = _get_party_or_404(party_id)
    ticket = _get_ticket_or_404(ticket_id)

    initiator_id = g.current_user.id

    try:
        event = ticket_user_checkin_service.check_in_user(
            party_id, ticket.id, initiator_id
        )
    except ticket_exceptions.UserAccountDeleted:
        flash_error(
            'Das dem Ticket zugewiesene Benutzerkonto ist gelöscht worden. '
            'Der Check-In ist nicht erlaubt.'
        )
        return
    except ticket_exceptions.UserAccountSuspended:
        flash_error(
            'Das dem Ticket zugewiesene Benutzerkonto ist gesperrt. '
            'Der Check-In ist nicht erlaubt.'
        )
        return

    ticketing_signals.ticket_checked_in.send(None, event=event)

    flash_success(f"Benutzer '{ticket.used_by.screen_name}' wurde eingecheckt.")

    occupies_seat = ticket.occupied_seat_id is not None
    if not occupies_seat:
        flash_notice('Das Ticket belegt noch keinen Sitzplatz.', icon='warning')

@blueprint.route(
    '/tickets/<uuid:ticket_id>/revert_user_check_in', methods=['POST']
)
@permission_required(TicketingPermission.checkin)
@respond_no_content
def revert_user_check_in(ticket_id):
    """Revert the user check-in state."""
    ticket = _get_ticket_or_404(ticket_id)

    initiator_id = g.current_user.id

    ticket_user_checkin_service.revert_user_check_in(ticket.id, initiator_id)

    flash_success('Der Check-In wurde rückgängig gemacht.')

def _get_party_or_404(party_id):
    party = party_service.find_party(party_id)

    if party is None:
        abort(404)

    return party

def _get_ticket_or_404(ticket_id):
    ticket = ticket_service.find_ticket(ticket_id)

    if ticket is None:
        abort(404)

    return ticket
<commit_msg>Include, link ticket code in check-in flash messages<commit_after>"""
byceps.blueprints.admin.ticketing.checkin.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2021 Jochen Kupperschmidt
:License: Revised BSD (see `LICENSE` file for details)
"""

from datetime import date

from flask import abort, g, request, url_for

from .....services.party import service as party_service
from .....services.ticketing import (
    exceptions as ticket_exceptions,
    ticket_service,
    ticket_user_checkin_service,
)
from .....signals import ticketing as ticketing_signals
from .....util.framework.blueprint import create_blueprint
from .....util.framework.flash import flash_error, flash_notice, flash_success
from .....util.framework.templating import templated
from .....util.views import permission_required, respond_no_content

from ...ticketing.authorization import TicketingPermission
from ...user import service as user_blueprint_service

blueprint = create_blueprint('ticketing_checkin', __name__)

MINIMUM_AGE_IN_YEARS = 18

@blueprint.route('/for_party/<party_id>')
@permission_required(TicketingPermission.checkin)
@templated
def index(party_id):
    """Provide form to find tickets, then check them in."""
    party = _get_party_or_404(party_id)

    search_term = request.args.get('search_term', default='').strip()

    limit = 10

    if search_term:
        latest_dob_for_checkin = _get_latest_date_of_birth_for_checkin()
        tickets = _search_tickets(party.id, search_term, limit)
        users = _search_users(search_term, limit)

        tickets += list(_get_tickets_for_users(party.id, users))
    else:
        latest_dob_for_checkin = None
        tickets = None
        users = None

    return {
        'party': party,
        'latest_dob_for_checkin': latest_dob_for_checkin,
        'search_term': search_term,
        'tickets': tickets,
        'users': users,
    }

def _get_latest_date_of_birth_for_checkin():
    today = date.today()
    return today.replace(year=today.year - MINIMUM_AGE_IN_YEARS)

def _search_tickets(party_id, search_term, limit):
    page = 1
    per_page = limit

    tickets_pagination = ticket_service.get_tickets_with_details_for_party_paginated(
        party_id, page, per_page, search_term=search_term
    )

    return tickets_pagination.items

def _search_users(search_term, limit):
    page = 1
    per_page = limit

    users_pagination = user_blueprint_service.get_users_paginated(
        page, per_page, search_term=search_term
    )

    # Exclude deleted users.
    users_pagination.items = [
        user for user in users_pagination.items if not user.deleted
    ]

    return users_pagination.items

def _get_tickets_for_users(party_id, users):
    for user in users:
        yield from ticket_service.find_tickets_related_to_user_for_party(
            user.id, party_id
        )

@blueprint.route(
    '/for_party/<party_id>/tickets/<uuid:ticket_id>/check_in_user',
    methods=['POST'],
)
@permission_required(TicketingPermission.checkin)
@respond_no_content
def check_in_user(party_id, ticket_id):
    """Check the user in."""
    party = _get_party_or_404(party_id)
    ticket = _get_ticket_or_404(ticket_id)

    initiator_id = g.current_user.id

    try:
        event = ticket_user_checkin_service.check_in_user(
            party_id, ticket.id, initiator_id
        )
    except ticket_exceptions.UserAccountDeleted:
        flash_error(
            'Das dem Ticket zugewiesene Benutzerkonto ist gelöscht worden. '
            'Der Check-In ist nicht erlaubt.'
        )
        return
    except ticket_exceptions.UserAccountSuspended:
        flash_error(
            'Das dem Ticket zugewiesene Benutzerkonto ist gesperrt. '
            'Der Check-In ist nicht erlaubt.'
        )
        return

    ticketing_signals.ticket_checked_in.send(None, event=event)

    ticket_url = url_for('ticketing_admin.view_ticket', ticket_id=ticket.id)

    flash_success(
        f'Benutzer {ticket.used_by.screen_name} wurde '
        f'mit Ticket {ticket.code} eingecheckt.',
        text_is_safe=True,
    )

    occupies_seat = ticket.occupied_seat_id is not None
    if not occupies_seat:
        flash_notice(
            f'Ticket {ticket.code} '
            'belegt noch keinen Sitzplatz.',
            icon='warning',
            text_is_safe=True,
        )

@blueprint.route(
    '/tickets/<uuid:ticket_id>/revert_user_check_in', methods=['POST']
)
@permission_required(TicketingPermission.checkin)
@respond_no_content
def revert_user_check_in(ticket_id):
    """Revert the user check-in state."""
    ticket = _get_ticket_or_404(ticket_id)

    initiator_id = g.current_user.id

    ticket_user_checkin_service.revert_user_check_in(ticket.id, initiator_id)

    flash_success('Der Check-In wurde rückgängig gemacht.')

def _get_party_or_404(party_id):
    party = party_service.find_party(party_id)

    if party is None:
        abort(404)

    return party

def _get_ticket_or_404(ticket_id):
    ticket = ticket_service.find_ticket(ticket_id)

    if ticket is None:
        abort(404)

    return ticket
<|endoftext|>


<commit_before>"""
Views for the crowdfund application
"""

from django.contrib import messages
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, redirect
from django.views.generic.detail import DetailView

from decimal import Decimal
import logging
import stripe

from muckrock.crowdfund.forms import CrowdfundRequestPaymentForm
from muckrock.crowdfund.models import CrowdfundRequest
from muckrock.settings import STRIPE_SECRET_KEY, STRIPE_PUB_KEY

logger = logging.getLogger(__name__)
stripe.api_key = STRIPE_SECRET_KEY

def process_payment(request, amount, token, crowdfund):
    """Helper function to create a Stripe charge and handle errors"""
    # double -> int conversion
    # http://stackoverflow.com/a/13528445/4256689
    amount = int(amount) * 100
    logging.debug(amount)
    try:
        stripe.Charge.create(
            amount=amount,
            source=token,
            currency='usd',
            description='Contribute to Crowdfunding: %s %s' %
                (crowdfund, crowdfund.pk),
        )
        return True
    except (
        stripe.InvalidRequestError,
        stripe.CardError,
        stripe.APIConnectionError,
        stripe.AuthenticationError
    ) as exception:
        logging.error('Processing a Stripe charge: %s', exception)
        messages.error(request, ('We encountered an error processing your card.'
                                ' Your card has not been charged.'))
        return False

class CrowdfundRequestDetail(DetailView):
    """
    Presents details about a crowdfunding campaign,
    as well as providing a private endpoint for contributions
    """
    model = CrowdfundRequest
    template_name = 'details/crowdfund_request_detail.html'

    def get_context_data(self, **kwargs):
        """Adds Stripe public key to context"""
        # pylint: disable=no-self-use
        # pylint: disable=unused-variable
        context = super(CrowdfundRequestDetail, self).get_context_data(**kwargs)
        context['stripe_pk'] = STRIPE_PUB_KEY
        return context

    def post(self, request, **kwargs):
        """
        First we validate the payment form, so we don't charge someone's card by accident.
        Next, we charge their card. Finally, use the validated payment form to create and
        return a CrowdfundRequestPayment object.
        """
        # pylint: disable=unused-argument
        # pylint: disable=no-self-use
        amount = request.POST.get('amount')
        show = request.POST.get('show')
        crowdfund = request.POST.get('crowdfund')
        email = request.POST.get('email')
        token = request.POST.get('token')
        user = None
        if request.user.is_authenticated():
            user = request.user
        logging.debug(user)
        crowdfund_object = get_object_or_404(CrowdfundRequest, pk=crowdfund)

        log_msg = '\n\
                   --- Crowdfunding Payment ---\n\
                   Amount:      %s\n\
                   Email:       %s\n\
                   Token:       %s\n\
                   Show:        %s\n\
                   Crowdfund:   %s\n\
                   User:        %s\n\
                   \n'

        logging.info(log_msg, amount, email, token, show, crowdfund, user)

        amount = Decimal(float(amount)/100)
        # check if the amount is greater than the amount required
        # if it is, only charge the amount required
        if amount > crowdfund_object.amount_remaining():
            amount = crowdfund_object.amount_remaining()

        payment_data = {'amount': amount, 'show': show, 'crowdfund': crowdfund}
        payment_form = CrowdfundRequestPaymentForm(payment_data)
        payment_object = None
        if payment_form.is_valid() and email and token:
            if process_payment(request, amount, token, crowdfund_object):
                payment_object = payment_form.save(commit=False)
                payment_object.user = user
                payment_object.save()
                logging.info(payment_object)
                crowdfund_object.update_payment_received()
                # if AJAX, return HTTP 200 OK
                # else, add a message to the session
                if request.is_ajax():
                    return HttpResponse(200)
                else:
                    messages.success(request, 'Thank you for your contribution!')
                    return redirect(crowdfund_object.foia)
        # if AJAX, return HTTP 400 ERROR
        # else, add a message to the session
        if request.is_ajax():
            return HttpResponse(400)
        else:
            messages.error(
                request,
                ('There was an error making your contribution. '
                'Your card has not been charged.')
            )
            return redirect(crowdfund_object.foia)
<commit_msg>refactors a crowdfund detail view out of the crowdfund request detail view<commit_after>"""
Views for the crowdfund application
"""

from django.contrib import messages
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, redirect
from django.views.generic.detail import DetailView

from decimal import Decimal
import logging
import stripe

from muckrock.crowdfund.forms import CrowdfundRequestPaymentForm
from muckrock.crowdfund.models import CrowdfundRequest
from muckrock.settings import STRIPE_SECRET_KEY, STRIPE_PUB_KEY

logger = logging.getLogger(__name__)
stripe.api_key = STRIPE_SECRET_KEY

def process_payment(request, amount, token, crowdfund):
    """Helper function to create a Stripe charge and handle errors"""
    # double -> int conversion
    # http://stackoverflow.com/a/13528445/4256689
    amount = int(amount) * 100
    logging.debug(amount)
    try:
        stripe.Charge.create(
            amount=amount,
            source=token,
            currency='usd',
            description='Contribute to Crowdfunding: %s %s' %
                (crowdfund, crowdfund.pk),
        )
        return True
    except (
        stripe.InvalidRequestError,
        stripe.CardError,
        stripe.APIConnectionError,
        stripe.AuthenticationError
    ) as exception:
        logging.error('Processing a Stripe charge: %s', exception)
        messages.error(request, ('We encountered an error processing your card.'
                                ' Your card has not been charged.'))
        return False

class CrowdfundDetailView(DetailView):
    """
    Presents details about a crowdfunding campaign,
    as well as providing a private endpoint for contributions.
    """
    def get_context_data(self, **kwargs):
        """Adds Stripe public key to context"""
        context = super(CrowdfundDetailView, self).get_context_data(**kwargs)
        context['stripe_pk'] = STRIPE_PUB_KEY
        return context

class CrowdfundRequestDetail(CrowdfundDetailView):
    """
    Specificies a CrowdfundDetailView to use the CrowdfundRequest method.
    """
    model = CrowdfundRequest
    template_name = 'details/crowdfund_request_detail.html'

    def post(self, request, **kwargs):
        """
        First we validate the payment form, so we don't charge someone's card by accident.
        Next, we charge their card. Finally, use the validated payment form to create and
        return a CrowdfundRequestPayment object.
        """
        # pylint: disable=unused-argument
        # pylint: disable=no-self-use
        amount = request.POST.get('amount')
        show = request.POST.get('show')
        crowdfund = request.POST.get('crowdfund')
        email = request.POST.get('email')
        token = request.POST.get('token')
        user = None
        if request.user.is_authenticated():
            user = request.user
        logging.debug(user)
        crowdfund_object = get_object_or_404(CrowdfundRequest, pk=crowdfund)

        log_msg = '\n\
                   --- Crowdfunding Payment ---\n\
                   Amount:      %s\n\
                   Email:       %s\n\
                   Token:       %s\n\
                   Show:        %s\n\
                   Crowdfund:   %s\n\
                   User:        %s\n\
                   \n'

        logging.info(log_msg, amount, email, token, show, crowdfund, user)

        amount = Decimal(float(amount)/100)
        # check if the amount is greater than the amount required
        # if it is, only charge the amount required
        if amount > crowdfund_object.amount_remaining():
            amount = crowdfund_object.amount_remaining()

        payment_data = {'amount': amount, 'show': show, 'crowdfund': crowdfund}
        payment_form = CrowdfundRequestPaymentForm(payment_data)
        payment_object = None
        if payment_form.is_valid() and email and token:
            if process_payment(request, amount, token, crowdfund_object):
                payment_object = payment_form.save(commit=False)
                payment_object.user = user
                payment_object.save()
                logging.info(payment_object)
                crowdfund_object.update_payment_received()
                # if AJAX, return HTTP 200 OK
                # else, add a message to the session
                if request.is_ajax():
                    return HttpResponse(200)
                else:
                    messages.success(request, 'Thank you for your contribution!')
                    return redirect(crowdfund_object.foia)
        # if AJAX, return HTTP 400 ERROR
        # else, add a message to the session
        if request.is_ajax():
            return HttpResponse(400)
        else:
            messages.error(
                request,
                ('There was an error making your contribution. '
                'Your card has not been charged.')
            )
            return redirect(crowdfund_object.foia)
<|endoftext|>


<commit_before>import optparse
import sys
import requests
import re
from bs4 import BeautifulSoup

major_version = 0
intermediary_version = 0
minor_version = 0
folder_plugins = None
folder_themes = None

# Detect the version of a SPIP install
# Version is in the header (for almost all versions)

def detect_version(header_composed_by):
    global major_version
    global intermediary_version
    global minor_version

    regex_version_spip = re.search(
        r"SPIP (\d+).(\d+).(\d+)", header_composed_by)
    try:
        major_version = regex_version_spip.group(1)
        intermediary_version = regex_version_spip.group(2)
        minor_version = regex_version_spip.group(3)

        print "[!] Version is : " + str(major_version) + "." + str(intermediary_version) + "." + str(minor_version)
    except:
        display_message("[-] Unable to find the version")
        pass

# Detect the plugins/themes folder of a SPIP install
# Moreover, if there's directory listing enabled, it recovers the plugins/themes
# And it does not do bruteforce attack on the retrieved elements.

def detect_folder(url, isForPlugins):
    global folder_themes
    global folder_plugins
    global opts

    plugins_folders = ['plugins/', 'plugins-dist/']
    themes_folders = ['themes/', 'theme/', 'Themes/', 'Theme/']

    folders = []

    if (isForPlugins):
        folders = plugins_folders
    else:
        folders = themes_folders

    for folder in folders:
        url_to_visit = url + folder
        req = requests.get(url_to_visit, timeout=10)

        # code for both status code 200/403
        if (req.status_code == 200 or req.status_code == 403):
            if (isForPlugins):
                folder_plugins = folder
                print "[!] Plugin folder is : " + folder_plugins
                if (req.status_code == 200):
                    opts.bruteforce_plugins_file = None
            else:
                folder_themes = folder
                print "[!] Theme folder is : " + folder_themes
                if (req.status_code == 200):
                    opts.bruteforce_themes_file = None

        # code only for 200 (directory listing)
        if (req.status_code == 200):
            url = url + folder # set up the url
            iterate_directory_listing(url, req.content)
            return True

        if (req.status_code == 403):
            display_message("[-] Access forbidden on folder.")
            return True

    return False

# Function to iterate on results if there's a directory listing
# will then (try to) detect the version of the plugin/theme

def iterate_directory_listing(url, content):
    print "[!] Directory listing on folder !"
    soup = BeautifulSoup(content)
    links_to_plugins = soup('a')
    for link in links_to_plugins:
        # grabbing the folder of the plugin
        try:
            regex_plugin = re.search(r"href=\"(\w+/)\">\s?(\w+)/<", str(link))
            folder_plugin = regex_plugin.group(1)
            name_plugin = regex_plugin.group(2)
            detect_version_of_plugin_or_theme_by_folder_name(url, folder_plugin)
        except:
            pass

# Detect the version of either a plugin and theme.
# Structure is the same, folder contains either plugin.xml or paquet.xml

def detect_version_of_plugin_or_theme_by_folder_name(url, folder):
    url_folder = url + folder + "plugin.xml"
    # HTTP GET to get the version of the plugin
    req_plugin_xml = requests.get(url_folder, timeout=10)
    display_message("[-] Trying : " + url_folder)
    if (req_plugin_xml.status_code == 200):
        regex_version_plugin = re.search(
            r"<version>\s*?(\d+(.\d+)?(.\d+)?)\s*?</version>", req_plugin_xml.content, re.S)
        print "[!] Plugin " + folder[:-1] + " detected. Version : " + str(regex_version_plugin.group(1))
        display_message("URL : " + url_folder)
    else:
        url_folder = url + folder + "paquet.xml"
        # HTTP GET to get the version of the plugin
        req_plugin_xml = requests.get(url_folder, timeout=10)
        display_message("[-] Trying : " + url_folder)
        if (req_plugin_xml.status_code == 200):
            regex_version_plugin = re.search(
                r"version=\"\s*?(\d+(.\d+)?(.\d+)?)\s*?\"", req_plugin_xml.content, re.S)
            print "[!] Plugin " + folder[:-1] + " detected. Version : " + str(regex_version_plugin.group(1))
            display_message("URL : " + url_folder)
        else:
            pass

# Remove new line character and replace it with another one if specified

def remove_new_line_from_name(name, char=''):
    return name[:-1] + char

# Detect vulnerabilities of the SPIP website

def detect_vulnerabilities():
    global major_version
    global intermediary_version
    global minor_version

    vulns = []
    with open('spip_vulns.db') as f:
        vulns = f.readlines()

    # removing new line
    vulns = [remove_new_line_from_name(vuln) for vuln in vulns]

    # parsing the db to check if there's any vuln
    for vuln in vulns:
        vals = vuln.split(';;')
        versions_vuln = vals[0]
        description_vuln = vals[1]
        url_vuln = vals[2]
        version_vuln = versions_vuln.split('/')
        for version in version_vuln:
            tmp = version.split('.')
            i = 0
            while i < len(tmp):
                if (i == 0 and tmp[i] != major_version):
                    break
                if (i == 1 and tmp[i] != intermediary_version):
                    break

                if (i == 1 and tmp[i] == intermediary_version and (i + 1) > len(tmp)):
                    print "[!] Potential Vulnerability : (versions : " + versions_vuln + "), " + description_vuln + ", details : " + url_vuln
                    break

                if ((i == 2) and (int(tmp[i]) >= int(minor_version))):
                    print "[!] Potential Vulnerability : (versions : " + versions_vuln + "), " + description_vuln + ", details : " + url_vuln
                i =               i = i+ 1

# This function allows you to do brute force to search for folders
# This function is used to bruteforce Plugin/Theme names

def bruteforce_folder(url, filename, isForPlugins):
    # uri for the plugins folder
    global folder_plugins
    global folder_themes

    # If we haven't been able to detect the plugins folder, we exit
    if (isForPlugins and folder_plugins is None):
        return

    # If we haven't been able to detect the themes folder, we exit
    if (isForPlugins is False and folder_themes is None):
        return

    if (isForPlugins is False and folder_themes is not None):
        url = url + folder_themes

    if (isForPlugins and folder_plugins is not None):
        url = url + folder_plugins

    folders = []
    with open(filename) as f:
        folders = f.readlines()

    # removing new line
    folders = [remove_new_line_from_name(name, '/') for name in folders]
    for folder in folders:
        detect_version_of_plugin_or_theme_by_folder_name(url, folder)

# Display function to only print message
# if verbose mode is ON

def display_message(m):
    global opts

    if (opts.verbose):
        print m

# option parser
parser = optparse.OptionParser()
parser.add_option('--website', help='Website to pentest', dest='website')
parser.add_option('--path', help='Path for webapp (default : "/")', dest='path', default='/')
parser.add_option('--plugins', help='Detect plugins installed', dest='detect_plugins', default=False, action='store_true')
parser.add_option('--themes', help='Detect themes installed', dest='detect_themes', default=False, action='store_true')
parser.add_option('--version', help='Detect version', dest='detect_version', default=False, action='store_true')
parser.add_option('--vulns', help='Detect possible vulns', dest='detect_vulns', default=False, action='store_true')
parser.add_option('--bruteforce_plugins_file', help='Bruteforce plugin file (eg. plugins_name.db)', dest='bruteforce_plugins_file', default=None)
parser.add_option('--bruteforce_themes_file', help='Bruteforce theme file (eg. themes_name.db)', dest='bruteforce_themes_file', default=None)
parser.add_option('--verbose', help='Verbose mode', dest='verbose', default=False, action='store_true')

if (len(sys.argv) <= 2):
    parser.print_help()
else:
    (opts, args) = parser.parse_args()

    url = opts.website + opts.path
    display_message("Application is located here : " + url)

    if (opts.detect_version or opts.detect_vulns):
        req = requests.get(url, timeout=10)
        detect_version(req.headers['composed-by'])

    if (opts.detect_plugins or opts.bruteforce_plugins_file is not None):
        if not detect_folder(url, True):
            display_message("[-] We haven't been able to locate the plugins folder")

    if (opts.detect_themes or opts.bruteforce_themes_file is not None):
        if not detect_folder(url, False):
            display_message("[-] We haven't been able to locate the themes folder")

    # detect plugin will do brute force attack if it finds a HTTP 403
    # (Restricted)
    # opts.detect_plugins is False and
    if (opts.bruteforce_plugins_file is not None and folder_plugins is not None):
        bruteforce_folder(url, opts.bruteforce_plugins_file, True)

    # brute force themes folder if 403 also
    if (opts.bruteforce_themes_file is not None and folder_themes is not None):
        bruteforce_folder(url, opts.bruteforce_themes_file, False)

    if (opts.detect_vulns):
        detect_vulnerabilities()
<commit_msg>Fixed some printing messages in non-verbose mode<commit_after>import optparse
import sys
import requests
import re
from bs4 import BeautifulSoup

major_version = 0
intermediary_version = 0
minor_version = 0
folder_plugins = None
folder_themes = None

# Detect the version of a SPIP install
# Version is in the header (for almost all versions)

def detect_version(header_composed_by):
    global major_version
    global intermediary_version
    global minor_version

    regex_version_spip = re.search(
        r"SPIP (\d+).(\d+).(\d+)", header_composed_by)
    try:
        major_version = regex_version_spip.group(1)
        intermediary_version = regex_version_spip.group(2)
        minor_version = regex_version_spip.group(3)

        print "[!] Version is : " + str(major_version) + "." + str(intermediary_version) + "." + str(minor_version)
    except:
        display_message("[-] Unable to find the version")
        pass

# Detect the plugins/themes folder of a SPIP install
# Moreover, if there's directory listing enabled, it recovers the plugins/themes
# And it does not do bruteforce attack on the retrieved elements.

def detect_folder(url, isForPlugins):
    global folder_themes
    global folder_plugins
    global opts

    plugins_folders = ['plugins/', 'plugins-dist/']
    themes_folders = ['themes/', 'theme/', 'Themes/', 'Theme/']

    folders = []

    if (isForPlugins):
        folders = plugins_folders
    else:
        folders = themes_folders

    for folder in folders:
        url_to_visit = url + folder
        req = requests.get(url_to_visit, timeout=10)

        # code for both status code 200/403
        if (req.status_code == 200 or req.status_code == 403):
            if (isForPlugins):
                folder_plugins = folder
                print "[!] Plugin folder is : " + folder_plugins
                if (req.status_code == 200):
                    opts.bruteforce_plugins_file = None
            else:
                folder_themes = folder
                print "[!] Theme folder is : " + folder_themes
                if (req.status_code == 200):
                    opts.bruteforce_themes_file = None

        # code only for 200 (directory listing)
        if (req.status_code == 200):
            url = url + folder # set up the url
            iterate_directory_listing(url, req.content)
            return True

        if (req.status_code == 403):
            print "[-] Access forbidden on folder."
            return True

    return False

# Function to iterate on results if there's a directory listing
# will then (try to) detect the version of the plugin/theme

def iterate_directory_listing(url, content):
    print "[!] Directory listing on folder !"
    soup = BeautifulSoup(content)
    links_to_plugins = soup('a')
    for link in links_to_plugins:
        # grabbing the folder of the plugin
        try:
            regex_plugin = re.search(r"href=\"(\w+/)\">\s?(\w+)/<", str(link))
            folder_plugin = regex_plugin.group(1)
            name_plugin = regex_plugin.group(2)
            detect_version_of_plugin_or_theme_by_folder_name(url, folder_plugin)
        except:
            pass

# Detect the version of either a plugin and theme.
# Structure is the same, folder contains either plugin.xml or paquet.xml

def detect_version_of_plugin_or_theme_by_folder_name(url, folder):
    url_folder = url + folder + "plugin.xml"
    # HTTP GET to get the version of the plugin
    req_plugin_xml = requests.get(url_folder, timeout=10)
    display_message("[-] Trying : " + url_folder)
    if (req_plugin_xml.status_code == 200):
        regex_version_plugin = re.search(
            r"<version>\s*?(\d+(.\d+)?(.\d+)?)\s*?</version>", req_plugin_xml.content, re.S)
        print "[!] Plugin " + folder[:-1] + " detected. Version : " + str(regex_version_plugin.group(1))
        display_message("URL : " + url_folder)
    else:
        url_folder = url + folder + "paquet.xml"
        # HTTP GET to get the version of the plugin
        req_plugin_xml = requests.get(url_folder, timeout=10)
        display_message("[-] Trying : " + url_folder)
        if (req_plugin_xml.status_code == 200):
            regex_version_plugin = re.search(
                r"version=\"\s*?(\d+(.\d+)?(.\d+)?)\s*?\"", req_plugin_xml.content, re.S)
            print "[!] Plugin " + folder[:-1] + " detected. Version : " + str(regex_version_plugin.group(1))
            display_message("URL : " + url_folder)
        else:
            pass

# Remove new line character and replace it with another one if specified

def remove_new_line_from_name(name, char=''):
    return name[:-1] + char

# Detect vulnerabilities of the SPIP website

def detect_vulnerabilities():
    global major_version
    global intermediary_version
    global minor_version

    vulns = []
    with open('spip_vulns.db') as f:
        vulns = f.readlines()

    # removing new line
    vulns = [remove_new_line_from_name(vuln) for vuln in vulns]

    # parsing the db to check if there's any vuln
    for vuln in vulns:
        vals = vuln.split(';;')
        versions_vuln = vals[0]
        description_vuln = vals[1]
        url_vuln = vals[2]
        version_vuln = versions_vuln.split('/')
        for version in version_vuln:
            tmp = version.split('.')
            i = 0
            while i < len(tmp):
                if (i == 0 and tmp[i] != major_version):
                    break
                if (i == 1 and tmp[i] != intermediary_version):
                    break

                if (i == 1 and tmp[i] == intermediary_version and (i + 1) > len(tmp)):
                    print "[!] Potential Vulnerability : (versions : " + versions_vuln + "), " + description_vuln + ", details : " + url_vuln
                    break

                if ((i == 2) and (int(tmp[i]) >= int(minor_version))):
                    print "[!] Potential Vulnerability : (versions : " + versions_vuln + "), " + description_vuln + ", details : " + url_vuln
                i =               i = i+ 1

# This function allows you to do brute force to search for folders
# This function is used to bruteforce Plugin/Theme names

def bruteforce_folder(url, filename, isForPlugins):
    # uri for the plugins folder
    global folder_plugins
    global folder_themes

    # If we haven't been able to detect the plugins folder, we exit
    if (isForPlugins and folder_plugins is None):
        return

    # If we haven't been able to detect the themes folder, we exit
    if (isForPlugins is False and folder_themes is None):
        return

    if (isForPlugins is False and folder_themes is not None):
        url = url + folder_themes

    if (isForPlugins and folder_plugins is not None):
        url = url + folder_plugins

    folders = []
    with open(filename) as f:
        folders = f.readlines()

    # removing new line
    folders = [remove_new_line_from_name(name, '/') for name in folders]
    for folder in folders:
        detect_version_of_plugin_or_theme_by_folder_name(url, folder)

# Display function to only print message
# if verbose mode is ON

def display_message(m):
    global opts

    if (opts.verbose):
        print m

# option parser
parser = optparse.OptionParser()
parser.add_option('--website', help='Website to pentest', dest='website')
parser.add_option('--path', help='Path for webapp (default : "/")', dest='path', default='/')
parser.add_option('--plugins', help='Detect plugins installed', dest='detect_plugins', default=False, action='store_true')
parser.add_option('--themes', help='Detect themes installed', dest='detect_themes', default=False, action='store_true')
parser.add_option('--version', help='Detect version', dest='detect_version', default=False, action='store_true')
parser.add_option('--vulns', help='Detect possible vulns', dest='detect_vulns', default=False, action='store_true')
parser.add_option('--bruteforce_plugins_file', help='Bruteforce plugin file (eg. plugins_name.db)', dest='bruteforce_plugins_file', default=None)
parser.add_option('--bruteforce_themes_file', help='Bruteforce theme file (eg. themes_name.db)', dest='bruteforce_themes_file', default=None)
parser.add_option('--verbose', help='Verbose mode', dest='verbose', default=False, action='store_true')

if (len(sys.argv) <= 2):
    parser.print_help()
else:
    (opts, args) = parser.parse_args()

    url = opts.website + opts.path
    display_message("Application is located here : " + url)

    if (opts.detect_version or opts.detect_vulns):
        req = requests.get(url, timeout=10)
        detect_version(req.headers['composed-by'])

    if (opts.detect_plugins or opts.bruteforce_plugins_file is not None):
        if not detect_folder(url, True):
            print "[-] We haven't been able to locate the plugins folder"

    if (opts.detect_themes or opts.bruteforce_themes_file is not None):
        if not detect_folder(url, False):
            print "[-] We haven't been able to locate the themes folder"

    # detect plugin will do brute force attack if it finds a HTTP 403
    # (Restricted)
    # opts.detect_plugins is False and
    if (opts.bruteforce_plugins_file is not None and folder_plugins is not None):
        bruteforce_folder(url, opts.bruteforce_plugins_file, True)

    # brute force themes folder if 403 also
    if (opts.bruteforce_themes_file is not None and folder_themes is not None):
        bruteforce_folder(url, opts.bruteforce_themes_file, False)

    if (opts.detect_vulns):
        detect_vulnerabilities()
<|endoftext|>


<commit_before># (c) 2013, Daniel Hokka Zakrisson <daniel@hozac.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import os
import ansible.constants as C
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible import utils
from ansible import errors

class InventoryDirectory(object):
    ''' Host inventory parser for ansible using a directory of inventories. '''

    def __init__(self, filename=C.DEFAULT_HOST_LIST):
        self.names = os.listdir(filename)
        self.names.sort()
        self.directory = filename
        self.parsers = []
        self.hosts = {}
        self.groups = {}

        for i in self.names:

            # Skip files that end with certain extensions or characters
            for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo"):
                if i.endswith(ext):
                    continue
            # Skip hidden files
            if i.startswith('.') and not i.startswith('./'):
                continue
            # These are things inside of an inventory basedir
            if i in ("host_vars", "group_vars", "vars_plugins"):
                continue
            fullpath = os.path.join(self.directory, i)
            if os.path.isdir(fullpath):
                parser = InventoryDirectory(filename=fullpath)
            elif utils.is_executable(fullpath):
                parser = InventoryScript(filename=fullpath)
            else:
                parser = InventoryParser(filename=fullpath)
            self.parsers.append(parser)
            # This takes a lot of code because we can't directly use any of the objects, as they have to blend
            for name, group in parser.groups.iteritems():
                if name not in self.groups:
                    self.groups[name] = group
                else:
                    # group is already there, copy variables
                    # note: depth numbers on duplicates may be bogus
                    for k, v in group.get_variables().iteritems():
                        self.groups[name].set_variable(k, v)
                for host in group.get_hosts():
                    if host.name not in self.hosts:
                        self.hosts[host.name] = host
                    else:
                        # host is already there, copy variables
                        # note: depth numbers on duplicates may be bogus
                        for k, v in host.vars.iteritems():
                            self.hosts[host.name].set_variable(k, v)
                    self.groups[name].add_host(self.hosts[host.name])

            # This needs to be a second loop to ensure all the parent groups exist
            for name, group in parser.groups.iteritems():
                for ancestor in group.get_ancestors():
                    self.groups[ancestor.name].add_child_group(self.groups[name])

    def get_host_variables(self, host):
        """ Gets additional host variables from all inventories """
        vars = {}
        for i in self.parsers:
            vars.update(i.get_host_variables(host))
        return vars

<commit_msg>Fixed extension filtering in InventoryDirectory<commit_after># (c) 2013, Daniel Hokka Zakrisson <daniel@hozac.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

#############################################

import os
import ansible.constants as C
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.inventory.ini import InventoryParser
from ansible.inventory.script import InventoryScript
from ansible import utils
from ansible import errors

class InventoryDirectory(object):
    ''' Host inventory parser for ansible using a directory of inventories. '''

    def __init__(self, filename=C.DEFAULT_HOST_LIST):
        self.names = os.listdir(filename)
        self.names.sort()
        self.directory = filename
        self.parsers = []
        self.hosts = {}
        self.groups = {}

        for i in self.names:

            # Skip files that end with certain extensions or characters
            if any(i.endswith(ext) for ext in ("~", ".orig", ".bak", ".ini", ".retry", ".pyc", ".pyo")):
                continue
            # Skip hidden files
            if i.startswith('.') and not i.startswith('./'):
                continue
            # These are things inside of an inventory basedir
            if i in ("host_vars", "group_vars", "vars_plugins"):
                continue
            fullpath = os.path.join(self.directory, i)
            if os.path.isdir(fullpath):
                parser = InventoryDirectory(filename=fullpath)
            elif utils.is_executable(fullpath):
                parser = InventoryScript(filename=fullpath)
            else:
                parser = InventoryParser(filename=fullpath)
            self.parsers.append(parser)
            # This takes a lot of code because we can't directly use any of the objects, as they have to blend
            for name, group in parser.groups.iteritems():
                if name not in self.groups:
                    self.groups[name] = group
                else:
                    # group is already there, copy variables
                    # note: depth numbers on duplicates may be bogus
                    for k, v in group.get_variables().iteritems():
                        self.groups[name].set_variable(k, v)
                for host in group.get_hosts():
                    if host.name not in self.hosts:
                        self.hosts[host.name] = host
                    else:
                        # host is already there, copy variables
                        # note: depth numbers on duplicates may be bogus
                        for k, v in host.vars.iteritems():
                            self.hosts[host.name].set_variable(k, v)
                    self.groups[name].add_host(self.hosts[host.name])

            # This needs to be a second loop to ensure all the parent groups exist
            for name, group in parser.groups.iteritems():
                for ancestor in group.get_ancestors():
                    self.groups[ancestor.name].add_child_group(self.groups[name])

    def get_host_variables(self, host):
        """ Gets additional host variables from all inventories """
        vars = {}
        for i in self.parsers:
            vars.update(i.get_host_variables(host))
        return vars

<|endoftext|>