<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# django-mysql documentation build configuration file, created by
# sphinx-quickstart on Tue Jul  9 22:26:36 2013.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another
# directory, add these directories to sys.path here. If the directory is
# relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# Get the project root dir, which is the parent dir of this
cwd = os.getcwd()
project_root = os.path.dirname(cwd)

# Insert the project root dir as the first element in the PYTHONPATH.
# This lets us ensure that the source package is imported, and that its
# version is used.
sys.path.insert(0, project_root)

from django.conf import settings
settings.configure()

import django_mysql

# -- General configuration ---------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.viewcode',
    'sphinx.ext.intersphinx',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Django MySQL'
copyright = u'2015, Adam Johnson'

# The version info for the project you're documenting, acts as replacement
# for |version| and |release|, also used in various other places throughout
# the built documents.
#
# The short X.Y version.
version = django_mysql.__version__
# The full version, including alpha/beta/rc tags.
release = django_mysql.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to
# some non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ["django_mysql."]

# If true, keep warnings as "system message" paragraphs in the built
# documents.
# keep_warnings = False

# -- Options for HTML output -------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
try:
    # When building locally, try to use the Read the Docs theme
    import sphinx_rtd_theme
    html_theme = "sphinx_rtd_theme"
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
except ImportError:
    # When actually building on Read the Docs, the theme is automatically
    # replaced
    html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a
# theme further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as
# html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the
# top of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon
# of the docs.  This file should be a Windows icon file (.ico) being
# 16x16 or 32x32 pixels large.
html_favicon = 'favicon.ico'

# Add any paths that contain custom static files (such as style sheets)
# here, relative to this directory. They are copied after the builtin
# static files, so a file named "default.css" will overwrite the builtin
# "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names
# to template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer.
# Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer.
# Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages
# will contain a <link> tag referring to it.  The value of this option
# must be the base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'django-mysqldoc'

# -- Options for LaTeX output ------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
    ('index', 'django-mysql.tex',
     u'Django MySQL Documentation',
     u'Adam Johnson', 'manual'),
]

# The name of an image file (relative to this directory) to place at
# the top of the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings
# are parts, not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'django-mysql',
     u'Django MySQL Documentation',
     [u'Adam Johnson'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ----------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'django-mysql',
     u'Django MySQL Documentation',
     u'Adam Johnson',
     'django-mysql',
     'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

intersphinx_mapping = {
    'django': ('http://docs.djangoproject.com/en/dev/',
               'http://docs.djangoproject.com/en/dev/_objects/'),
}
<commit_msg>Use recommended method of checking if on readthedocs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# django-mysql documentation build configuration file, created by
# sphinx-quickstart on Tue Jul  9 22:26:36 2013.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another
# directory, add these directories to sys.path here. If the directory is
# relative to the documentation root, use os.path.abspath to make it
# absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# Get the project root dir, which is the parent dir of this
cwd = os.getcwd()
project_root = os.path.dirname(cwd)

# Insert the project root dir as the first element in the PYTHONPATH.
# This lets us ensure that the source package is imported, and that its
# version is used.
sys.path.insert(0, project_root)

from django.conf import settings
settings.configure()

import django_mysql

# -- General configuration ---------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.viewcode',
    'sphinx.ext.intersphinx',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Django MySQL'
copyright = u'2015, Adam Johnson'

# The version info for the project you're documenting, acts as replacement
# for |version| and |release|, also used in various other places throughout
# the built documents.
#
# The short X.Y version.
version = django_mysql.__version__
# The full version, including alpha/beta/rc tags.
release = django_mysql.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to
# some non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
modindex_common_prefix = ["django_mysql."]

# If true, keep warnings as "system message" paragraphs in the built
# documents.
# keep_warnings = False

# -- Options for HTML output -------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Theme options are theme-specific and customize the look and feel of a
# theme further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as
# html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the
# top of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon
# of the docs.  This file should be a Windows icon file (.ico) being
# 16x16 or 32x32 pixels large.
html_favicon = 'favicon.ico'

# Add any paths that contain custom static files (such as style sheets)
# here, relative to this directory. They are copied after the builtin
# static files, so a file named "default.css" will overwrite the builtin
# "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page
# bottom, using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names
# to template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer.
# Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer.
# Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages
# will contain a <link> tag referring to it.  The value of this option
# must be the base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'django-mysqldoc'

# -- Options for LaTeX output ------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
    ('index', 'django-mysql.tex',
     u'Django MySQL Documentation',
     u'Adam Johnson', 'manual'),
]

# The name of an image file (relative to this directory) to place at
# the top of the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings
# are parts, not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'django-mysql',
     u'Django MySQL Documentation',
     [u'Adam Johnson'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ----------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'django-mysql',
     u'Django MySQL Documentation',
     u'Adam Johnson',
     'django-mysql',
     'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

intersphinx_mapping = {
    'django': ('http://docs.djangoproject.com/en/dev/',
               'http://docs.djangoproject.com/en/dev/_objects/'),
}
<|endoftext|>


<commit_before>import random
import string
import json
import pandas

import pygraphistry
import util

class Plotter(object):
    def __init__(self):
        # Bindings
        self.edges = None
        self.nodes = None
        self.source = None
        self.destination = None
        self.node = None
        self.edge_title = None
        self.edge_label = None
        self.edge_color = None
        self.edge_weight = None
        self.point_title = None
        self.point_label = None
        self.point_color = None
        self.point_size = None
        # Settings
        self.height = 500
        self.url_params = {'info': 'true'}

    def bind(self, source=None, destination=None, node=None,
             edge_title=None, edge_label=None, edge_color=None, edge_weight=None,
             point_title=None, point_label=None, point_color=None, point_size=None):
        self.source = source or self.source
        self.destination = destination or self.destination
        self.node = node or self.node

        self.edge_title = edge_title or self.edge_title
        self.edge_label = edge_label or self.edge_label
        self.edge_color = edge_color or self.edge_color
        self.edge_weight = edge_weight or self.edge_weight

        self.point_title = point_title or self.point_title
        self.point_label = point_label or self.point_label
        self.point_color = point_color or self.point_color
        self.point_size = point_size or self.point_size
        return self

    def nodes(nodes):
        self.nodes = nodes
        return self

    def edges(edges):
        self.edges = edges
        return self

    def graph(ig):
        self.edges = ig
        self.nodes = None
        return self

    def settings(self, height=None, url_params={}):
        self.height = height or self.height
        self.url_params = dict(self.url_params, **url_params)
        return self

    def plot(self, graph=None, nodes=None):
        if graph is None:
            if self.edges is None:
                raise ValueError('Must specify graph/edges')
            g = self.edges
        else:
            g = graph
        n = self.nodes if nodes is None else nodes
        dataset = self._plot_dispatch(g, n)
        if dataset is None:
            raise TypeError('Expected Pandas dataframe or Igraph graph')

        dataset_name = pygraphistry.PyGraphistry._etl(json.dumps(dataset))
        viz_url = pygraphistry.PyGraphistry._viz_url(dataset_name, self.url_params)

        if util.in_ipython() is True:
            from IPython.core.display import HTML
            return HTML(self._iframe(viz_url))
        else:
            print 'Url: ', viz_url
            import webbrowser
            webbrowser.open(viz_url)
            return self

    def pandas2igraph(self, edges):
        import igraph
        eattribs = edges.columns.values.tolist()
        eattribs.remove(self.source)
        eattribs.remove(self.destination)
        cols = [self.source, self.destination] + eattribs
        etuples = [tuple(x) for x in edges[cols].values]
        return igraph.Graph.TupleList(etuples, directed=True, edge_attrs=eattribs,
                                      vertex_name_attr=self.node)

    def igraph2pandas(self, ig):
        def get_edgelist(ig):
            idmap = dict(enumerate(ig.vs[self.node]))
            for e in ig.es:
                t = e.tuple
                yield dict({self.source: idmap[t[0]], self.destination: idmap[t[1]]},
                            **e.attributes())

        edata = get_edgelist(ig)
        ndata = [v.attributes() for v in ig.vs]
        nodes = pandas.DataFrame(ndata, columns=ig.vs.attributes())
        cols = [self.source, self.destination] + ig.es.attributes()
        edges = pandas.DataFrame(edata, columns=cols)
        return (edges, nodes)

    def _plot_dispatch(self, graph, nodes):
        if isinstance(graph, pandas.core.frame.DataFrame):
            return self._pandas2dataset(graph, nodes)

        try:
            import igraph
            if isinstance(graph, igraph.Graph):
                (e, n) = self.igraph2pandas(graph)
                return self._pandas2dataset(e, n)
        except ImportError:
            pass

        return None

    def _pandas2dataset(self, edges, nodes):
        elist = edges.reset_index()
        if self.edge_color:
            elist['edgeColor'] = elist[self.edge_color]
        if self.edge_label:
            elist['edgeLabel'] = elist[self.edge_label]
        if self.edge_title:
            elist['edgeTitle'] = elist[self.edge_title]
        if self.edge_weight:
            elist['edgeWeight'] = elist[self.edge_weight]
        if nodes is None:
            self.node = '__nodeid__'
            nodes = pandas.DataFrame()
            nodes[self.node] = pandas.concat([edges[self.source], edges[self.destination]],
                                             ignore_index=True).drop_duplicates()
            self.point_title = self.node

        nlist = nodes.reset_index()
        if self.point_color:
            nlist['pointColor'] = nlist[self.point_color]
        if self.point_label:
            nlist['pointLabel'] = nlist[self.point_label]
        if self.point_title:
            nlist['pointTitle'] = nlist[self.point_title]
        if self.point_size:
            nlist['pointSize'] = nlist[self.point_size]
        return self._make_dataset(elist.to_dict(orient='records'),
                                  nlist.to_dict(orient='records'))

    def _make_dataset(self, elist, nlist=None):
        name = ''.join(random.choice(string.ascii_uppercase +
                                     string.digits) for _ in range(10))
        bindings = {'idField': self.node, 'destinationField': self.destination,
                    'sourceField': self.source}
        dataset = {'name': pygraphistry.PyGraphistry._dataset_prefix + name,
                   'bindings': bindings, 'type': 'edgelist', 'graph': elist}
        if nlist:
            dataset['labels'] = nlist
        return dataset

    def _iframe(self, url):
        tag = ''
        return tag % (url, self.height)
<commit_msg>NewAPI: Check that at least source/dest are bound before attempting to plot<commit_after>import random
import string
import json
import pandas

import pygraphistry
import util

class Plotter(object):
    def __init__(self):
        # Bindings
        self.edges = None
        self.nodes = None
        self.source = None
        self.destination = None
        self.node = None
        self.edge_title = None
        self.edge_label = None
        self.edge_color = None
        self.edge_weight = None
        self.point_title = None
        self.point_label = None
        self.point_color = None
        self.point_size = None
        # Settings
        self.height = 500
        self.url_params = {'info': 'true'}

    def bind(self, source=None, destination=None, node=None,
             edge_title=None, edge_label=None, edge_color=None, edge_weight=None,
             point_title=None, point_label=None, point_color=None, point_size=None):
        self.source = source or self.source
        self.destination = destination or self.destination
        self.node = node or self.node

        self.edge_title = edge_title or self.edge_title
        self.edge_label = edge_label or self.edge_label
        self.edge_color = edge_color or self.edge_color
        self.edge_weight = edge_weight or self.edge_weight

        self.point_title = point_title or self.point_title
        self.point_label = point_label or self.point_label
        self.point_color = point_color or self.point_color
        self.point_size = point_size or self.point_size
        return self

    def nodes(nodes):
        self.nodes = nodes
        return self

    def edges(edges):
        self.edges = edges
        return self

    def graph(ig):
        self.edges = ig
        self.nodes = None
        return self

    def settings(self, height=None, url_params={}):
        self.height = height or self.height
        self.url_params = dict(self.url_params, **url_params)
        return self

    def plot(self, graph=None, nodes=None):
        if graph is None:
            if self.edges is None:
                raise ValueError('Must specify graph/edges')
            g = self.edges
        else:
            g = graph
        n = self.nodes if nodes is None else nodes

        if self.source is None or self.destination is None:
            raise ValueError('Source/destination must be bound before plotting.')
        if n is not None and self.node is None:
            raise ValueError('Node identifier must be bound when using node dataframe')
        dataset = self._plot_dispatch(g, n)
        if dataset is None:
            raise TypeError('Expected Pandas dataframe or Igraph graph')

        dataset_name = pygraphistry.PyGraphistry._etl(json.dumps(dataset))
        viz_url = pygraphistry.PyGraphistry._viz_url(dataset_name, self.url_params)

        if util.in_ipython() is True:
            from IPython.core.display import HTML
            return HTML(self._iframe(viz_url))
        else:
            print 'Url: ', viz_url
            import webbrowser
            webbrowser.open(viz_url)
            return self

    def pandas2igraph(self, edges):
        import igraph
        eattribs = edges.columns.values.tolist()
        eattribs.remove(self.source)
        eattribs.remove(self.destination)
        cols = [self.source, self.destination] + eattribs
        etuples = [tuple(x) for x in edges[cols].values]
        return igraph.Graph.TupleList(etuples, directed=True, edge_attrs=eattribs,
                                      vertex_name_attr=self.node)

    def igraph2pandas(self, ig):
        def get_edgelist(ig):
            idmap = dict(enumerate(ig.vs[self.node]))
            for e in ig.es:
                t = e.tuple
                yield dict({self.source: idmap[t[0]], self.destination: idmap[t[1]]},
                            **e.attributes())

        edata = get_edgelist(ig)
        ndata = [v.attributes() for v in ig.vs]
        nodes = pandas.DataFrame(ndata, columns=ig.vs.attributes())
        cols = [self.source, self.destination] + ig.es.attributes()
        edges = pandas.DataFrame(edata, columns=cols)
        return (edges, nodes)

    def _plot_dispatch(self, graph, nodes):
        if isinstance(graph, pandas.core.frame.DataFrame):
            return self._pandas2dataset(graph, nodes)

        try:
            import igraph
            if isinstance(graph, igraph.Graph):
                (e, n) = self.igraph2pandas(graph)
                return self._pandas2dataset(e, n)
        except ImportError:
            pass

        return None

    def _pandas2dataset(self, edges, nodes):
        elist = edges.reset_index()
        if self.edge_color:
            elist['edgeColor'] = elist[self.edge_color]
        if self.edge_label:
            elist['edgeLabel'] = elist[self.edge_label]
        if self.edge_title:
            elist['edgeTitle'] = elist[self.edge_title]
        if self.edge_weight:
            elist['edgeWeight'] = elist[self.edge_weight]
        if nodes is None:
            self.node = '__nodeid__'
            nodes = pandas.DataFrame()
            nodes[self.node] = pandas.concat([edges[self.source], edges[self.destination]],
                                             ignore_index=True).drop_duplicates()
            self.point_title = self.node

        nlist = nodes.reset_index()
        if self.point_color:
            nlist['pointColor'] = nlist[self.point_color]
        if self.point_label:
            nlist['pointLabel'] = nlist[self.point_label]
        if self.point_title:
            nlist['pointTitle'] = nlist[self.point_title]
        if self.point_size:
            nlist['pointSize'] = nlist[self.point_size]
        return self._make_dataset(elist.to_dict(orient='records'),
                                  nlist.to_dict(orient='records'))

    def _make_dataset(self, elist, nlist=None):
        name = ''.join(random.choice(string.ascii_uppercase +
                                     string.digits) for _ in range(10))
        bindings = {'idField': self.node, 'destinationField': self.destination,
                    'sourceField': self.source}
        dataset = {'name': pygraphistry.PyGraphistry._dataset_prefix + name,
                   'bindings': bindings, 'type': 'edgelist', 'graph': elist}
        if nlist:
            dataset['labels'] = nlist
        return dataset

    def _iframe(self, url):
        tag = ''
        return tag % (url, self.height)
<|endoftext|>


<commit_before>"""
 Common modifiers factories for use in resources.

 Although modifier concept is defined as a callable (Explanation)->Explanation,
 different resources may need to tweak its behavior more or less.

 So, most modifiers here are parametrised and all modifiers are supposed
 to be created by their factory functions.

 But for debugging convenience it could be nice to have an informative
 representation of parametrized modifier, not just its family.
 For this purpose every modifier is a subclass of Modifier class,
 which remembers parameters it was constructed with in his repr.

 For factory writing convenience there are two decorators:
 - modifier_factory, which just wraps your factory with Modifier.__init__ call;
 - title_text_modifier_factory is for modifiers such as re_replace, which can
   equally successful work with explanations title or text.
   So, the factory should just produce functions str->str, the decorator will
   use them properly.
   By default produced modifiers affect Explanation.text, but it can be changed
   by passing target_field='title' keyword parameter to the factory.
"""

import itertools
import re
import copy

from hb_res.explanations import Explanation, ExplanationKey
from preparation.lang_utils.cognates import are_cognates
from preparation.lang_utils.morphology import get_valid_noun_initial_form
from preparation.lang_utils.morphology import is_remarkable

GAP_VALUE = '*пропуск*'

ALPH_RE = '[А-Яа-я]'
NOTALPH_RE = '[^А-Яа-я]'
WORD_RE = ALPH_RE + '+'

class Modifier:
    def __init__(self, *args, _name=None, **kwargs):
        if _name is None:
            _name = self.__name__
        self.__repr = '<modifier {}({})>'.format(
            _name,
            ', '.join(itertools.chain(
                map(repr, args),
                itertools.starmap('{}={!r}'.format, kwargs.items())
            ))
        )

    def __repr__(self):
        return self.__repr

    def __call__(self, e: Explanation):
        raise NotImplementedError

def modifier_factory(factory, name=None)->Modifier:
    """
    Wraps modifier factory's products in Modifier subclass, remembering parameters the modifier was parametrized.

    :param factory:
    :param name: factory's name
    :return: Modifier
    """
    if name is None:
        name = factory.__name__

    class DecoratedModifier(Modifier):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, _name=name, **kwargs)
            self._mod = factory(*args, **kwargs)

        def __call__(self, e):
            return self._mod(e)

    return DecoratedModifier

def title_text_modifier_factory(factory, name=None)->Modifier:
    """
    Adds target_field kwarg to modifier factory, allowing modification either 'title' or 'text' fields.

    :param factory:
    :param name:
    :return: Modifier
    """

    def decorator(*args, target_field='text', **kwargs):
        assert target_field in {'text', 'title'}

        mod = factory(*args, **kwargs)

        def apply(explanation):
            ret = copy.copy(explanation)
            setattr(ret, target_field, mod(getattr(ret, target_field)))
            if getattr(ret, target_field) is None:
                ret = None
            return ret

        return apply

    if name is None:
        name = factory.__name__

    return modifier_factory(decorator, name)

@title_text_modifier_factory
def strip(chars: str=None):
    """
    Constructs modifier that calls `str.strip` on field specified in `target_field` kwarg.

    `target_field` is a kwarg that defaults to 'text'.

    :param chars: same as for str.strip
    :return Modifier
    """
    return lambda s: s.strip(chars)

@title_text_modifier_factory
def re_replace(pattern, replacement: str, flags: int=0):
    """
    Constructs modifier that replaces e.`target_field`
    with re.sub(pattern, replacement, e.`target_field`, flags=flags)
    (for given Explanation e).

    `target_field` is a kwarg that defaults to 'text'.

    :param pattern: str or compiled regexp to replace
    :param replacement: replacement
    :param flags: re compilation flags
    :return Modifier
    """
    if isinstance(pattern, (str, bytes)):
        pattern = re.compile(pattern, flags)
    return lambda s: pattern.sub(replacement, s)

@title_text_modifier_factory
def re_fullmatch_ban(pattern, flags: int=0):
    """
    Constructs modifier that bans explanations whose `target_field` matches exactly to `pattern` regexp.

    target_field is a kwarg that defaults to 'text'.

    :param pattern: regexp
    :param flags: re construction flags
    :return Modifier
    """
    #we emulate re.fullmatch behaviour by adding '^' and '$' to the pattern
    assert isinstance(pattern, (str, bytes))
    if isinstance(pattern, bytes):
        pattern = b'^' + pattern + b'$'
    else:
        pattern = '^' + pattern + '$'
    pattern = re.compile(pattern, flags)
    return lambda s: s if re.match(pattern, s, flags) is None else None

@modifier_factory
def calculate_key():
    """
    Constructs modifier that fills .key field if it is None.

    :return Modifier
    """

    def apply(e: Explanation):
        if e.key is not None:
            return e
        ret = copy.copy(e)
        ret.key = ExplanationKey.for_text(ret.text)
        return ret

    return apply

@modifier_factory
def normalize_title(score_threshold: float=0.):
    """
    Constructs modifier that replaces explanation's title with normalized noun if possible.
    Otherwise bans the explanation (returns None).

    score_threshold is used as minimum confidence enough to consider a parse variant.

    :param score_threshold:
    :return: Modifier
    """

    def apply(e: Explanation):
        new_title = get_valid_noun_initial_form(e.title, score_threshold)
        if new_title is None:
            return None
        ret = copy.copy(e)
        ret.title = new_title
        return ret

    return apply

@modifier_factory
def shadow_cognates(length_threshold: int=None, sep_re='\\s+'):
    """
    Constructs modifier that splits explanation's text by sep_re regexp and replaces title's cognates with
    GAP_VALUE.

    If length_threshold is specified, the modifier treats any word having with title a common substring of
    at least length_threshold len as cognate.

    :param length_threshold:
    :param sep_re:
    :return: Modifier
    """
    if isinstance(sep_re, (str, bytes)):
        sep_re = re.compile(sep_re)

    def apply(e: Explanation):
        ret = copy.copy(e)
        for w in sep_re.split(ret.text):
            if are_cognates(w, e.title, length_threshold=length_threshold):
                ret.text = re.sub('(^|(?<={notalph})){badword}($|(?={notalph}))'.format(badword=w, notalph=NOTALPH_RE),
                                  GAP_VALUE,
                                  ret.text)
        return ret

    return apply

@modifier_factory
def choose_normal_words_in_explanation(separator: str):
    def apply(e: Explanation):
        ret = copy.copy(e)
        new_list = [w for w in e.text.split(separator)
                    if get_valid_noun_initial_form(w) is not None]
        if len(new_list) == 0:
            return None
        ret.text = separator.join(new_list)
        return ret
    return apply

@modifier_factory
def delete_cognates(length_threshold: int, separator: str):
    def apply(e: Explanation):
        ret = copy.copy(e)
        new_list = [w for w in e.text.split(separator)
                    if not are_cognates(w, e.title, length_threshold)]
        if len(new_list) == 0:
            return None
        ret.text = separator.join(new_list)
        return ret
    return apply

@modifier_factory
def check_contains_valid_parts(required, enough_score, sep_re, gap='пропуск'):
    def apply(e: Explanation):
        have = 0
        for word in re.split(sep_re, e.text):
            if len(word) > 0 and word != gap and is_remarkable(word, enough_score):
                have += 1
        if have >= required:
            return e
        else:
            return None
    return apply

@modifier_factory
def delete_multiple_gaps(limit: int):
    def apply(e: Explanation):
        num = len(re.findall('\*пропуск\*', e.text))
        if num > limit:
            return None
        else:
            return e
    return apply
<commit_msg>Pull some cool modifiers from clean-definitions<commit_after>"""
 Common modifiers factories for use in resources.

 Although modifier concept is defined as a callable (Explanation)->Explanation,
 different resources may need to tweak its behavior more or less.

 So, most modifiers here are parametrised and all modifiers are supposed
 to be created by their factory functions.

 But for debugging convenience it could be nice to have an informative
 representation of parametrized modifier, not just its family.
 For this purpose every modifier is a subclass of Modifier class,
 which remembers parameters it was constructed with in his repr.

 For factory writing convenience there are two decorators:
 - modifier_factory, which just wraps your factory with Modifier.__init__ call;
 - title_text_modifier_factory is for modifiers such as re_replace, which can
   equally successful work with explanations title or text.
   So, the factory should just produce functions str->str, the decorator will
   use them properly.
   By default produced modifiers affect Explanation.text, but it can be changed
   by passing target_field='title' keyword parameter to the factory.
"""

import itertools
import re
import copy

from hb_res.explanations import Explanation, ExplanationKey
from preparation.lang_utils.cognates import are_cognates
from preparation.lang_utils.morphology import get_valid_noun_initial_form
from preparation.lang_utils.morphology import is_remarkable

GAP_VALUE = '*пропуск*'

ALPH_RE = '[ЁёА-Яа-я]'
NOTALPH_RE = '[^ЁёА-Яа-я]'
WORD_RE = ALPH_RE + '+'

class Modifier:
    def __init__(self, *args, _name=None, **kwargs):
        if _name is None:
            _name = self.__name__
        self.__repr = '<modifier {}({})>'.format(
            _name,
            ', '.join(itertools.chain(
                map(repr, args),
                itertools.starmap('{}={!r}'.format, kwargs.items())
            ))
        )

    def __repr__(self):
        return self.__repr

    def __call__(self, e: Explanation):
        raise NotImplementedError

def modifier_factory(factory, name=None)->Modifier:
    """
    Wraps modifier factory's products in Modifier subclass, remembering parameters the modifier was parametrized.

    :param factory:
    :param name: factory's name
    :return: Modifier
    """
    if name is None:
        name = factory.__name__

    class DecoratedModifier(Modifier):
        def __init__(self, *args, **kwargs):
            super().__init__(*args, _name=name, **kwargs)
            self._mod = factory(*args, **kwargs)

        def __call__(self, e):
            return self._mod(e)

    return DecoratedModifier

def title_text_modifier_factory(factory, name=None)->Modifier:
    """
    Adds target_field kwarg to modifier factory, allowing modification either 'title' or 'text' fields.

    :param factory:
    :param name:
    :return: Modifier
    """

    def decorator(*args, target_field='text', **kwargs):
        assert target_field in {'text', 'title'}

        mod = factory(*args, **kwargs)

        def apply(explanation):
            ret = copy.copy(explanation)
            setattr(ret, target_field, mod(getattr(ret, target_field)))
            if getattr(ret, target_field) is None:
                ret = None
            return ret

        return apply

    if name is None:
        name = factory.__name__

    return modifier_factory(decorator, name)

@title_text_modifier_factory
def strip(chars: str=None):
    """
    Constructs modifier that calls `str.strip` on field specified in `target_field` kwarg.

    `target_field` is a kwarg that defaults to 'text'.

    :param chars: same as for str.strip
    :return Modifier
    """
    return lambda s: s.strip(chars)

@title_text_modifier_factory
def str_replace(pattern: str, replacement: str, count: int=-1):
    """
    Constructs modifier that replaces e.`target_field` with e.`target_field`.replace(pattern, replacement, count)
    (for given Explanation e).

    `target_field` is a kwarg that defaults to 'text'.

    :param pattern: pattern to replace
    :param replacement: replacement
    :param count: number of times to perform replacement
    :return Modifier
    """
    return lambda s: s.replace(pattern, replacement, count)

@title_text_modifier_factory
def translate(*args):
    """
    Constructs a modifier that applies str.translate(e.`target_field`, str.maketrans(*args))
    on given explanation e.

    :return: Modifier
    """
    trans = str.maketrans(*args)
    return lambda s: s.translate(trans)

@title_text_modifier_factory
def re_replace(pattern, replacement: str, flags: int=0):
    """
    Constructs modifier that replaces e.`target_field`
    with re.sub(pattern, replacement, e.`target_field`, flags=flags)
    (for given Explanation e).

    `target_field` is a kwarg that defaults to 'text'.

    :param pattern: str or compiled regexp to replace
    :param replacement: replacement
    :param flags: re compilation flags
    :return Modifier
    """
    if isinstance(pattern, (str, bytes)):
        pattern = re.compile(pattern, flags)
    return lambda s: pattern.sub(replacement, s)

@title_text_modifier_factory
def re_fullmatch_ban(pattern, flags: int=0):
    """
    Constructs modifier that bans explanations whose `target_field` matches exactly to `pattern` regexp.

    target_field is a kwarg that defaults to 'text'.

    :param pattern: regexp
    :param flags: re construction flags
    :return Modifier
    """
    #we emulate re.fullmatch behaviour by adding '^' and '$' to the pattern
    assert isinstance(pattern, (str, bytes))
    if isinstance(pattern, bytes):
        pattern = b'^' + pattern + b'$'
    else:
        pattern = '^' + pattern + '$'
    pattern = re.compile(pattern, flags)
    return lambda s: s if re.match(pattern, s, flags) is None else None

@title_text_modifier_factory
def re_search_ban(pattern, flags: int=0):
    """
    Constructs modifier that bans explanations whose `target_field` contains a substring that
    matches `pattern` regexp.

    target_field is a kwarg that defaults to 'text'.

    :param pattern: regexp
    :param flags: re construction flags
    :return Modifier
    """
    pattern = re.compile(pattern, flags)
    return lambda s: s if pattern.search(s) is None else None

@modifier_factory
def calculate_key():
    """
    Constructs modifier that fills .key field if it is None.

    :return Modifier
    """

    def apply(e: Explanation):
        if e.key is not None:
            return e
        ret = copy.copy(e)
        ret.key = ExplanationKey.for_text(ret.text)
        return ret

    return apply

@modifier_factory
def normalize_title(score_threshold: float=0.):
    """
    Constructs modifier that replaces explanation's title with normalized noun if possible.
    Otherwise bans the explanation (returns None).

    score_threshold is used as minimum confidence enough to consider a parse variant.

    :param score_threshold:
    :return: Modifier
    """

    def apply(e: Explanation):
        new_title = get_valid_noun_initial_form(e.title, score_threshold)
        if new_title is None:
            return None
        ret = copy.copy(e)
        ret.title = new_title
        return ret

    return apply

@modifier_factory
def shadow_cognates(length_threshold: int=None, sep_re='\\s+'):
    """
    Constructs modifier that splits explanation's text by sep_re regexp and replaces title's cognates with
    GAP_VALUE.

    If length_threshold is specified, the modifier treats any word having with title a common substring of
    at least length_threshold len as cognate.

    :param length_threshold:
    :param sep_re:
    :return: Modifier
    """
    if isinstance(sep_re, (str, bytes)):
        sep_re = re.compile(sep_re)

    def apply(e: Explanation):
        ret = copy.copy(e)
        for w in sep_re.split(ret.text):
            if are_cognates(w, e.title, length_threshold=length_threshold):
                ret.text = re.sub('(^|(?<={notalph})){badword}($|(?={notalph}))'.format(badword=w, notalph=NOTALPH_RE),
                                  GAP_VALUE,
                                  ret.text)
        return ret

    return apply

@modifier_factory
def choose_normal_words_in_explanation(separator: str):
    def apply(e: Explanation):
        ret = copy.copy(e)
        new_list = [w for w in e.text.split(separator)
                    if get_valid_noun_initial_form(w) is not None]
        if len(new_list) == 0:
            return None
        ret.text = separator.join(new_list)
        return ret
    return apply

@modifier_factory
def delete_cognates(length_threshold: int, separator: str):
    def apply(e: Explanation):
        ret = copy.copy(e)
        new_list = [w for w in e.text.split(separator)
                    if not are_cognates(w, e.title, length_threshold)]
        if len(new_list) == 0:
            return None
        ret.text = separator.join(new_list)
        return ret
    return apply

@modifier_factory
def check_contains_valid_parts(required, enough_score, sep_re, gap='пропуск'):
    def apply(e: Explanation):
        have = 0
        for word in re.split(sep_re, e.text):
            if len(word) > 0 and word != gap and is_remarkable(word, enough_score):
                have += 1
        if have >= required:
            return e
        else:
            return None
    return apply

@modifier_factory
def delete_multiple_gaps(limit: int):
    def apply(e: Explanation):
        num = len(re.findall('\*пропуск\*', e.text))
        if num > limit:
            return None
        else:
            return e
    return apply
<|endoftext|>


<commit_before>"""
Django settings for hello_world project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '80o&#_i#(1haanv=!4w1ns3r5oal!rih-)lm8+@aens#^i(eu7'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'mod_wsgi.server',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'hello_world.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'hello_world.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
<commit_msg>Remove loading of mod_wsgi.server application.<commit_after>"""
Django settings for hello_world project.

Generated by 'django-admin startproject' using Django 1.9.1.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.9/ref/settings/
"""

import os

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/

# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = '80o&#_i#(1haanv=!4w1ns3r5oal!rih-)lm8+@aens#^i(eu7'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True

ALLOWED_HOSTS = []

# Application definition

INSTALLED_APPS = [
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
]

MIDDLEWARE_CLASSES = [
    'django.middleware.security.SecurityMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
]

ROOT_URLCONF = 'hello_world.urls'

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'DIRS': [],
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.template.context_processors.debug',
                'django.template.context_processors.request',
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
            ],
        },
    },
]

WSGI_APPLICATION = 'hello_world.wsgi.application'

# Database
# https://docs.djangoproject.com/en/1.9/ref/settings/#databases

DATABASES = {
    'default': {
        'ENGINE': 'django.db.backends.sqlite3',
        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
    }
}

# Password validation
# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators

AUTH_PASSWORD_VALIDATORS = [
    {
        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
    },
    {
        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
    },
]

# Internationalization
# https://docs.djangoproject.com/en/1.9/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.9/howto/static-files/

STATIC_URL = '/static/'

STATIC_ROOT = os.path.join(BASE_DIR, 'static')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Metagenomics Training Material RoundUp documentation build configuration file, created by
# sphinx-quickstart on Fri Apr  8 11:11:20 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Metagenomics Training Material RoundUp'
copyright = u'2016, Adina Howe (from GOBLET/Elixir Hackathon)'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'MetagenomicsTrainingMaterialRoundUpdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'MetagenomicsTrainingMaterialRoundUp.tex', u'Metagenomics Training Material RoundUp Documentation',
   u'Adina Howe (from GOBLET/Elixir Hackathon)', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'metagenomicstrainingmaterialroundup', u'Metagenomics Training Material RoundUp Documentation',
     [u'Adina Howe (from GOBLET/Elixir Hackathon)'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'MetagenomicsTrainingMaterialRoundUp', u'Metagenomics Training Material RoundUp Documentation',
   u'Adina Howe (from GOBLET/Elixir Hackathon)', 'MetagenomicsTrainingMaterialRoundUp', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Options for extensions.
disqus_shortname = 'metag-training'  # Add this line to conf.py.
<commit_msg>disqus add<commit_after># -*- coding: utf-8 -*-
#
# Metagenomics Training Material RoundUp documentation build configuration file, created by
# sphinx-quickstart on Fri Apr  8 11:11:20 2016.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinxcontrib.disqus']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Metagenomics Training Material RoundUp'
copyright = u'2016, Adina Howe (from GOBLET/Elixir Hackathon)'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0'
# The full version, including alpha/beta/rc tags.
release = '0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'MetagenomicsTrainingMaterialRoundUpdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'MetagenomicsTrainingMaterialRoundUp.tex', u'Metagenomics Training Material RoundUp Documentation',
   u'Adina Howe (from GOBLET/Elixir Hackathon)', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'metagenomicstrainingmaterialroundup', u'Metagenomics Training Material RoundUp Documentation',
     [u'Adina Howe (from GOBLET/Elixir Hackathon)'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'MetagenomicsTrainingMaterialRoundUp', u'Metagenomics Training Material RoundUp Documentation',
   u'Adina Howe (from GOBLET/Elixir Hackathon)', 'MetagenomicsTrainingMaterialRoundUp', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Options for extensions.
disqus_shortname = 'metag-training'  # Add this line to conf.py.
<|endoftext|>


<commit_before><commit_msg>add version<commit_after>
__version__ = '0.0.1'
<|endoftext|>


<commit_before>#
# Copyright (c) SAS Institute Inc
#

from urlparse import urljoin
import hashlib
import json
import logging
import os

from conary.lib import util

import requests

from .pompackage import PomPackage

__all__ = ('Client', 'PomPackage')

log = logging.getLogger(__name__)

# Turn down the logging level on requeests
requests_logger = logging.getLogger('requests.packages.urllib3.connectionpool')
requests_logger.setLevel(logging.WARN)

def detail(func):
    def wrapper(*args, **kwargs):
        info = kwargs.pop('info', False)
        properties = kwargs.pop('properties', False)

        detail = []
        if info:
            detail.append('info')
        elif properties:
            detail.append('properties')
        if detail:
            kwargs.setdefault(
                'headers', {})['x-result-detail'] = ','.join(detail)
        return func(*args, **kwargs)
    return wrapper

def repos(func):
    def wrapper(*args, **kwargs):
        repos = kwargs.pop('repos', None)
        if repos:
            if isinstance(repos, (str, unicode)):
                repos = [repos]
            kwargs.setdefault('params', {})['repos'] = ','.join(repos)
        return func(*args, **kwargs)
    return wrapper

class Struct(object):
    def __init__(self):
        self.status_code = None
        self.text = None

    def json(self):
        return json.loads(self.text)

class Cache(object):
    def __init__(self, cacheDir='/tmp/artifactory_cache'):
        self.cacheDir = cacheDir
        util.mkdirChain(self.cacheDir)

    def cache(self, method, uri, res=None):
        key = method + uri
        h = hashlib.sha1(key).hexdigest()
        dh = '%s.data' % h

        hpath = os.path.join(self.cacheDir, h)
        dhpath = os.path.join(self.cacheDir, dh)

        if os.path.exists(hpath) and os.path.exists(dhpath) and not res:
            data = json.load(open(dhpath))
            data['text'] = open(hpath).read().decode('utf-8')
            res = Struct()
            for k, v in data.iteritems():
                setattr(res, k, v)

        elif res is not None:
            with open(hpath, 'wb') as fh:
                fh.write(res.text.encode('utf-8'))
            with open(dhpath, 'wb') as fh:
                json.dump({
                    'status_code': res.status_code,
                    'method': method,
                    'uri': uri,
                }, fh)

        return res

class Client(object):
    """
    Client for talking to artfactory api
    """

    def __init__(self, cfg, headers=None):
        """
        Create a client from a updatebot config object.

        If there are default headers you want used for every request, pass in
        a dictionary:
            Client("http://example.com,
                headers={'my-default-header': 'foo'})

        :param UpdateBotConfig cfg: cfg object to use for configuring the client
        :param dict headers: dictionary of headers to use for all requests
        """
        # append a / to url if it doesn't end with one
        self._cfg = cfg
        self._url = cfg.repositoryUrl + '/'
        self._api_url = urljoin(self._url, "api/")

        self._cache = Cache()

    def _get(self, uri, **kwargs):
        res = self._request('GET', uri, **kwargs)
        return res

    def _post(self, uri,  **kwargs):
        res = self._request('POST', uri, **kwargs)
        return res

    def _head(self, uri, **kwargs):
        res = self._request('HEAD', uri, return_json=False, **kwargs)
        return res

    def _request(self, method, uri, return_json=True, **kwargs):
        url = urljoin(self._url, uri)
        res = self._cache.cache(method, uri)
        if not res:
            log.debug('requesting %s %s', method, url)
            res = requests.request(method, url, **kwargs)
            self._cache.cache(method, uri, res)
        else:
            log.debug('hit %s %s', method, url)
        if return_json:
            return res.json()
        return res

    def _search(self, path, **kwargs):
        log.debug("search(%s, kwargs=%s)", path, kwargs)
        if 'auth' not in kwargs:
            kwargs['auth'] = self._cfg.artifactoryUser.find(self._cfg.repositoryUrl)

        res = self._get(urljoin("api/search/", path), **kwargs)
        urls = [r['uri'] for r in res.get('results', [])]

        for url in urls:
            res = self._get(url, return_json=False)
            if res.status_code == requests.codes.not_found:
                log.warn('error fetching %s', url)
                continue
            yield res.json()

    @staticmethod
    def constructPath(groupId, artifactId, version=None, artifactName=None,
                      extension='pom', relative=False):
        """Construct a maven 2 path

        :param groupId: group name
        :type groupId: string
        :param artifactId: artifact name
        :type artifactId: string
        :param version: (None) version of artifact
        :type version: string
        :param artifactName: (None) artifact name if not the artifactId
        :type artifactName: string
        :param extension: (pom) file extention of artifact
        :type extension: string
        :param relative: (False) generate a relative path
        :type relative: bool
        """
        path = '/{0}/{1}'
        if relative:
            path = path[1:]

        if version:
            path += '/{2}'

        if artifactName:
            path += '/{3}.{4}'
        else:
            path += '/{1}-{2}.{4}'

        return path.format(
            groupId.replace('.', '/'),
            artifactId,
            version,
            artifactName,
            extension,
            )

    def artifactUrl(self, group, artifact, version, extension='pom'):
        path = self.constructPath(group, artifact, version, extension=extension,
                                  relative=True)
        uri = urljoin('repo/', path)
        return urljoin(self._url, uri)

    def checkJar(self, group, artifact, version):
        path = self.constructPath(group, artifact, version, extension='jar',
                                  relative=True)
        uri = urljoin('repo/', path)
        res = self._head(uri)
        return res.status_code == requests.codes.ok

    @repos
    @detail
    def quick_search(self, name, **kwargs):
        kwargs.setdefault('params', {})['name'] = name
        results = self._search("artifact/", **kwargs)
        return results

    @repos
    def class_search(self, name, **kwargs):
        kwargs.setdefault('params', {})['name'] = name
        return self._get("archive/", **kwargs)

    @detail
    @repos
    def gavc_search(self, group=None, artifact=None, version=None,
                    classifier=None, **kwargs):
        """
        Search by Maven coordinates: GroupId, ArtifactId, Version &
        Classifier. Search must contain at least one argument. Can
        limit search to specific repositories (local or caches).

        You can retrieve extra information or properites by setting
        the `info` or `properties` flags to True

        :param str group: a group id to search for
        :param str artifact: an artifact id to search fo
        :param str version: a version string
        :param str classifier: a classifier string
        :param repos: limit search to these repositories, local or cache
        :type repos: str or list
        :param bool info: add all extra information
        :param bool properties: get the properties of found artifacts
        """
        params = {}
        if group:
            params["g"] = group
        if artifact:
            params["a"] = artifact
        if version:
            params['v'] = version
        if classifier:
            params['c'] = classifier
        if not params:
            raise ValueError("Must provide one of: group, artifact, version,"
                             " or classifier")
        kwargs.setdefault('params', {}).update(params)
        results = self._search("gavc/", **kwargs)
        return results

    @detail
    @repos
    def property_search(self, properties, **kwargs):
        if not isinstance(properties, dict):
            properties = dict(properties)
        kwargs.setdefault('params', {}).update(properties)
        return self._search("prop/", **kwargs)

    @detail
    @repos
    def checksum_search(self, checksum, checksum_type, **kwargs):
        if checksum_type not in ('md5', 'sha1'):
            raise ValueError("Invalid checksum type: '%s'" % checksum_type)
        kwargs.setdefault('params', {})[checksum_type] = checksum
        return self._search('checksum/', **kwargs)

    def pattern_search(self, pattern, **kwargs):
        kwargs.setdefault('params', {})['pattern'] = pattern
        return self._search('pattern/', **kwargs)

    @repos
    def version_search(self, group=None, artifact=None, **kwargs):
        params = {}
        if group:
            params['g'] = group
        if artifact:
            params['a'] = artifact

        if not params:
            raise ValueError("Must specify at least one of: group, artifact")

        if 'version' in kwargs:
            params['version'] = kwargs.pop('version')

        if 'remote' in kwargs:
            params['remote'] = str(1 if kwargs.pop('remote') else 0)

        kwargs.setdefault('params', {}).update(params)
        return self._search('versions/', **kwargs)

    def retrieve_artifact(self, path, stream=False):
        url = path.replace(':', '', 1)
        res = self._request('GET', url, stream=stream, return_json=False)
        if res.status_code == requests.codes.ok:
            if stream:
                return res.raw
            return res.text.encode('utf-8')

    @property
    def repositories(self):
        return self._get('repositories/')

    def walk(self, repo, top=None, topdown=True, ignorehidden=True):
        base_uri = 'storage/' + repo

        # normalize path argument
        if top is None:
            top = '/'
        elif not top.startswith('/'):
            top = '/' + top

        parent = self._get(base_uri + top)
        folders, artifacts = [], []
        for child in parent['children']:
            if child['uri'].startswith('/.') and ignorehidden:
                continue

            if top == '/':
                uri = child['uri']
            else:
                uri = parent['path'] + child['uri']

            child_obj = self._get(base_uri + uri)
            if child['folder']:
                folders.append(child_obj)
            else:
                artifacts.append(child_obj)

        if topdown:
            yield parent, folders, artifacts

        for folder in folders:
            for x in self.walk(repo, folder['path'], topdown):
                yield x

        if not topdown:
            yield parent, folders, artifacts
<commit_msg>Use query paramaters as part of the cache key<commit_after>#
# Copyright (c) SAS Institute Inc
#

from urlparse import urljoin
import hashlib
import json
import logging
import os

from conary.lib import util

import requests

from .pompackage import PomPackage

__all__ = ('Client', 'PomPackage')

log = logging.getLogger(__name__)

# Turn down the logging level on requeests
requests_logger = logging.getLogger('requests.packages.urllib3.connectionpool')
requests_logger.setLevel(logging.WARN)

def detail(func):
    def wrapper(*args, **kwargs):
        info = kwargs.pop('info', False)
        properties = kwargs.pop('properties', False)

        detail = []
        if info:
            detail.append('info')
        elif properties:
            detail.append('properties')
        if detail:
            kwargs.setdefault(
                'headers', {})['x-result-detail'] = ','.join(detail)
        return func(*args, **kwargs)
    return wrapper

def repos(func):
    def wrapper(*args, **kwargs):
        repos = kwargs.pop('repos', None)
        if repos:
            if isinstance(repos, (str, unicode)):
                repos = [repos]
            kwargs.setdefault('params', {})['repos'] = ','.join(repos)
        return func(*args, **kwargs)
    return wrapper

class Struct(object):
    def __init__(self):
        self.status_code = None
        self.text = None

    def json(self):
        return json.loads(self.text)

class Cache(object):
    def __init__(self, cacheDir='/tmp/artifactory_cache'):
        self.cacheDir = cacheDir
        util.mkdirChain(self.cacheDir)

    def cache(self, method, uri, query_params=None, res=None):
        if query_params is None:
            query_params = {}

        key = method + ' ' + uri
        if query_params:
            key += '?' + '&'.join(('%s=%s' % kv for kv in query_params.iteritems()))

        h = hashlib.sha1(key).hexdigest()
        dh = '%s.data' % h

        hpath = os.path.join(self.cacheDir, h)
        dhpath = os.path.join(self.cacheDir, dh)

        if os.path.exists(hpath) and os.path.exists(dhpath) and not res:
            log.debug('hit %s with key %s', key, h)
            data = json.load(open(dhpath))
            data['text'] = open(hpath).read().decode('utf-8')
            res = Struct()
            for k, v in data.iteritems():
                setattr(res, k, v)

        elif res is not None:
            log.debug("Caching response %s with key %s", key, h)
            with open(hpath, 'wb') as fh:
                fh.write(res.text.encode('utf-8'))
            with open(dhpath, 'wb') as fh:
                json.dump({
                    'status_code': res.status_code,
                    'method': method,
                    'uri': uri,
                    'param': query_params,
                }, fh)

        return res

class Client(object):
    """
    Client for talking to artfactory api
    """

    def __init__(self, cfg, headers=None):
        """
        Create a client from a updatebot config object.

        If there are default headers you want used for every request, pass in
        a dictionary:
            Client("http://example.com,
                headers={'my-default-header': 'foo'})

        :param UpdateBotConfig cfg: cfg object to use for configuring the client
        :param dict headers: dictionary of headers to use for all requests
        """
        # append a / to url if it doesn't end with one
        self._cfg = cfg
        self._url = cfg.repositoryUrl + '/'
        self._api_url = urljoin(self._url, "api/")

        self._cache = Cache()

    def _get(self, uri, **kwargs):
        res = self._request('GET', uri, **kwargs)
        return res

    def _post(self, uri,  **kwargs):
        res = self._request('POST', uri, **kwargs)
        return res

    def _head(self, uri, **kwargs):
        res = self._request('HEAD', uri, return_json=False, **kwargs)
        return res

    def _request(self, method, uri, return_json=True, **kwargs):
        url = urljoin(self._url, uri)
        res = self._cache.cache(method, uri, kwargs.get('params'))
        if not res:
            log.debug('requesting %s %s', method, url)
            res = requests.request(method, url, **kwargs)
            self._cache.cache(method, uri, kwargs.get('params'), res)
        if return_json:
            return res.json()
        return res

    def _search(self, path, **kwargs):
        log.debug("search(%s, kwargs=%s)", path, kwargs)
        if 'auth' not in kwargs:
            kwargs['auth'] = self._cfg.artifactoryUser.find(self._cfg.repositoryUrl)

        res = self._get(urljoin("api/search/", path), **kwargs)
        urls = [r['uri'] for r in res.get('results', [])]

        for url in urls:
            res = self._get(url, return_json=False)
            if res.status_code == requests.codes.not_found:
                log.warn('error fetching %s', url)
                continue
            yield res.json()

    @staticmethod
    def constructPath(groupId, artifactId, version=None, artifactName=None,
                      extension='pom', relative=False):
        """Construct a maven 2 path

        :param groupId: group name
        :type groupId: string
        :param artifactId: artifact name
        :type artifactId: string
        :param version: (None) version of artifact
        :type version: string
        :param artifactName: (None) artifact name if not the artifactId
        :type artifactName: string
        :param extension: (pom) file extention of artifact
        :type extension: string
        :param relative: (False) generate a relative path
        :type relative: bool
        """
        path = '/{0}/{1}'
        if relative:
            path = path[1:]

        if version:
            path += '/{2}'

        if artifactName:
            path += '/{3}.{4}'
        else:
            path += '/{1}-{2}.{4}'

        return path.format(
            groupId.replace('.', '/'),
            artifactId,
            version,
            artifactName,
            extension,
            )

    def artifactUrl(self, group, artifact, version, extension='pom'):
        path = self.constructPath(group, artifact, version, extension=extension,
                                  relative=True)
        uri = urljoin('repo/', path)
        return urljoin(self._url, uri)

    def checkJar(self, group, artifact, version):
        path = self.constructPath(group, artifact, version, extension='jar',
                                  relative=True)
        uri = urljoin('repo/', path)
        res = self._head(uri)
        return res.status_code == requests.codes.ok

    @repos
    @detail
    def quick_search(self, name, **kwargs):
        kwargs.setdefault('params', {})['name'] = name
        results = self._search("artifact/", **kwargs)
        return results

    @repos
    def class_search(self, name, **kwargs):
        kwargs.setdefault('params', {})['name'] = name
        return self._get("archive/", **kwargs)

    @detail
    @repos
    def gavc_search(self, group=None, artifact=None, version=None,
                    classifier=None, **kwargs):
        """
        Search by Maven coordinates: GroupId, ArtifactId, Version &
        Classifier. Search must contain at least one argument. Can
        limit search to specific repositories (local or caches).

        You can retrieve extra information or properites by setting
        the `info` or `properties` flags to True

        :param str group: a group id to search for
        :param str artifact: an artifact id to search fo
        :param str version: a version string
        :param str classifier: a classifier string
        :param repos: limit search to these repositories, local or cache
        :type repos: str or list
        :param bool info: add all extra information
        :param bool properties: get the properties of found artifacts
        """
        params = {}
        if group:
            params["g"] = group
        if artifact:
            params["a"] = artifact
        if version:
            params['v'] = version
        if classifier:
            params['c'] = classifier
        if not params:
            raise ValueError("Must provide one of: group, artifact, version,"
                             " or classifier")
        kwargs.setdefault('params', {}).update(params)
        results = self._search("gavc/", **kwargs)
        return results

    @detail
    @repos
    def property_search(self, properties, **kwargs):
        if not isinstance(properties, dict):
            properties = dict(properties)
        kwargs.setdefault('params', {}).update(properties)
        return self._search("prop/", **kwargs)

    @detail
    @repos
    def checksum_search(self, checksum, checksum_type, **kwargs):
        if checksum_type not in ('md5', 'sha1'):
            raise ValueError("Invalid checksum type: '%s'" % checksum_type)
        kwargs.setdefault('params', {})[checksum_type] = checksum
        return self._search('checksum/', **kwargs)

    def pattern_search(self, pattern, **kwargs):
        kwargs.setdefault('params', {})['pattern'] = pattern
        return self._search('pattern/', **kwargs)

    @repos
    def version_search(self, group=None, artifact=None, **kwargs):
        params = {}
        if group:
            params['g'] = group
        if artifact:
            params['a'] = artifact

        if not params:
            raise ValueError("Must specify at least one of: group, artifact")

        if 'version' in kwargs:
            params['version'] = kwargs.pop('version')

        if 'remote' in kwargs:
            params['remote'] = str(1 if kwargs.pop('remote') else 0)

        kwargs.setdefault('params', {}).update(params)
        return self._search('versions/', **kwargs)

    def retrieve_artifact(self, path, stream=False):
        url = path.replace(':', '', 1)
        res = self._request('GET', url, stream=stream, return_json=False)
        if res.status_code == requests.codes.ok:
            if stream:
                return res.raw
            return res.text.encode('utf-8')

    @property
    def repositories(self):
        return self._get('repositories/')

    def walk(self, repo, top=None, topdown=True, ignorehidden=True):
        base_uri = 'storage/' + repo

        # normalize path argument
        if top is None:
            top = '/'
        elif not top.startswith('/'):
            top = '/' + top

        parent = self._get(base_uri + top)
        folders, artifacts = [], []
        for child in parent['children']:
            if child['uri'].startswith('/.') and ignorehidden:
                continue

            if top == '/':
                uri = child['uri']
            else:
                uri = parent['path'] + child['uri']

            child_obj = self._get(base_uri + uri)
            if child['folder']:
                folders.append(child_obj)
            else:
                artifacts.append(child_obj)

        if topdown:
            yield parent, folders, artifacts

        for folder in folders:
            for x in self.walk(repo, folder['path'], topdown):
                yield x

        if not topdown:
            yield parent, folders, artifacts
<|endoftext|>


<commit_before>import os
import sqlite3
from contextlib import closing
from datetime import datetime
from os.path import join, exists

from flask import Flask, request, send_from_directory, g, render_template
from flask import flash
from werkzeug.utils import secure_filename

from src.xmind2testlink import convert_xmind

UPLOAD_FOLDER = './uploads'
ALLOWED_EXTENSIONS = ['xmind']
DEBUG = True
DATABASE = './data.sqlite3'

app = Flask(__name__)
app.config.from_object(__name__)
app.secret_key = os.urandom(32)

def connect_db():
    return sqlite3.connect(app.config['DATABASE'])

def init_db():
    with closing(connect_db()) as db:
        with app.open_resource('schema.sql', mode='r') as f:
            db.cursor().executescript(f.read())
        db.commit()

@app.before_request
def before_request():
    g.db = connect_db()

@app.teardown_request
def teardown_request(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        db.close()

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.route('/', methods=['GET', 'POST'])
def index():
    download_xml = None
    if request.method == 'POST':
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            upload_to = join(app.config['UPLOAD_FOLDER'], filename)

            if exists(upload_to):
                filename = '{}_{}.xmind'.format(filename[:-6], datetime.now().strftime('%Y%m%d_%H%M%S'))
                upload_to = join(app.config['UPLOAD_FOLDER'], filename)

            file.save(upload_to)
            convert_xmind(upload_to)
            download_xml = filename[:-5] + 'xml'
        else:
            flash("{} is not an xmind file!".format(file.filename))

    return render_template('index.html', download_xml=download_xml)

@app.route('/testlink/<filename>')
def download_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True)

@app.route('/uploads/<filename>')
def uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)

if __name__ == '__main__':

    if not exists(UPLOAD_FOLDER):
        os.mkdir(UPLOAD_FOLDER)

    if not exists(DATABASE):
        init_db()

    app.run(debug=True)
<commit_msg>message improved.<commit_after>import os
import sqlite3
from contextlib import closing
from datetime import datetime
from os.path import join, exists

from flask import Flask, request, send_from_directory, g, render_template
from flask import flash
from werkzeug.utils import secure_filename

from src.xmind2testlink import convert_xmind

UPLOAD_FOLDER = './uploads'
ALLOWED_EXTENSIONS = ['xmind']
DEBUG = True
DATABASE = './data.sqlite3'

app = Flask(__name__)
app.config.from_object(__name__)
app.secret_key = os.urandom(32)

def connect_db():
    return sqlite3.connect(app.config['DATABASE'])

def init_db():
    with closing(connect_db()) as db:
        with app.open_resource('schema.sql', mode='r') as f:
            db.cursor().executescript(f.read())
        db.commit()

@app.before_request
def before_request():
    g.db = connect_db()

@app.teardown_request
def teardown_request(exception):
    db = getattr(g, 'db', None)
    if db is not None:
        db.close()

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

@app.route('/', methods=['GET', 'POST'])
def index():
    download_xml = None
    if request.method == 'POST':
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            upload_to = join(app.config['UPLOAD_FOLDER'], filename)

            if exists(upload_to):
                filename = '{}_{}.xmind'.format(filename[:-6], datetime.now().strftime('%Y%m%d_%H%M%S'))
                upload_to = join(app.config['UPLOAD_FOLDER'], filename)

            file.save(upload_to)
            convert_xmind(upload_to)
            download_xml = filename[:-5] + 'xml'
            flash('Success!')
        else:
            flash("<b>{}</b> is not an xmind file!".format(file.filename))

    return render_template('index.html', download_xml=download_xml)

@app.route('/testlink/<filename>')
def download_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename, as_attachment=True)

@app.route('/uploads/<filename>')
def uploaded_file(filename):
    return send_from_directory(app.config['UPLOAD_FOLDER'], filename)

if __name__ == '__main__':

    if not exists(UPLOAD_FOLDER):
        os.mkdir(UPLOAD_FOLDER)

    if not exists(DATABASE):
        init_db()

    app.run(debug=True)
<|endoftext|>