<commit_before># Software License Agreement (BSD License)
#
# Copyright (c) 2009, Willow Garage, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Willow Garage, Inc. nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import traceback
import os
import copy
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse
# choosing multiprocessing over threading for clean Control-C
# interrupts (provides terminate())
from multiprocessing import Process, Manager
from vcstools.vcs_base import VcsError

class MultiProjectException(Exception):
    pass

def samefile(file1, file2):
    """
    Test whether two pathnames reference the same actual file
    This is a workaround for the fact that some platforms
    do not have os.path.samefile (particularly windows). This
    is the patch that was integrated in python 3.0 (at which
    time we can probably remove this workaround).
    """
    try:
        return os.path.samefile(file1, file2)
    except AttributeError:
        try:
            from nt import _getfinalpathname
            return _getfinalpathname(file1) == _getfinalpathname(file2)
        except (NotImplementedError, ImportError):
            # On Windows XP and earlier, two files are the same if their
            #  absolute pathnames are the same.
            # Also, on other operating systems, fake this method with a
            #  Windows-XP approximation.
            return os.path.abspath(file1) == os.path.abspath(file2)

def conditional_abspath(uri):
    """
    @param uri: The uri to check
    @return: abspath(uri) if local path otherwise pass through uri
    """
    uri2 = urlparse(uri)
    # maybe it's a local file?
    if uri2.scheme == '':
        return os.path.abspath(uri)
    else:
        return uri

def is_web_uri(source_uri):
    """
    Uses heuristics to check whether uri is a web uri (as opposed to a file path)
    :param source_uri: string representing web uri or file path
    :returns: bool
    """
    if source_uri is None or source_uri == '':
        return False
    parsed_uri = urlparse(source_uri)
    if (parsed_uri.scheme == '' and
        parsed_uri.netloc == '' and
        not '@' in parsed_uri.path.split('/')[0]):

        return False
    return True

def normalize_uri(source_uri, base_path):
    """
    If source_uri is none or a web uri, return it.
    If source_uri is a relative path, make it an absolute path.
    Else return it normalized
    :param source_uri: some uri to a file, folder, or web resource
    :param base_path: path to use to make relative paths absolute
    :returns: normalized string
    """
    if source_uri is not None and not is_web_uri(source_uri):
        if os.path.isabs(source_uri):
            source_uri = os.path.normpath(source_uri)
        else:
            source_uri2 = os.path.normpath(os.path.join(base_path, source_uri))
            # sys.stderr.write("Warning: Converted relative uri path %s to abspath %s\n" %
            #       (source_uri, source_uri2))
            source_uri = source_uri2
    return source_uri

def string_diff(str1_orig, str2_orig, maxlen=11, backtrack=7):
    """
    Compares strings, returns a part of str2 depending on how many
    chars into the string the first difference can be found. If the
    difference is after maxlen, a prefix of str is removed so that
    only the 'backtrack'-last letters of the common prefix remain in str2.

    This only makes sense if str1 != str2, really.

    The purpose is to print str1 -> str2 without repeating a same long prefix

    :returns: a representation of where str2 differs from str1.
    """
    result = str2_orig or ''
    if str1_orig is not None and str2_orig is not None:
        # we cannot be sure we have strings, might be lists,
        # gracefully fail convert to string
        str1 = str(str1_orig)
        str2 = str(str2_orig)
        result = str2

        if len(str2) > len(str1):
            str1 = str1.ljust(len(str2))
        charcompare = [x[0] == x[1] for x in zip(str(str2), str(str1))]
        if False in charcompare:
            commonprefix = str2[:charcompare.index(False)]
            if len(commonprefix) > maxlen:
                result = "...%s" % str2[len(commonprefix) - backtrack:]
    return result

def normabspath(localname, path):
    """
    if localname is absolute, return it normalized. If relative,
    return normalized join of path and localname
    """
    # do not use realpath here as we want to keep symlinked path as such
    if os.path.isabs(localname) or path is None:
        return os.path.normpath(localname)
    abs_path = os.path.normpath(os.path.join(path, localname))
    return abs_path

def realpath_relation(abspath1, abspath2):
    """
    Computes the relationship abspath1 to abspath2
    :returns: None, 'SAME_AS', 'PARENT_OF', 'CHILD_OF'
    """
    assert os.path.isabs(abspath1), "Bug, %s is not absolute path" % abspath1
    assert os.path.isabs(abspath2), "Bug, %s is not absolute path" % abspath2
    realpath1 = os.path.realpath(abspath1)
    realpath2 = os.path.realpath(abspath2)
    if os.path.dirname(realpath1) == os.path.dirname(realpath2):
        if os.path.basename(realpath1) == os.path.basename(realpath2):
            return 'SAME_AS'
        return None
    else:
        commonprefix = os.path.commonprefix([realpath1, realpath2])
        prefixlen = len(commonprefix)
        # Note: os.path.commonprefix operates on character basis, so
        #       take extra care of situations like '/foo/' and '/foobar/'
        if commonprefix == realpath1 and realpath2[prefixlen] == os.sep:
            return 'PARENT_OF'
        elif commonprefix == realpath2 and realpath1[prefixlen] == os.sep:
            return 'CHILD_OF'
    return None

def select_element(elements, localname):
    """
    selects entry among elements where path or localname matches.
    Prefers localname matches in case of ambiguity.
    """
    path_candidate = None
    if localname is not None:
        realpath = os.path.realpath(localname)
        for element in elements:
            if localname == element.get_local_name():
                path_candidate = element
                break
            elif realpath == os.path.realpath(element.get_path()):
                path_candidate = element
    return path_candidate

def select_elements(config, localnames):
    """
    selects config elements with given localnames, returns in the
    order given in config If localnames has one element which is path
    of the config, return all elements
    """
    if config is None:
        return []
    if localnames is None:
        return config.get_config_elements()
    elements = config.get_config_elements()
    selected = []
    notfound = []
    for localname in localnames:
        element = select_element(elements, localname)
        if element is not None:
            selected.append(element)
        else:
            notfound.append(localname)
    if notfound != []:
        # if we just passed workspace path, return all workspace entries
        if (len(localnames) == 1 and
            os.path.realpath(localnames[0]) == os.path.realpath(config.get_base_path())):

            return config.get_config_elements()
        raise MultiProjectException("Unknown elements '%s'" % notfound)
    result = []
    # select in order and remove duplicates
    for element in config.get_config_elements():
        if element in selected:
            result.append(element)
    return result

## Multithreading The following classes help with distributing work
## over several instances, providing wrapping for starting, joining,
## collecting results, and catching Exceptions. Also they provide
## support for running groups of threads sequentially, for the case
## that some library is not thread-safe.

class WorkerThread(Process):

    def __init__(self, worker, outlist, index):
        Process.__init__(self)
        self.worker = worker
        if worker is None or worker.element is None:
            raise MultiProjectException("Bug: Invalid Worker")
        self.outlist = outlist
        self.index = index

    def run(self):
        result = {}
        try:
            result = {'entry': self.worker.element.get_path_spec()}
            result_dict = self.worker.do_work()
            if result_dict is not None:
                result.update(result_dict)
            else:
                result.update(
                    {'error': MultiProjectException("worker returned None")})
        except MultiProjectException as mpe:
            result.update({'error': mpe})
        except VcsError as vcse:
            result.update({'error': vcse})
        except OSError as ose:
            result.update({'error': ose})
        except Exception as exc:
            # this would be a bug, and we need trace to find them in
            # multithreaded cases.
            traceback.print_exc(file=sys.stderr)
            result.update({'error': exc})
        self.outlist[self.index] = result

class DistributedWork():

    def __init__(self, capacity, num_threads=10, silent=True):
         # need managed array since we need the results later
        man = Manager()
        self.outputs = man.list([None for _ in range(capacity)])
        self.threads = []
        self.sequentializers = {}
        self.index = 0
        self.num_threads = num_threads
        self.silent = silent

    def add_thread(self, worker):
        thread = WorkerThread(worker, self.outputs, self.index)
        if self.index >= len(self.outputs):
            raise MultiProjectException(
                "Bug: Declared capacity exceeded %s >= %s" % (self.index,
                                                              len(self.outputs)))
        self.index += 1
        self.threads.append(thread)

    def run(self):
        """
        Execute all collected workers, terminate all on KeyboardInterrupt
        """
        if self.threads == []:
            return []
        if (self.num_threads == 1):
            for thread in self.threads:
                thread.run()
        else:
            # The following code is rather delicate and may behave differently
            # using threading or multiprocessing. running_threads is
            # intentionally not used as a shrinking list because of al the
            # possible multithreading / interruption corner cases
            # Not using Pool because of KeyboardInterrupt cases
            try:
                waiting_index = 0
                maxthreads = self.num_threads
                running_threads = []
                missing_threads = copy.copy(self.threads)
                # we are done if all threads have finished
                while len(missing_threads) > 0:
                    # we spawn more threads whenever some threads have finished
                    if len(running_threads) < maxthreads:
                        to_index = min(
                            waiting_index + maxthreads - len(running_threads),
                            len(self.threads))
                        for i in range(waiting_index, to_index):
                            self.threads[i].start()
                            running_threads.append(self.threads[i])
                        waiting_index = to_index
                    # threads have exitcode only once they terminated
                    missing_threads = [t for t in missing_threads if t.exitcode is None]
                    running_threads = [t for t in running_threads if t.exitcode is None]
                    if (not self.silent
                        and len(running_threads) > 0):

                        print("[%s] still active" % ",".join([th.worker.element.get_local_name() for th in running_threads]))
                    for thread in running_threads:
                        # this should prevent busy waiting
                        thread.join(1)
            except KeyboardInterrupt as k:
                for thread in self.threads:
                    if thread is not None and thread.is_alive():
                        print("[%s] terminated while active" % thread.worker.element.get_local_name())
                        thread.terminate()
                raise k

        self.outputs = [x for x in self.outputs if x is not None]
        message = ''
        for output in self.outputs:
            if "error" in output:
                if 'entry' in output:
                    message += "Error processing '%s' : %s\n" % (
                        output['entry'].get_local_name(), output["error"])
                else:
                    message += "%s\n" % output["error"]
        if message != '':
            raise MultiProjectException(message)
        return self.outputs
<commit_msg>Fix realpath_relation to consider os.sep and os.altsep.<commit_after># Software License Agreement (BSD License)
#
# Copyright (c) 2009, Willow Garage, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above
#    copyright notice, this list of conditions and the following
#    disclaimer in the documentation and/or other materials provided
#    with the distribution.
#  * Neither the name of Willow Garage, Inc. nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import traceback
import os
import copy
try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse
# choosing multiprocessing over threading for clean Control-C
# interrupts (provides terminate())
from multiprocessing import Process, Manager
from vcstools.vcs_base import VcsError

class MultiProjectException(Exception):
    pass

def samefile(file1, file2):
    """
    Test whether two pathnames reference the same actual file
    This is a workaround for the fact that some platforms
    do not have os.path.samefile (particularly windows). This
    is the patch that was integrated in python 3.0 (at which
    time we can probably remove this workaround).
    """
    try:
        return os.path.samefile(file1, file2)
    except AttributeError:
        try:
            from nt import _getfinalpathname
            return _getfinalpathname(file1) == _getfinalpathname(file2)
        except (NotImplementedError, ImportError):
            # On Windows XP and earlier, two files are the same if their
            #  absolute pathnames are the same.
            # Also, on other operating systems, fake this method with a
            #  Windows-XP approximation.
            return os.path.abspath(file1) == os.path.abspath(file2)

def conditional_abspath(uri):
    """
    @param uri: The uri to check
    @return: abspath(uri) if local path otherwise pass through uri
    """
    uri2 = urlparse(uri)
    # maybe it's a local file?
    if uri2.scheme == '':
        return os.path.abspath(uri)
    else:
        return uri

def is_web_uri(source_uri):
    """
    Uses heuristics to check whether uri is a web uri (as opposed to a file path)
    :param source_uri: string representing web uri or file path
    :returns: bool
    """
    if source_uri is None or source_uri == '':
        return False
    parsed_uri = urlparse(source_uri)
    if (parsed_uri.scheme == '' and
        parsed_uri.netloc == '' and
        not '@' in parsed_uri.path.split('/')[0]):

        return False
    return True

def normalize_uri(source_uri, base_path):
    """
    If source_uri is none or a web uri, return it.
    If source_uri is a relative path, make it an absolute path.
    Else return it normalized
    :param source_uri: some uri to a file, folder, or web resource
    :param base_path: path to use to make relative paths absolute
    :returns: normalized string
    """
    if source_uri is not None and not is_web_uri(source_uri):
        if os.path.isabs(source_uri):
            source_uri = os.path.normpath(source_uri)
        else:
            source_uri2 = os.path.normpath(os.path.join(base_path, source_uri))
            # sys.stderr.write("Warning: Converted relative uri path %s to abspath %s\n" %
            #       (source_uri, source_uri2))
            source_uri = source_uri2
    return source_uri

def string_diff(str1_orig, str2_orig, maxlen=11, backtrack=7):
    """
    Compares strings, returns a part of str2 depending on how many
    chars into the string the first difference can be found. If the
    difference is after maxlen, a prefix of str is removed so that
    only the 'backtrack'-last letters of the common prefix remain in str2.

    This only makes sense if str1 != str2, really.

    The purpose is to print str1 -> str2 without repeating a same long prefix

    :returns: a representation of where str2 differs from str1.
    """
    result = str2_orig or ''
    if str1_orig is not None and str2_orig is not None:
        # we cannot be sure we have strings, might be lists,
        # gracefully fail convert to string
        str1 = str(str1_orig)
        str2 = str(str2_orig)
        result = str2

        if len(str2) > len(str1):
            str1 = str1.ljust(len(str2))
        charcompare = [x[0] == x[1] for x in zip(str(str2), str(str1))]
        if False in charcompare:
            commonprefix = str2[:charcompare.index(False)]
            if len(commonprefix) > maxlen:
                result = "...%s" % str2[len(commonprefix) - backtrack:]
    return result

def normabspath(localname, path):
    """
    if localname is absolute, return it normalized. If relative,
    return normalized join of path and localname
    """
    # do not use realpath here as we want to keep symlinked path as such
    if os.path.isabs(localname) or path is None:
        return os.path.normpath(localname)
    abs_path = os.path.normpath(os.path.join(path, localname))
    return abs_path

def is_path_sep(string):
    """Return true if string is path separator.

    Consider both :obj:`os.sep` and :obj:`os.altsep`.
    """
    return string == os.sep or (os.altsep and string == os.altsep)

def realpath_relation(abspath1, abspath2):
    """
    Computes the relationship abspath1 to abspath2
    :returns: None, 'SAME_AS', 'PARENT_OF', 'CHILD_OF'
    """
    assert os.path.isabs(abspath1), "Bug, %s is not absolute path" % abspath1
    assert os.path.isabs(abspath2), "Bug, %s is not absolute path" % abspath2
    realpath1 = os.path.realpath(abspath1)
    realpath2 = os.path.realpath(abspath2)
    if os.path.dirname(realpath1) == os.path.dirname(realpath2):
        if os.path.basename(realpath1) == os.path.basename(realpath2):
            return 'SAME_AS'
        return None
    else:
        commonprefix = os.path.commonprefix([realpath1, realpath2])
        prefixlen = len(commonprefix)
        # Note: os.path.commonprefix operates on character basis, so
        #       take extra care of situations like '/foo/' and '/foobar/'
        if commonprefix == realpath1 and is_path_sep(realpath2[prefixlen]):
            return 'PARENT_OF'
        elif commonprefix == realpath2 and is_path_sep(realpath1[prefixlen]):
            return 'CHILD_OF'
    return None

def select_element(elements, localname):
    """
    selects entry among elements where path or localname matches.
    Prefers localname matches in case of ambiguity.
    """
    path_candidate = None
    if localname is not None:
        realpath = os.path.realpath(localname)
        for element in elements:
            if localname == element.get_local_name():
                path_candidate = element
                break
            elif realpath == os.path.realpath(element.get_path()):
                path_candidate = element
    return path_candidate

def select_elements(config, localnames):
    """
    selects config elements with given localnames, returns in the
    order given in config If localnames has one element which is path
    of the config, return all elements
    """
    if config is None:
        return []
    if localnames is None:
        return config.get_config_elements()
    elements = config.get_config_elements()
    selected = []
    notfound = []
    for localname in localnames:
        element = select_element(elements, localname)
        if element is not None:
            selected.append(element)
        else:
            notfound.append(localname)
    if notfound != []:
        # if we just passed workspace path, return all workspace entries
        if (len(localnames) == 1 and
            os.path.realpath(localnames[0]) == os.path.realpath(config.get_base_path())):

            return config.get_config_elements()
        raise MultiProjectException("Unknown elements '%s'" % notfound)
    result = []
    # select in order and remove duplicates
    for element in config.get_config_elements():
        if element in selected:
            result.append(element)
    return result

## Multithreading The following classes help with distributing work
## over several instances, providing wrapping for starting, joining,
## collecting results, and catching Exceptions. Also they provide
## support for running groups of threads sequentially, for the case
## that some library is not thread-safe.

class WorkerThread(Process):

    def __init__(self, worker, outlist, index):
        Process.__init__(self)
        self.worker = worker
        if worker is None or worker.element is None:
            raise MultiProjectException("Bug: Invalid Worker")
        self.outlist = outlist
        self.index = index

    def run(self):
        result = {}
        try:
            result = {'entry': self.worker.element.get_path_spec()}
            result_dict = self.worker.do_work()
            if result_dict is not None:
                result.update(result_dict)
            else:
                result.update(
                    {'error': MultiProjectException("worker returned None")})
        except MultiProjectException as mpe:
            result.update({'error': mpe})
        except VcsError as vcse:
            result.update({'error': vcse})
        except OSError as ose:
            result.update({'error': ose})
        except Exception as exc:
            # this would be a bug, and we need trace to find them in
            # multithreaded cases.
            traceback.print_exc(file=sys.stderr)
            result.update({'error': exc})
        self.outlist[self.index] = result

class DistributedWork():

    def __init__(self, capacity, num_threads=10, silent=True):
         # need managed array since we need the results later
        man = Manager()
        self.outputs = man.list([None for _ in range(capacity)])
        self.threads = []
        self.sequentializers = {}
        self.index = 0
        self.num_threads = num_threads
        self.silent = silent

    def add_thread(self, worker):
        thread = WorkerThread(worker, self.outputs, self.index)
        if self.index >= len(self.outputs):
            raise MultiProjectException(
                "Bug: Declared capacity exceeded %s >= %s" % (self.index,
                                                              len(self.outputs)))
        self.index += 1
        self.threads.append(thread)

    def run(self):
        """
        Execute all collected workers, terminate all on KeyboardInterrupt
        """
        if self.threads == []:
            return []
        if (self.num_threads == 1):
            for thread in self.threads:
                thread.run()
        else:
            # The following code is rather delicate and may behave differently
            # using threading or multiprocessing. running_threads is
            # intentionally not used as a shrinking list because of al the
            # possible multithreading / interruption corner cases
            # Not using Pool because of KeyboardInterrupt cases
            try:
                waiting_index = 0
                maxthreads = self.num_threads
                running_threads = []
                missing_threads = copy.copy(self.threads)
                # we are done if all threads have finished
                while len(missing_threads) > 0:
                    # we spawn more threads whenever some threads have finished
                    if len(running_threads) < maxthreads:
                        to_index = min(
                            waiting_index + maxthreads - len(running_threads),
                            len(self.threads))
                        for i in range(waiting_index, to_index):
                            self.threads[i].start()
                            running_threads.append(self.threads[i])
                        waiting_index = to_index
                    # threads have exitcode only once they terminated
                    missing_threads = [t for t in missing_threads if t.exitcode is None]
                    running_threads = [t for t in running_threads if t.exitcode is None]
                    if (not self.silent
                        and len(running_threads) > 0):

                        print("[%s] still active" % ",".join([th.worker.element.get_local_name() for th in running_threads]))
                    for thread in running_threads:
                        # this should prevent busy waiting
                        thread.join(1)
            except KeyboardInterrupt as k:
                for thread in self.threads:
                    if thread is not None and thread.is_alive():
                        print("[%s] terminated while active" % thread.worker.element.get_local_name())
                        thread.terminate()
                raise k

        self.outputs = [x for x in self.outputs if x is not None]
        message = ''
        for output in self.outputs:
            if "error" in output:
                if 'entry' in output:
                    message += "Error processing '%s' : %s\n" % (
                        output['entry'].get_local_name(), output["error"])
                else:
                    message += "%s\n" % output["error"]
        if message != '':
            raise MultiProjectException(message)
        return self.outputs
<|endoftext|>


<commit_before># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate docs for the TensorFlow Python API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import inspect
import os
import sys

import six
import tensorflow as tf

from tensorflow.python import debug as tf_debug

from tensorflow.tools.common import public_api
from tensorflow.tools.common import traverse
from tensorflow.tools.docs import doc_generator_visitor
from tensorflow.tools.docs import parser
from tensorflow.tools.docs import py_guide_parser

def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
               reverse_index, doc_index, guide_index):
  """Write previously extracted docs to disk.

  Write a docs page for each symbol in `index` to a tree of docs at
  `output_dir`.

  Symbols with multiple aliases will have only one page written about them,
  which is referenced for all aliases. `duplicate_of` and `duplicates` are used
  to determine which docs pages to write.

  Args:
    output_dir: Directory to write documentation markdown files to. Will be
      created if it doesn't exist.
    base_dir: Base directory of the code being documented. This prefix is
      stripped from all file paths that are part of the documentation.
    duplicate_of: A `dict` mapping fully qualified names to "master" names. This
      is used to resolve "@{symbol}" references to the "master" name.
    duplicates: A `dict` mapping fully qualified names to a set of all
      aliases of this name. This is used to automatically generate a list of all
      aliases for each name.
    index: A `dict` mapping fully qualified names to the corresponding Python
      objects. Used to produce docs for child objects, and to check the validity
      of "@{symbol}" references.
    tree: A `dict` mapping a fully qualified name to the names of all its
      members. Used to populate the members section of a class or module page.
    reverse_index: A `dict` mapping object ids to fully qualified names.
    doc_index: A `dict` mapping a doc key to a DocInfo.
    guide_index: A `dict` mapping symbol name strings to GuideRef.
  """
  # Make output_dir.
  try:
    if not os.path.exists(output_dir):
      os.makedirs(output_dir)
  except OSError as e:
    print('Creating output dir "%s" failed: %s' % (output_dir, e))
    raise

  # Parse and write Markdown pages, resolving cross-links (@{symbol}).
  for full_name, py_object in six.iteritems(index):

    if full_name in duplicate_of:
      continue

    # Methods and some routines are documented only as part of their class.
    if not (inspect.ismodule(py_object) or
            inspect.isclass(py_object) or
            inspect.isfunction(py_object)):
      continue

    print('Writing docs for %s (%r).' % (full_name, py_object))

    # Generate docs for `py_object`, resolving references.
    markdown = parser.generate_markdown(full_name, py_object,
                                        duplicate_of=duplicate_of,
                                        duplicates=duplicates,
                                        index=index,
                                        tree=tree,
                                        reverse_index=reverse_index,
                                        doc_index=doc_index,
                                        guide_index=guide_index,
                                        base_dir=base_dir)

    # TODO(deannarubin): use _tree to generate sidebar information.

    path = os.path.join(output_dir, parser.documentation_path(full_name))
    directory = os.path.dirname(path)
    try:
      if not os.path.exists(directory):
        os.makedirs(directory)
      with open(path, 'w') as f:
        f.write(markdown)
    except OSError as e:
      print('Cannot write documentation for %s to %s: %s' % (full_name,
                                                             directory, e))
      raise
    # TODO(deannarubin): write sidebar file?

  # Write a global index containing all full names with links.
  with open(os.path.join(output_dir, 'index.md'), 'w') as f:
    f.write(parser.generate_global_index('TensorFlow', index, duplicate_of))

def extract():
  """Extract docs from tf namespace and write them to disk."""
  visitor = doc_generator_visitor.DocGeneratorVisitor('tf')
  api_visitor = public_api.PublicAPIVisitor(visitor)

  # Access something in contrib so tf.contrib is properly loaded (it's hidden
  # behind lazy loading)
  _ = tf.contrib.__name__

  # Exclude some libaries in contrib from the documentation altogether.
  # TODO(wicke): Shrink this list.
  api_visitor.do_not_descend_map.update({
      'contrib': [
          'compiler',
          'factorization',
          'grid_rnn',
          'labeled_tensor',
          'ndlstm',
          'quantization',
          'session_bundle',
          'slim',
          'solvers',
          'specs',
          'tensor_forest',
          'tensorboard',
          'testing',
          'tfprof',
      ],
      'contrib.bayesflow': [
          'entropy', 'monte_carlo',
          'special_math', 'stochastic_gradient_estimators',
          'stochastic_graph', 'stochastic_tensor',
          'stochastic_variables', 'variational_inference'
      ],
      'contrib.distributions': ['bijector'],
      'contrib.ffmpeg': ['ffmpeg_ops'],
      'contrib.graph_editor': [
          'edit',
          'match',
          'subgraph',
          'transform',
          'select',
          'util'
      ],
      'contrib.layers': [
          'feature_column',
          'summaries'
      ],
      'contrib.learn': [
          'datasets',
          'head',
          'graph_actions',
          'io',
          'models',
          'monitors',
          'ops',
          'preprocessing',
          'utils',
      ],
      'contrib.util': ['loader'],
  })

  traverse.traverse(tf, api_visitor)

  # tf_debug is not imported with tf, it's a separate module altogether
  visitor.set_root_name('tfdbg')
  traverse.traverse(tf_debug, api_visitor)

  return visitor

class GetMarkdownTitle(py_guide_parser.PyGuideParser):
  """Extract the title from a .md file."""

  def __init__(self):
    self.title = None
    py_guide_parser.PyGuideParser.__init__(self)

  def process_title(self, _, title):
    if self.title is None:  # only use the first title
      self.title = title

class DocInfo(object):
  """A simple struct for holding a doc's url and title."""

  def __init__(self, url, title):
    self.url = url
    self.title = title

def build_doc_index(src_dir):
  """Build an index from a keyword designating a doc to DocInfo objects."""
  doc_index = {}
  for dirpath, _, filenames in os.walk(src_dir):
    suffix = os.path.relpath(path=dirpath, start=src_dir)
    for base_name in filenames:
      if not base_name.endswith('.md'): continue
      title_parser = GetMarkdownTitle()
      title_parser.process(os.path.join(dirpath, base_name))
      key_parts = os.path.join(suffix, base_name[:-3]).split('/')
      if key_parts[-1] == 'index':
        key_parts = key_parts[:-1]
      doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title)
      doc_index[key_parts[-1]] = doc_info
      if len(key_parts) > 1:
        doc_index['/'.join(key_parts[-2:])] = doc_info

  return doc_index

class GuideRef(object):

  def __init__(self, base_name, title, section_title, section_tag):
    self.url = 'api_guides/python/' + (
        ('%s#%s' % (base_name, section_tag)) if section_tag else base_name)
    self.link_text = (('%s > %s' % (title, section_title))
                      if section_title else title)

  def make_md_link(self, url_prefix):
    return '[%s](%s%s)' % (self.link_text, url_prefix, self.url)

class GenerateGuideIndex(py_guide_parser.PyGuideParser):
  """Turn guide files into an index from symbol name to a list of GuideRefs."""

  def __init__(self):
    self.index = {}
    py_guide_parser.PyGuideParser.__init__(self)

  def process(self, full_path, base_name):
    """Index a file, reading from `full_path`, with `base_name` as the link."""
    self.full_path = full_path
    self.base_name = base_name
    self.title = None
    self.section_title = None
    self.section_tag = None
    py_guide_parser.PyGuideParser.process(self, full_path)

  def process_title(self, _, title):
    if self.title is None:  # only use the first title
      self.title = title

  def process_section(self, _, section_title, tag):
    self.section_title = section_title
    self.section_tag = tag

  def process_line(self, _, line):
    """Index @{symbol} references as in the current file & section."""
    for match in parser.SYMBOL_REFERENCE_RE.finditer(line):
      val = self.index.get(match.group(1), [])
      val.append(GuideRef(
          self.base_name, self.title, self.section_title, self.section_tag))
      self.index[match.group(1)] = val

def build_guide_index(guide_src_dir):
  """Return dict: symbol name -> GuideRef from the files in `guide_src_dir`."""
  index_generator = GenerateGuideIndex()
  for full_path, base_name in py_guide_parser.md_files_in_dir(guide_src_dir):
    index_generator.process(full_path, base_name)
  return index_generator.index

def write(output_dir, base_dir, doc_index, guide_index, visitor):
  """Write documentation for an index in a `DocGeneratorVisitor` to disk.

  This function will create `output_dir` if it doesn't exist, and write
  the documentation contained in `visitor`.

  Args:
    output_dir: The directory to write documentation to. Must not exist.
    base_dir: The base dir of the library `visitor` has traversed. This is used
      to compute relative paths for file references.
    doc_index: A `dict` mapping a doc key to a DocInfo.
    guide_index: A `dict` mapping symbol name strings to GuideRef.
    visitor: A `DocGeneratorVisitor` that has traversed a library located at
      `base_dir`.
  """
  write_docs(output_dir, os.path.abspath(base_dir),
             visitor.duplicate_of, visitor.duplicates,
             visitor.index, visitor.tree, visitor.reverse_index,
             doc_index, guide_index)

class UpdateTags(py_guide_parser.PyGuideParser):
  """Rewrites a Python guide so that each section has an explicit tag."""

  def process_section(self, line_number, section_title, tag):
    self.replace_line(line_number, '
%s
' % (tag, section_title))

def other_docs(src_dir, output_dir, visitor, doc_index):
  """Convert all the files in `src_dir` and write results to `output_dir`."""
  header = '\n'

  # Iterate through all the source files and process them.
  tag_updater = UpdateTags()
  for dirpath, _, filenames in os.walk(src_dir):
    # How to get from `dirpath` to api_docs/python/
    relative_path_to_root = os.path.relpath(
        path=os.path.join(src_dir, 'api_docs/python'), start=dirpath)

    # Make the directory under output_dir.
    new_dir = os.path.join(output_dir,
                           os.path.relpath(path=dirpath, start=src_dir))
    try:
      if not os.path.exists(new_dir):
        os.makedirs(new_dir)
    except OSError as e:
      print('Creating output dir "%s" failed: %s' % (new_dir, e))
      raise

    for base_name in filenames:
      full_in_path = os.path.join(dirpath, base_name)
      suffix = os.path.relpath(path=full_in_path, start=src_dir)
      full_out_path = os.path.join(output_dir, suffix)
      if not base_name.endswith('.md'):
        print('Copying non-md file %s...' % suffix)
        open(full_out_path, 'w').write(open(full_in_path).read())
        continue
      if dirpath.endswith('/api_guides/python'):
        print('Processing Python guide %s...' % base_name)
        md_string = tag_updater.process(full_in_path)
      else:
        print('Processing doc %s...' % suffix)
        md_string = open(full_in_path).read()

      output = parser.replace_references(
          md_string, relative_path_to_root, visitor.duplicate_of,
          doc_index=doc_index, index=visitor.index)
      with open(full_out_path, 'w') as f:
        f.write(header + output)

  print('Done.')

def _main(src_dir, output_dir, base_dir):
  doc_index = build_doc_index(src_dir)
  visitor = extract()
  write(os.path.join(output_dir, 'api_docs/python'), base_dir,
        doc_index,
        build_guide_index(os.path.join(src_dir, 'api_guides/python')),
        visitor)
  other_docs(src_dir, output_dir, visitor, doc_index)

if __name__ == '__main__':
  argument_parser = argparse.ArgumentParser()
  argument_parser.add_argument(
      '--output_dir',
      type=str,
      default=None,
      required=True,
      help='Directory to write docs to.'
  )

  argument_parser.add_argument(
      '--src_dir',
      type=str,
      default=None,
      required=True,
      help='Directory with the source docs.'
  )

  # This doc generator works on the TensorFlow codebase. Since this script lives
  # at tensorflow/tools/docs, and all code is defined somewhere inside
  # tensorflow/, we can compute the base directory (two levels up), which is
  # valid unless we're trying to apply this to a different code base, or are
  # moving the script around.
  script_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
  default_base_dir = os.path.join(script_dir, '..', '..')

  argument_parser.add_argument(
      '--base_dir',
      type=str,
      default=default_base_dir,
      help=('Base directory to to strip from file names referenced in docs. '
            'Defaults to two directories up from the location of this file.')
  )

  flags, _ = argument_parser.parse_known_args()
  _main(flags.src_dir, flags.output_dir, flags.base_dir)
  if parser.all_errors:
    print('Errors during processing:' + '\n  '.join(parser.all_errors))
    sys.exit(1)
<commit_msg>Generate table of contents .yaml. Change: 147539687<commit_after># Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Generate docs for the TensorFlow Python API."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import inspect
import os
import sys

import six
import tensorflow as tf

from tensorflow.python import debug as tf_debug

from tensorflow.tools.common import public_api
from tensorflow.tools.common import traverse
from tensorflow.tools.docs import doc_generator_visitor
from tensorflow.tools.docs import parser
from tensorflow.tools.docs import py_guide_parser

def write_docs(output_dir, base_dir, duplicate_of, duplicates, index, tree,
               reverse_index, doc_index, guide_index):
  """Write previously extracted docs to disk.

  Write a docs page for each symbol in `index` to a tree of docs at
  `output_dir`.

  Symbols with multiple aliases will have only one page written about them,
  which is referenced for all aliases. `duplicate_of` and `duplicates` are used
  to determine which docs pages to write.

  Args:
    output_dir: Directory to write documentation markdown files to. Will be
      created if it doesn't exist.
    base_dir: Base directory of the code being documented. This prefix is
      stripped from all file paths that are part of the documentation.
    duplicate_of: A `dict` mapping fully qualified names to "master" names. This
      is used to resolve "@{symbol}" references to the "master" name.
    duplicates: A `dict` mapping fully qualified names to a set of all
      aliases of this name. This is used to automatically generate a list of all
      aliases for each name.
    index: A `dict` mapping fully qualified names to the corresponding Python
      objects. Used to produce docs for child objects, and to check the validity
      of "@{symbol}" references.
    tree: A `dict` mapping a fully qualified name to the names of all its
      members. Used to populate the members section of a class or module page.
    reverse_index: A `dict` mapping object ids to fully qualified names.
    doc_index: A `dict` mapping a doc key to a DocInfo.
    guide_index: A `dict` mapping symbol name strings to GuideRef.
  """
  # Make output_dir.
  try:
    if not os.path.exists(output_dir):
      os.makedirs(output_dir)
  except OSError as e:
    print('Creating output dir "%s" failed: %s' % (output_dir, e))
    raise

  # These dictionaries are used for table-of-contents generation below
  # They will contain, after the for-loop below::
  #  - module name(string):classes and functions the module contains(list)
  module_children = {}
  #  - symbol name(string):pathname (string)
  symbol_to_file = {}

  # Parse and write Markdown pages, resolving cross-links (@{symbol}).
  for full_name, py_object in six.iteritems(index):

    if full_name in duplicate_of:
      continue

    # Methods and some routines are documented only as part of their class.
    if not (inspect.ismodule(py_object) or
            inspect.isclass(py_object) or
            inspect.isfunction(py_object)):
      continue

    sitepath = os.path.join('api_docs/python',
                            parser.documentation_path(full_name)[:-3])

    # For TOC, we need to store a mapping from full_name to the file
    # we're generating
    symbol_to_file[full_name] = sitepath

    # For a module, remember the module for the table-of-contents
    if inspect.ismodule(py_object):
      if full_name in tree:
        module_children.setdefault(full_name, [])

    # For something else that's documented,
    # figure out what module it lives in
    else:
      subname = str(full_name)
      while True:
        subname = subname[:subname.rindex('.')]
        if inspect.ismodule(index[subname]):
          module_children.setdefault(subname, []).append(full_name)
          break

    print('Writing docs for %s (%r).' % (full_name, py_object))

    # Generate docs for `py_object`, resolving references.
    markdown = parser.generate_markdown(full_name, py_object,
                                        duplicate_of=duplicate_of,
                                        duplicates=duplicates,
                                        index=index,
                                        tree=tree,
                                        reverse_index=reverse_index,
                                        doc_index=doc_index,
                                        guide_index=guide_index,
                                        base_dir=base_dir)

    path = os.path.join(output_dir, parser.documentation_path(full_name))
    directory = os.path.dirname(path)
    try:
      if not os.path.exists(directory):
        os.makedirs(directory)
      with open(path, 'w') as f:
        f.write(markdown)
    except OSError as e:
      print('Cannot write documentation for %s to %s: %s' % (full_name,
                                                             directory, e))
      raise

  # Generate table of contents

  # Put modules in alphabetical order, case-insensitive
  modules = sorted(module_children.keys(), key=lambda a: a.upper())

  leftnav_path = os.path.join(output_dir, '_toc.yaml')
  with open(leftnav_path, 'w') as f:

    # Generate header
    f.write('# Automatically generated file; please do not edit\ntoc:\n')
    for module in modules:
      f.write('  - title: ' + module + '\n'
              '    section:\n' +
              '    - title: Overview\n' +
              '      path: /TARGET_DOC_ROOT/' + symbol_to_file[module] + '\n')

      symbols_in_module = module_children.get(module, [])
      symbols_in_module.sort(key=lambda a: a.upper())

      for full_name in symbols_in_module:
        f.write('    - title: ' + full_name[len(module)+1:] + '\n'
                '      path: /TARGET_DOC_ROOT/' +
                symbol_to_file[full_name] + '\n')

  # Write a global index containing all full names with links.
  with open(os.path.join(output_dir, 'index.md'), 'w') as f:
    f.write(parser.generate_global_index('TensorFlow', index, duplicate_of))

def extract():
  """Extract docs from tf namespace and write them to disk."""
  visitor = doc_generator_visitor.DocGeneratorVisitor('tf')
  api_visitor = public_api.PublicAPIVisitor(visitor)

  # Access something in contrib so tf.contrib is properly loaded (it's hidden
  # behind lazy loading)
  _ = tf.contrib.__name__

  # Exclude some libaries in contrib from the documentation altogether.
  # TODO(wicke): Shrink this list.
  api_visitor.do_not_descend_map.update({
      'contrib': [
          'compiler',
          'factorization',
          'grid_rnn',
          'labeled_tensor',
          'ndlstm',
          'quantization',
          'session_bundle',
          'slim',
          'solvers',
          'specs',
          'tensor_forest',
          'tensorboard',
          'testing',
          'tfprof',
      ],
      'contrib.bayesflow': [
          'entropy', 'monte_carlo',
          'special_math', 'stochastic_gradient_estimators',
          'stochastic_graph', 'stochastic_tensor',
          'stochastic_variables', 'variational_inference'
      ],
      'contrib.distributions': ['bijector'],
      'contrib.ffmpeg': ['ffmpeg_ops'],
      'contrib.graph_editor': [
          'edit',
          'match',
          'subgraph',
          'transform',
          'select',
          'util'
      ],
      'contrib.layers': [
          'feature_column',
          'summaries'
      ],
      'contrib.learn': [
          'datasets',
          'head',
          'graph_actions',
          'io',
          'models',
          'monitors',
          'ops',
          'preprocessing',
          'utils',
      ],
      'contrib.util': ['loader'],
  })

  traverse.traverse(tf, api_visitor)

  # tf_debug is not imported with tf, it's a separate module altogether
  visitor.set_root_name('tfdbg')
  traverse.traverse(tf_debug, api_visitor)

  return visitor

class GetMarkdownTitle(py_guide_parser.PyGuideParser):
  """Extract the title from a .md file."""

  def __init__(self):
    self.title = None
    py_guide_parser.PyGuideParser.__init__(self)

  def process_title(self, _, title):
    if self.title is None:  # only use the first title
      self.title = title

class DocInfo(object):
  """A simple struct for holding a doc's url and title."""

  def __init__(self, url, title):
    self.url = url
    self.title = title

def build_doc_index(src_dir):
  """Build an index from a keyword designating a doc to DocInfo objects."""
  doc_index = {}
  for dirpath, _, filenames in os.walk(src_dir):
    suffix = os.path.relpath(path=dirpath, start=src_dir)
    for base_name in filenames:
      if not base_name.endswith('.md'): continue
      title_parser = GetMarkdownTitle()
      title_parser.process(os.path.join(dirpath, base_name))
      key_parts = os.path.join(suffix, base_name[:-3]).split('/')
      if key_parts[-1] == 'index':
        key_parts = key_parts[:-1]
      doc_info = DocInfo(os.path.join(suffix, base_name), title_parser.title)
      doc_index[key_parts[-1]] = doc_info
      if len(key_parts) > 1:
        doc_index['/'.join(key_parts[-2:])] = doc_info

  return doc_index

class GuideRef(object):

  def __init__(self, base_name, title, section_title, section_tag):
    self.url = 'api_guides/python/' + (
        ('%s#%s' % (base_name, section_tag)) if section_tag else base_name)
    self.link_text = (('%s > %s' % (title, section_title))
                      if section_title else title)

  def make_md_link(self, url_prefix):
    return '[%s](%s%s)' % (self.link_text, url_prefix, self.url)

class GenerateGuideIndex(py_guide_parser.PyGuideParser):
  """Turn guide files into an index from symbol name to a list of GuideRefs."""

  def __init__(self):
    self.index = {}
    py_guide_parser.PyGuideParser.__init__(self)

  def process(self, full_path, base_name):
    """Index a file, reading from `full_path`, with `base_name` as the link."""
    self.full_path = full_path
    self.base_name = base_name
    self.title = None
    self.section_title = None
    self.section_tag = None
    py_guide_parser.PyGuideParser.process(self, full_path)

  def process_title(self, _, title):
    if self.title is None:  # only use the first title
      self.title = title

  def process_section(self, _, section_title, tag):
    self.section_title = section_title
    self.section_tag = tag

  def process_line(self, _, line):
    """Index @{symbol} references as in the current file & section."""
    for match in parser.SYMBOL_REFERENCE_RE.finditer(line):
      val = self.index.get(match.group(1), [])
      val.append(GuideRef(
          self.base_name, self.title, self.section_title, self.section_tag))
      self.index[match.group(1)] = val

def build_guide_index(guide_src_dir):
  """Return dict: symbol name -> GuideRef from the files in `guide_src_dir`."""
  index_generator = GenerateGuideIndex()
  for full_path, base_name in py_guide_parser.md_files_in_dir(guide_src_dir):
    index_generator.process(full_path, base_name)
  return index_generator.index

def write(output_dir, base_dir, doc_index, guide_index, visitor):
  """Write documentation for an index in a `DocGeneratorVisitor` to disk.

  This function will create `output_dir` if it doesn't exist, and write
  the documentation contained in `visitor`.

  Args:
    output_dir: The directory to write documentation to. Must not exist.
    base_dir: The base dir of the library `visitor` has traversed. This is used
      to compute relative paths for file references.
    doc_index: A `dict` mapping a doc key to a DocInfo.
    guide_index: A `dict` mapping symbol name strings to GuideRef.
    visitor: A `DocGeneratorVisitor` that has traversed a library located at
      `base_dir`.
  """
  write_docs(output_dir, os.path.abspath(base_dir),
             visitor.duplicate_of, visitor.duplicates,
             visitor.index, visitor.tree, visitor.reverse_index,
             doc_index, guide_index)

class UpdateTags(py_guide_parser.PyGuideParser):
  """Rewrites a Python guide so that each section has an explicit tag."""

  def process_section(self, line_number, section_title, tag):
    self.replace_line(line_number, '
%s
' % (tag, section_title))

def other_docs(src_dir, output_dir, visitor, doc_index):
  """Convert all the files in `src_dir` and write results to `output_dir`."""
  header = '\n'

  # Iterate through all the source files and process them.
  tag_updater = UpdateTags()
  for dirpath, _, filenames in os.walk(src_dir):
    # How to get from `dirpath` to api_docs/python/
    relative_path_to_root = os.path.relpath(
        path=os.path.join(src_dir, 'api_docs/python'), start=dirpath)

    # Make the directory under output_dir.
    new_dir = os.path.join(output_dir,
                           os.path.relpath(path=dirpath, start=src_dir))
    try:
      if not os.path.exists(new_dir):
        os.makedirs(new_dir)
    except OSError as e:
      print('Creating output dir "%s" failed: %s' % (new_dir, e))
      raise

    for base_name in filenames:
      full_in_path = os.path.join(dirpath, base_name)
      suffix = os.path.relpath(path=full_in_path, start=src_dir)
      full_out_path = os.path.join(output_dir, suffix)
      if not base_name.endswith('.md'):
        print('Copying non-md file %s...' % suffix)
        open(full_out_path, 'w').write(open(full_in_path).read())
        continue
      if dirpath.endswith('/api_guides/python'):
        print('Processing Python guide %s...' % base_name)
        md_string = tag_updater.process(full_in_path)
      else:
        print('Processing doc %s...' % suffix)
        md_string = open(full_in_path).read()

      output = parser.replace_references(
          md_string, relative_path_to_root, visitor.duplicate_of,
          doc_index=doc_index, index=visitor.index)
      with open(full_out_path, 'w') as f:
        f.write(header + output)

  print('Done.')

def _main(src_dir, output_dir, base_dir):
  doc_index = build_doc_index(src_dir)
  visitor = extract()
  write(os.path.join(output_dir, 'api_docs/python'), base_dir,
        doc_index,
        build_guide_index(os.path.join(src_dir, 'api_guides/python')),
        visitor)
  other_docs(src_dir, output_dir, visitor, doc_index)

if __name__ == '__main__':
  argument_parser = argparse.ArgumentParser()
  argument_parser.add_argument(
      '--output_dir',
      type=str,
      default=None,
      required=True,
      help='Directory to write docs to.'
  )

  argument_parser.add_argument(
      '--src_dir',
      type=str,
      default=None,
      required=True,
      help='Directory with the source docs.'
  )

  # This doc generator works on the TensorFlow codebase. Since this script lives
  # at tensorflow/tools/docs, and all code is defined somewhere inside
  # tensorflow/, we can compute the base directory (two levels up), which is
  # valid unless we're trying to apply this to a different code base, or are
  # moving the script around.
  script_dir = os.path.dirname(inspect.getfile(inspect.currentframe()))
  default_base_dir = os.path.join(script_dir, '..', '..')

  argument_parser.add_argument(
      '--base_dir',
      type=str,
      default=default_base_dir,
      help=('Base directory to to strip from file names referenced in docs. '
            'Defaults to two directories up from the location of this file.')
  )

  flags, _ = argument_parser.parse_known_args()
  _main(flags.src_dir, flags.output_dir, flags.base_dir)
  if parser.all_errors:
    print('Errors during processing:' + '\n  '.join(parser.all_errors))
    sys.exit(1)
<|endoftext|>


<commit_before>from fmn.consumer.backends.base import BaseBackend
import fedmsg.meta

import datetime

class DebugBackend(BaseBackend):
    __context_name__ = "debug"

    def __init__(self, *args, **kwargs):
        super(DebugBackend, self).__init__(*args, **kwargs)

    def handle(self, session, recipient, msg, streamline=False):
        topic = msg['topic']
        category = topic.split('.')[3]

        link = fedmsg.meta.msg2link(msg, **self.config) or u''
        content = fedmsg.meta.msg2long_form(msg, **self.config) or u''
        subject = fedmsg.meta.msg2subtitle(msg, **self.config) or u''

        usernames = fedmsg.meta.msg2usernames(msg, **self.config)
        packages = fedmsg.meta.msg2packages(msg, **self.config)

        print '  -- Single message --'
        print 'recipient:   ', recipient
        print 'subject:     ', subject,
        print 'content:     ', content + "\n\t" + link
        print 'topic:       ', [topic]
        print 'category:    ', [category]
        print 'usernames:   ', usernames
        print 'packages:    ', packages

    def handle_batch(self, session, recipient, queued_messages):
        def _format_line(msg):
            timestamp = datetime.datetime.fromtimestamp(msg['timestamp'])
            link = fedmsg.meta.msg2link(msg, **self.config) or u''
            payload = fedmsg.meta.msg2subtitle(msg, **self.config) or u''

            if recipient.get('verbose', True):
                longform = fedmsg.meta.msg2long_form(msg, **self.config) or u''
                if longform:
                    payload += "\n" + longform

            return timestamp.strftime("%c") + ", " + payload + "\n\t" + link

        n = len(queued_messages)
        subject = u"Fedora Notifications Digest (%i updates)" % n
        summary = u"Digest summary:\n"
        for i, msg in enumerate(queued_messages):
            line = fedmsg.meta.msg2subtitle(msg.message, **self.config) or u''
            summary += str(i+1) + ".\t" + line + "\n"

        separator = "\n\n" + "-"*79 + "\n\n"
        if recipient.get('verbose', True):
            content = summary + separator
        else:
            content = u''

        content += separator.join([
            _format_line(queued_message.message)
            for queued_message in queued_messages])

        topics = set([q.message['topic'] for q in queued_messages])
        categories = set([topic.split('.')[3] for topic in topics])

        squash = lambda items: reduce(set.union, items, set())
        usernames = squash([
            fedmsg.meta.msg2usernames(q.message, **self.config)
            for q in queued_messages])
        packages = squash([
            fedmsg.meta.msg2packages(q.message, **self.config)
            for q in queued_messages])

        print '  -- Batch --'
        print 'recipient:   ', recipient
        print 'subject:     ', subject,
        print 'content:     ', content
        print 'topic:       ', topics
        print 'category:    ', categories
        print 'usernames:   ', usernames
        print 'packages:    ', packages

    def handle_confirmation(self, session, confirmation):
        confirmation.set_status(session, 'valid')
        acceptance_url = self.config['fmn.acceptance_url'].format(
            secret=confirmation.secret)
        rejection_url = self.config['fmn.rejection_url'].format(
            secret=confirmation.secret)

        template = self.config.get('fmn.mail_confirmation_template',
                                   CONFIRMATION_TEMPLATE)
        content = template.format(
            acceptance_url=acceptance_url,
            rejection_url=rejection_url,
            support_email=self.config['fmn.support_email'],
            username=confirmation.openid,
        ).strip()

        recipient = {
            'user': confirmation.user.openid,
            'email address': confirmation.detail_value,
            'triggered_by_links': False,
        }

        print '  -- Confirmation --'
        print 'recipient:   ', recipient
        print 'content:     ', content

<commit_msg>Add missing CONFIRMATION_TEMPLATE for DebugBackend<commit_after>from fmn.consumer.backends.base import BaseBackend
import fedmsg.meta

import datetime

CONFIRMATION_TEMPLATE = u"""
{username} has requested that notifications be sent to this email address
* To accept, visit this address:
  {acceptance_url}
* Or, to reject you can visit this address:
  {rejection_url}
Alternatively, you can ignore this.  This is an automated message, please
email {support_email} if you have any concerns/issues/abuse.
"""

class DebugBackend(BaseBackend):
    __context_name__ = "debug"

    def __init__(self, *args, **kwargs):
        super(DebugBackend, self).__init__(*args, **kwargs)

    def handle(self, session, recipient, msg, streamline=False):
        topic = msg['topic']
        category = topic.split('.')[3]

        link = fedmsg.meta.msg2link(msg, **self.config) or u''
        content = fedmsg.meta.msg2long_form(msg, **self.config) or u''
        subject = fedmsg.meta.msg2subtitle(msg, **self.config) or u''

        usernames = fedmsg.meta.msg2usernames(msg, **self.config)
        packages = fedmsg.meta.msg2packages(msg, **self.config)

        print '  -- Single message --'
        print 'recipient:   ', recipient
        print 'subject:     ', subject,
        print 'content:     ', content + "\n\t" + link
        print 'topic:       ', [topic]
        print 'category:    ', [category]
        print 'usernames:   ', usernames
        print 'packages:    ', packages

    def handle_batch(self, session, recipient, queued_messages):
        def _format_line(msg):
            timestamp = datetime.datetime.fromtimestamp(msg['timestamp'])
            link = fedmsg.meta.msg2link(msg, **self.config) or u''
            payload = fedmsg.meta.msg2subtitle(msg, **self.config) or u''

            if recipient.get('verbose', True):
                longform = fedmsg.meta.msg2long_form(msg, **self.config) or u''
                if longform:
                    payload += "\n" + longform

            return timestamp.strftime("%c") + ", " + payload + "\n\t" + link

        n = len(queued_messages)
        subject = u"Fedora Notifications Digest (%i updates)" % n
        summary = u"Digest summary:\n"
        for i, msg in enumerate(queued_messages):
            line = fedmsg.meta.msg2subtitle(msg.message, **self.config) or u''
            summary += str(i+1) + ".\t" + line + "\n"

        separator = "\n\n" + "-"*79 + "\n\n"
        if recipient.get('verbose', True):
            content = summary + separator
        else:
            content = u''

        content += separator.join([
            _format_line(queued_message.message)
            for queued_message in queued_messages])

        topics = set([q.message['topic'] for q in queued_messages])
        categories = set([topic.split('.')[3] for topic in topics])

        squash = lambda items: reduce(set.union, items, set())
        usernames = squash([
            fedmsg.meta.msg2usernames(q.message, **self.config)
            for q in queued_messages])
        packages = squash([
            fedmsg.meta.msg2packages(q.message, **self.config)
            for q in queued_messages])

        print '  -- Batch --'
        print 'recipient:   ', recipient
        print 'subject:     ', subject,
        print 'content:     ', content
        print 'topic:       ', topics
        print 'category:    ', categories
        print 'usernames:   ', usernames
        print 'packages:    ', packages

    def handle_confirmation(self, session, confirmation):
        confirmation.set_status(session, 'valid')
        acceptance_url = self.config['fmn.acceptance_url'].format(
            secret=confirmation.secret)
        rejection_url = self.config['fmn.rejection_url'].format(
            secret=confirmation.secret)

        template = self.config.get('fmn.mail_confirmation_template',
                                   CONFIRMATION_TEMPLATE)
        content = template.format(
            acceptance_url=acceptance_url,
            rejection_url=rejection_url,
            support_email=self.config['fmn.support_email'],
            username=confirmation.openid,
        ).strip()

        recipient = {
            'user': confirmation.user.openid,
            'email address': confirmation.detail_value,
            'triggered_by_links': False,
        }

        print '  -- Confirmation --'
        print 'recipient:   ', recipient
        print 'content:     ', content

<|endoftext|>


<commit_before>'''A convenient class for parsing HTML pages.'''

from __future__ import unicode_literals

from HTMLParser import HTMLParser
import logging
import re

from vkfeed.core import Error

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)

class HTMLPageParser(HTMLParser):
    '''A convenient class for parsing HTML pages.'''

    tag_name_regex = '[a-zA-Z][-.a-zA-Z0-9:_]*'
    '''A regular expression for tag name.'''

    attribute_name_regex = tag_name_regex
    '''A regular expression for attribute name.'''

    tag_attrs_regex = re.sub(r'\s*', '', r'''
        (?:\s+
          ''' + attribute_name_regex + r'''
          (?:\s*=\s*
            (?:
              '[^']*'
              |"[^"]*"
              |[^'"/>\s]+
            )
          )?
        )*
    ''')
    '''A regular expression for tag attributes.'''

    script_regex = re.compile('', re.DOTALL | re.IGNORECASE)
    '''A regular expression for matching scripts.'''

    __invalid_tag_attr_spacing_regex = re.compile(r'''
        (
          # Tag name
          <''' + tag_name_regex + r'''

          # Zero or several attributes
          ''' + tag_attrs_regex + r'''

          # Two attributes without a space between them
          \s+                                # whitespace before attribute name
          ''' + attribute_name_regex + r'''  # attribute name
          \s*=\s*                            # value indicator
          (?:
            '[^']*'                          # LITA-enclosed value
            |"[^"]*"                         # LIT-enclosed value
          )
        )
        ([^\s>])                             # Do not include / to make the preparation replacement for __invalid_tag_attr_regex
    ''', re.VERBOSE)
    '''
    A regular expression for matching a common error in specifying tag
    attributes.
    '''

    __invalid_tag_attr_regex = re.compile(r'''
        (
          # Tag name
          <''' + tag_name_regex + r'''

          # Zero or several attributes
          ''' + tag_attrs_regex + r'''
        )
        \s+(?:
            # Invalid characters instead of an attribute
            [^\sa-zA-Z/>]\S*
            |
            # Sole slash
            /\s
            |
            # Invalid characters starting from slash instead of an attribute
            /[^>\s]+
        )
    ''', re.VERBOSE)
    '''
    A regular expression for matching HTML errors like:

    '''

    __empty_tags = 'area|base|basefont|br|col|frame|hr|img|input|link|meta|param'
    '''A list of all HTML empty tags.'''

    __misopened_tag_regex = re.compile(r'<(' + __empty_tags + tag_attrs_regex + r')\s*>', re.IGNORECASE)
    '''A regular expression for matching opened tags that should be closed.'''

    __tag_stack = None
    '''A stack of currently opened HTML tags.'''

    __cur_data = None
    '''
    Accumulates data between handle_charref(), handle_entityref() and
    handle_data() calls.
    '''

    def __init__(self):
        HTMLParser.__init__(self)

    def handle_charref(self, name):
        '''Handles a character reference of the form &#ref;.'''

        self.__accumulate_data('&#' + name + ';')

    def handle_data(self, data):
        '''Handles data.'''

        self.__accumulate_data(data)

    def handle_endtag(self, tag_name):
        '''Handles end of a tag.'''

        self.__handle_data_if_exists()

        if self.__get_cur_tag()['name'] == tag_name:
            self.__close_tag(self.__tag_stack.pop())
        else:
            for tag_id in xrange(len(self.__tag_stack) - 1, -1, -1):
                if self.__tag_stack[tag_id]['name'] == tag_name:
                    for tag in reversed(self.__tag_stack[tag_id + 1:]):
                        self.__close_tag(tag, forced = True)
                        self.__tag_stack.pop()

                    self.__close_tag(self.__tag_stack.pop())
                    break
            else:
                LOG.debug('Dropping excess end tag "%s"...', tag_name)

    def handle_entityref(self, name):
        '''Handles a general entity reference of the form &name;.'''

        self.__accumulate_data('&' + name + ';')

    def handle_root_data(self, tag, data):
        '''Handles data inside of the root of the document.'''

        LOG.debug('%s', data)

    def handle_root(self, tag, attrs, empty):
        '''Handles a tag inside of the root of the document.'''

        LOG.debug('<%s %s%s>', tag['name'], attrs, '/' if empty else '')
        tag['new_tag_handler'] = self.handle_root
        tag['data_handler'] = self.handle_root_data
        tag['end_tag_handler'] = self.handle_root_end

    def handle_root_end(self, tag):
        '''Handles end of the root of the document.'''

        LOG.debug('</%s>', tag['name'])

    def handle_startendtag(self, tag, attrs):
        '''Handles start of an XHTML-style empty tag.'''

        self.__handle_data_if_exists()
        self.__handle_start_tag(tag, attrs, True)

    def handle_starttag(self, tag, attrs):
        '''Handles start of a tag.'''

        self.__handle_data_if_exists()
        self.__handle_start_tag(tag, attrs, False)

    def reset(self):
        '''Resets the parser.'''

        HTMLParser.reset(self)

        self.__tag_stack = [{
            # Add fake root tag
            'name':              None,
            'new_tag_handler':   self.handle_root,
            'data_handler':      self.handle_root_data,
            'end_tag_handler':   self.handle_root_end,
        }]

    def parse(self, html):
        '''Parses the specified HTML page.'''

        html = self.__fix_html(html)

        self.reset()

        try:
            # Run the parser
            self.feed(html)
            self.close()
        finally:
            # Close all unclosed tags
            for tag in self.__tag_stack[1:]:
                self.__close_tag(tag, True)

    def __accumulate_data(self, data):
        '''
        Accumulates data between handle_charref(), handle_entityref() and
        handle_data() calls.
        '''

        if self.__cur_data is None:
            self.__cur_data = data
        else:
            self.__cur_data += data

    def __close_tag(self, tag, forced = False):
        '''Forces closing of an unclosed tag.'''

        if forced:
            LOG.debug('Force closing of unclosed tag "%s".', tag['name'])
        else:
            LOG.debug('Tag %s closed.', tag)

        if 'end_tag_handler' in tag:
            tag['end_tag_handler'](tag)

        LOG.debug('Current tag: %s.', self.__get_cur_tag())

    def __fix_html(self, html):
        '''Fixes various things that may confuse the Python's HTML parser.'''

        html = self.script_regex.sub('', html)

        loop_replacements = (
            lambda html: self.__invalid_tag_attr_spacing_regex.subn(r'\1 \2', html),
            lambda html: self.__invalid_tag_attr_regex.subn(r'\1 ', html),
        )

        for loop_replacement in loop_replacements:
            for i in xrange(0, 1000):
                html, changed = loop_replacement(html)

                if not changed:
                    break
            else:
                raise Error('Too many errors in the HTML or infinite loop.')

        html = self.__misopened_tag_regex.sub(r'<\1 />', html)

        return html

    def __get_cur_tag(self):
        '''Returns currently opened tag.'''

        return self.__tag_stack[-1]

    def __handle_data_if_exists(self):
        '''Handles accumulated data (if exists).'''

        data = self.__cur_data
        if data is None:
            return

        self.__cur_data = None

        tag = self.__get_cur_tag()
        handler = tag.get('data_handler')

        if handler is not None:
            LOG.debug('Data "%s" in "%s" with handler %s.',
                data, tag['name'], handler.func_name)

            handler(tag, data)

    def __handle_start_tag(self, tag_name, attrs, empty):
        '''Handles start of any tag.'''

        tag = { 'name': tag_name }
        handler = self.__get_cur_tag().get('new_tag_handler')

        if handler is not None:
            attrs = self.__parse_attrs(attrs)

            LOG.debug('Start tag: %s %s with handler %s.',
                tag, attrs, handler.func_name)

            handler(tag, attrs, empty)

        if not empty:
            self.__tag_stack.append(tag)

    def __parse_attrs(self, attrs_tuple):
        '''Converts tag attributes from a tuple to a dictionary.'''

        attrs = {}

        for attr, value in attrs_tuple:
            attrs[attr.lower()] = value

        return attrs
<commit_msg>Update html_parser.py<commit_after>'''A convenient class for parsing HTML pages.'''

from __future__ import unicode_literals

from HTMLParser import HTMLParser
import logging
import re

from RSSvk.core import Error

LOG = logging.getLogger(__name__)
LOG.setLevel(logging.INFO)

class HTMLPageParser(HTMLParser):
    '''A convenient class for parsing HTML pages.'''

    tag_name_regex = '[a-zA-Z][-.a-zA-Z0-9:_]*'
    '''A regular expression for tag name.'''

    attribute_name_regex = tag_name_regex
    '''A regular expression for attribute name.'''

    tag_attrs_regex = re.sub(r'\s*', '', r'''
        (?:\s+
          ''' + attribute_name_regex + r'''
          (?:\s*=\s*
            (?:
              '[^']*'
              |"[^"]*"
              |[^'"/>\s]+
            )
          )?
        )*
    ''')
    '''A regular expression for tag attributes.'''

    script_regex = re.compile('', re.DOTALL | re.IGNORECASE)
    '''A regular expression for matching scripts.'''

    __invalid_tag_attr_spacing_regex = re.compile(r'''
        (
          # Tag name
          <''' + tag_name_regex + r'''

          # Zero or several attributes
          ''' + tag_attrs_regex + r'''

          # Two attributes without a space between them
          \s+                                # whitespace before attribute name
          ''' + attribute_name_regex + r'''  # attribute name
          \s*=\s*                            # value indicator
          (?:
            '[^']*'                          # LITA-enclosed value
            |"[^"]*"                         # LIT-enclosed value
          )
        )
        ([^\s>])                             # Do not include / to make the preparation replacement for __invalid_tag_attr_regex
    ''', re.VERBOSE)
    '''
    A regular expression for matching a common error in specifying tag
    attributes.
    '''

    __invalid_tag_attr_regex = re.compile(r'''
        (
          # Tag name
          <''' + tag_name_regex + r'''

          # Zero or several attributes
          ''' + tag_attrs_regex + r'''
        )
        \s+(?:
            # Invalid characters instead of an attribute
            [^\sa-zA-Z/>]\S*
            |
            # Sole slash
            /\s
            |
            # Invalid characters starting from slash instead of an attribute
            /[^>\s]+
        )
    ''', re.VERBOSE)
    '''
    A regular expression for matching HTML errors like:

    '''

    __empty_tags = 'area|base|basefont|br|col|frame|hr|img|input|link|meta|param'
    '''A list of all HTML empty tags.'''

    __misopened_tag_regex = re.compile(r'<(' + __empty_tags + tag_attrs_regex + r')\s*>', re.IGNORECASE)
    '''A regular expression for matching opened tags that should be closed.'''

    __tag_stack = None
    '''A stack of currently opened HTML tags.'''

    __cur_data = None
    '''
    Accumulates data between handle_charref(), handle_entityref() and
    handle_data() calls.
    '''

    def __init__(self):
        HTMLParser.__init__(self)

    def handle_charref(self, name):
        '''Handles a character reference of the form &#ref;.'''

        self.__accumulate_data('&#' + name + ';')

    def handle_data(self, data):
        '''Handles data.'''

        self.__accumulate_data(data)

    def handle_endtag(self, tag_name):
        '''Handles end of a tag.'''

        self.__handle_data_if_exists()

        if self.__get_cur_tag()['name'] == tag_name:
            self.__close_tag(self.__tag_stack.pop())
        else:
            for tag_id in xrange(len(self.__tag_stack) - 1, -1, -1):
                if self.__tag_stack[tag_id]['name'] == tag_name:
                    for tag in reversed(self.__tag_stack[tag_id + 1:]):
                        self.__close_tag(tag, forced = True)
                        self.__tag_stack.pop()

                    self.__close_tag(self.__tag_stack.pop())
                    break
            else:
                LOG.debug('Dropping excess end tag "%s"...', tag_name)

    def handle_entityref(self, name):
        '''Handles a general entity reference of the form &name;.'''

        self.__accumulate_data('&' + name + ';')

    def handle_root_data(self, tag, data):
        '''Handles data inside of the root of the document.'''

        LOG.debug('%s', data)

    def handle_root(self, tag, attrs, empty):
        '''Handles a tag inside of the root of the document.'''

        LOG.debug('<%s %s%s>', tag['name'], attrs, '/' if empty else '')
        tag['new_tag_handler'] = self.handle_root
        tag['data_handler'] = self.handle_root_data
        tag['end_tag_handler'] = self.handle_root_end

    def handle_root_end(self, tag):
        '''Handles end of the root of the document.'''

        LOG.debug('</%s>', tag['name'])

    def handle_startendtag(self, tag, attrs):
        '''Handles start of an XHTML-style empty tag.'''

        self.__handle_data_if_exists()
        self.__handle_start_tag(tag, attrs, True)

    def handle_starttag(self, tag, attrs):
        '''Handles start of a tag.'''

        self.__handle_data_if_exists()
        self.__handle_start_tag(tag, attrs, False)

    def reset(self):
        '''Resets the parser.'''

        HTMLParser.reset(self)

        self.__tag_stack = [{
            # Add fake root tag
            'name':              None,
            'new_tag_handler':   self.handle_root,
            'data_handler':      self.handle_root_data,
            'end_tag_handler':   self.handle_root_end,
        }]

    def parse(self, html):
        '''Parses the specified HTML page.'''

        html = self.__fix_html(html)

        self.reset()

        try:
            # Run the parser
            self.feed(html)
            self.close()
        finally:
            # Close all unclosed tags
            for tag in self.__tag_stack[1:]:
                self.__close_tag(tag, True)

    def __accumulate_data(self, data):
        '''
        Accumulates data between handle_charref(), handle_entityref() and
        handle_data() calls.
        '''

        if self.__cur_data is None:
            self.__cur_data = data
        else:
            self.__cur_data += data

    def __close_tag(self, tag, forced = False):
        '''Forces closing of an unclosed tag.'''

        if forced:
            LOG.debug('Force closing of unclosed tag "%s".', tag['name'])
        else:
            LOG.debug('Tag %s closed.', tag)

        if 'end_tag_handler' in tag:
            tag['end_tag_handler'](tag)

        LOG.debug('Current tag: %s.', self.__get_cur_tag())

    def __fix_html(self, html):
        '''Fixes various things that may confuse the Python's HTML parser.'''

        html = self.script_regex.sub('', html)

        loop_replacements = (
            lambda html: self.__invalid_tag_attr_spacing_regex.subn(r'\1 \2', html),
            lambda html: self.__invalid_tag_attr_regex.subn(r'\1 ', html),
        )

        for loop_replacement in loop_replacements:
            for i in xrange(0, 1000):
                html, changed = loop_replacement(html)

                if not changed:
                    break
            else:
                raise Error('Too many errors in the HTML or infinite loop.')

        html = self.__misopened_tag_regex.sub(r'<\1 />', html)

        return html

    def __get_cur_tag(self):
        '''Returns currently opened tag.'''

        return self.__tag_stack[-1]

    def __handle_data_if_exists(self):
        '''Handles accumulated data (if exists).'''

        data = self.__cur_data
        if data is None:
            return

        self.__cur_data = None

        tag = self.__get_cur_tag()
        handler = tag.get('data_handler')

        if handler is not None:
            LOG.debug('Data "%s" in "%s" with handler %s.',
                data, tag['name'], handler.func_name)

            handler(tag, data)

    def __handle_start_tag(self, tag_name, attrs, empty):
        '''Handles start of any tag.'''

        tag = { 'name': tag_name }
        handler = self.__get_cur_tag().get('new_tag_handler')

        if handler is not None:
            attrs = self.__parse_attrs(attrs)

            LOG.debug('Start tag: %s %s with handler %s.',
                tag, attrs, handler.func_name)

            handler(tag, attrs, empty)

        if not empty:
            self.__tag_stack.append(tag)

    def __parse_attrs(self, attrs_tuple):
        '''Converts tag attributes from a tuple to a dictionary.'''

        attrs = {}

        for attr, value in attrs_tuple:
            attrs[attr.lower()] = value

        return attrs
<|endoftext|>


<commit_before>import datetime
import json
import re
import sqlite3

import markovify
import tweepy

import config
import utils

class SimulatorText(markovify.Text):
    def sentence_split(self, text):
        return text.split('<>')

def get_recent_tweets():
    pass

def generate_random_tweet():
    db_connection = sqlite3.connect(config.SQLITE_DB)
    db_cursor = db_connection.cursor()
    db_cursor.execute('SELECT * FROM tweets')

    dataset = ''

    for tweet in db_cursor.fetchall():
        dataset += tweet[1]
        dataset += '<>' # delimiter
    #print(dataset)
    db_connection.close()

    model = SimulatorText(dataset, state_size=2)
    tweet = model.make_short_sentence(140, tries=100)
    tweet = re.sub(r'&', '&', tweet)
    tweet = re.sub(r'<', '<', tweet)
    tweet = re.sub(r'>', '>', tweet)

    return tweet

def send_random_tweet():
    tweet = generate_random_tweet()
    with open(config.RECENT_DB) as f:
        recent = json.load(f)
    #if len(recent) > 30:
    #    recent.pop(0)
    while tweet in recent:
        tweet = generate_random_tweet()

    api = utils.get_api()
    try:
        api.update_status(status=tweet)
    except:
        pass

    recent.append(tweet)
    with open(config.RECENT_DB, 'w') as f:
        json.dump(recent, f)
    return tweet

if __name__ == '__main__':
    t = send_random_tweet()
    print(len(t), t)
    #print(get_recent_tweets())<commit_msg>Use String.replace instead of re.sub for better performance<commit_after>import datetime
import json
import sqlite3

import markovify
import tweepy

import config
import utils

class SimulatorText(markovify.Text):
    def sentence_split(self, text):
        return text.split('<>')

def get_recent_tweets():
    pass

def generate_random_tweet():
    db_connection = sqlite3.connect(config.SQLITE_DB)
    db_cursor = db_connection.cursor()
    db_cursor.execute('SELECT * FROM tweets')

    dataset = ''

    for tweet in db_cursor.fetchall():
        dataset += tweet[1]
        dataset += '<>' # delimiter
    #print(dataset)
    db_connection.close()

    model = SimulatorText(dataset, state_size=2)
    tweet = model.make_short_sentence(140, tries=100)
    tweet = tweet.replace('&', '&')
    tweet = tweet.replace('<', '<')
    tweet = tweet.replace('>', '>')

    return tweet

def send_random_tweet():
    tweet = generate_random_tweet()
    with open(config.RECENT_DB) as f:
        recent = json.load(f)
    #if len(recent) > 30:
    #    recent.pop(0)
    while tweet in recent:
        tweet = generate_random_tweet()

    api = utils.get_api()
    try:
        api.update_status(status=tweet)
    except:
        pass

    recent.append(tweet)
    with open(config.RECENT_DB, 'w') as f:
        json.dump(recent, f)
    return tweet

if __name__ == '__main__':
    t = send_random_tweet()
    print(len(t), t)
    #print(get_recent_tweets())<|endoftext|>


<commit_before>""" Some example rules for the test suite. """

def wat_rule(config, message):
    return message['wat'] == 'blah'

def not_wat_rule(config, message):
    return message['wat'] != 'blah'
<commit_msg>Add example rule for test.<commit_after>""" Some example rules for the test suite. """

import fmn.lib.hinting

def wat_rule(config, message):
    return message['wat'] == 'blah'

def not_wat_rule(config, message):
    return message['wat'] != 'blah'

@fmn.lib.hinting.hint(categories=['whatever'])
def hint_masked_rule(config, message, argument1):
    """ This is a docstring.

    For real, it is a docstring.
    """
    return True
<|endoftext|>