<commit_before># -*- coding: utf-8 -*-
#
# Ryu book documentation build configuration file, created by
# sphinx-quickstart on Wed Oct 16 15:05:08 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['japanesesupport', 'ryubuilder']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Ryu book'
copyright = u'2013, Ryuプロジェクト'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
language = 'ja'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Ryubookdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
'papersize': 'a4paper',
'pointsize': '10pt',
'preamble': r"""
\usepackage{listings,jlisting}
\lstset{
 language={Python},
 frame=single,
 framerule=0pt,
 backgroundcolor={\color[rgb]{0.92,0.92,1}},
 basicstyle={\ttfamily\footnotesize},
 commentstyle={\color[rgb]{0.5,0.5,0.5}},
 keywordstyle={\bfseries\color[rgb]{0,0.4,0.1}},
 emph={self,super}, emphstyle=\color[rgb]{0.4,0.6,0.5},
 showstringspaces=false,
 breaklines=true,
 breakatwhitespace=false,
 breakindent=0pt,
 breakautoindent=false
 }
\lstnewenvironment{sourcecode}{}{}
\lstnewenvironment{console}{\lstset{
 language={},
 frame=single,
 framerule=0pt,
 backgroundcolor={\color[rgb]{0.1,0.1,0.1}},
 basicstyle={\ttfamily\footnotesize\color[rgb]{0.9,0.9,0.9}},
 breaklines=true,
 breakatwhitespace=false,
 breakindent=0pt,
 breakautoindent=false
 }}{}"""
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'Ryubook.tex', u'RyuによるOpenFlowプログラミング',
   u'Ryuプロジェクト', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

latex_docclass = {'manual': 'jsbook'}

latex_additional_files = ['jlisting.sty',
                          'sphinxryu.cls', 'images/coverpage.eps']

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'ryubook', u'RyuによるOpenFlowプログラミング',
     [u'Ryuプロジェクト'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'Ryubook', u'RyuによるOpenFlowプログラミング',
   u'Ryuプロジェクト', 'Ryubook', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'RyuによるOpenFlowプログラミング'
epub_author = u'Ryuプロジェクト'
epub_publisher = u'Ryuプロジェクト'
epub_copyright = u'2013, Ryuプロジェクト'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True
<commit_msg>add decoration to admonition on latex/pdf<commit_after># -*- coding: utf-8 -*-
#
# Ryu book documentation build configuration file, created by
# sphinx-quickstart on Wed Oct 16 15:05:08 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('.'))

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['japanesesupport', 'ryubuilder']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Ryu book'
copyright = u'2013, Ryuプロジェクト'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
language = 'ja'

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'default'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'Ryubookdoc'

# -- Options for LaTeX output --------------------------------------------------

latex_elements = {
'papersize': 'a4paper',
'pointsize': '10pt',
'preamble': r"""
\usepackage{listings,jlisting}
\lstset{
 language={Python},
 frame=single,
 framerule=0pt,
 backgroundcolor={\color[rgb]{0.92,0.92,1}},
 basicstyle={\ttfamily\footnotesize},
 commentstyle={\color[rgb]{0.5,0.5,0.5}},
 keywordstyle={\bfseries\color[rgb]{0,0.4,0.1}},
 emph={self,super}, emphstyle=\color[rgb]{0.4,0.6,0.5},
 showstringspaces=false,
 breaklines=true,
 breakatwhitespace=false,
 breakindent=0pt,
 breakautoindent=false
 }
\lstnewenvironment{sourcecode}{}{}
\lstnewenvironment{console}{\lstset{
 language={},
 frame=single,
 framerule=0pt,
 backgroundcolor={\color[rgb]{0.1,0.1,0.1}},
 basicstyle={\ttfamily\footnotesize\color[rgb]{0.9,0.9,0.9}},
 breaklines=true,
 breakatwhitespace=false,
 breakindent=0pt,
 breakautoindent=false
 }}{}
\makeatletter

\definecolor{notecolor}{rgb}{1,1,.6}
\definecolor{notefrcolor}{rgb}{.6,.6,.3}
\definecolor{hintcolor}{rgb}{.8,1,.8}
\definecolor{hintfrcolor}{rgb}{.5,.6,.5}
\definecolor{importantcolor}{rgb}{1,.8,.8}
\definecolor{importantfrcolor}{rgb}{.6,.4,.4}
\definecolor{tipcolor}{rgb}{.8,1,.8}
\definecolor{tipfrcolor}{rgb}{.5,.6,.5}
\definecolor{warncolor}{rgb}{1,1,0}
\definecolor{warnfrcolor}{rgb}{.5,.5,0}
\definecolor{dangercolor}{rgb}{.8,.2,.2}
\definecolor{dangerfrcolor}{rgb}{.4,.1,.1}

\newcommand{\py@ryubox}[2]{
  \small
  \color[rgb]{.3,.3,.3}
  \setlength\fboxsep{5pt}
  %\setlength\fboxrule{2pt}
  %\def\fbox{\CustomFBox{}{}\z@\fboxrule\z@\fboxrule\fcolorbox}
  \def\FrameCommand{\fcolorbox{#2}{#1}}
  \MakeFramed {\FrameRestore}
}
\newcommand{\py@endryubox}{
  \endMakeFramed
}
\renewcommand{\py@noticestart@note}{\py@ryubox{notecolor}{notefrcolor}}
\renewcommand{\py@noticeend@note}{\py@endryubox}
\renewcommand{\py@noticestart@hint}{\py@ryubox{hintcolor}{hintfrcolor}}
\renewcommand{\py@noticeend@hint}{\py@endryubox}
\renewcommand{\py@noticestart@important}{\py@ryubox{importantcolor}{importantfrcolor}}
\renewcommand{\py@noticeend@important}{\py@endryubox}
\renewcommand{\py@noticestart@tip}{\py@ryubox{tipcolor}{tipfrcolor}}
\renewcommand{\py@noticeend@tip}{\py@endryubox}
\renewcommand{\py@noticestart@warning}{\py@ryubox{warncolor}{warnfrcolor}}
\renewcommand{\py@noticeend@warning}{\py@endryubox}
\renewcommand{\py@noticestart@caution}{\py@ryubox{warncolor}{warnfrcolor}}
\renewcommand{\py@noticeend@caution}{\py@endryubox}
\renewcommand{\py@noticestart@attention}{\py@ryubox{warncolor}{warnfrcolor}}
\renewcommand{\py@noticeend@attention}{\py@endryubox}
\renewcommand{\py@noticestart@danger}{\py@ryubox{dangercolor}{dangerfrcolor}}
\renewcommand{\py@noticeend@danger}{\py@endryubox}
\renewcommand{\py@noticestart@error}{\py@ryubox{dangercolor}{dangerfrcolor}}
\renewcommand{\py@noticeend@error}{\py@endryubox}
\makeatother
"""
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('index', 'Ryubook.tex', u'RyuによるOpenFlowプログラミング',
   u'Ryuプロジェクト', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

latex_docclass = {'manual': 'jsbook'}

latex_additional_files = ['jlisting.sty',
                          'sphinxryu.cls', 'images/coverpage.eps']

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'ryubook', u'RyuによるOpenFlowプログラミング',
     [u'Ryuプロジェクト'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'Ryubook', u'RyuによるOpenFlowプログラミング',
   u'Ryuプロジェクト', 'Ryubook', 'One line description of project.',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# -- Options for Epub output ---------------------------------------------------

# Bibliographic Dublin Core info.
epub_title = u'RyuによるOpenFlowプログラミング'
epub_author = u'Ryuプロジェクト'
epub_publisher = u'Ryuプロジェクト'
epub_copyright = u'2013, Ryuプロジェクト'

# The language of the text. It defaults to the language option
# or en if the language is not set.
#epub_language = ''

# The scheme of the identifier. Typical schemes are ISBN or URL.
#epub_scheme = ''

# The unique identifier of the text. This can be a ISBN number
# or the project homepage.
#epub_identifier = ''

# A unique identification for the text.
#epub_uid = ''

# A tuple containing the cover image and cover page html template filenames.
#epub_cover = ()

# HTML files that should be inserted before the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_pre_files = []

# HTML files shat should be inserted after the pages created by sphinx.
# The format is a list of tuples containing the path and title.
#epub_post_files = []

# A list of files that should not be packed into the epub file.
#epub_exclude_files = []

# The depth of the table of contents in toc.ncx.
#epub_tocdepth = 3

# Allow duplicate toc entries.
#epub_tocdup = True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# This file is part of the Rocket Web Server
# Copyright (c) 2010 Timothy Farrell

# Import System Modules
import os
import sys
import time
import socket
import mimetypes
from glob import glob
from email.utils import formatdate
from wsgiref.headers import Headers
from wsgiref.util import FileWrapper
# Import Package Modules
from .. import HTTP_SERVER_SOFTWARE, b, u, BUF_SIZE
from ..worker import Worker

# Define Constants
CHUNK_SIZE = 2**16 # 64 Kilobyte chunks
HEADER_RESPONSE = '''HTTP/1.1 %s\r\n%s'''
INDEX_HEADER = '''\

<title>Directory Index: %(path)s</title>

Directory Index: %(path)s

Filename	Size

'''
INDEX_ROW = '''
%(name)s	%(size)s
'''
INDEX_FOOTER = '''
\r\n'''

class LimitingFileWrapper(FileWrapper):
    def __init__(self, limit=None, *args, **kwargs):
        self.limit = limit
        FileWrapper.__init__(self, *args, **kwargs)

    def read(self, amt):
        if amt > self.limit:
            amt = self.limit
        self.limit -= amt
        return FileWrapper.read(self, amt)

class FileSystemWorker(Worker):
    def __init__(self):
        """Builds some instance variables that will last the life of the
        thread."""

        Worker.__init__(self)

        self.root = os.path.abspath(self.app_info['document_root'])
        self.display_index = self.app_info['display_index']

    def serve_file(self, filepath, headers):
        filestat = os.stat(filepath)
        self.size = filestat.st_size
        modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                time.gmtime(filestat.st_mtime))
        self.headers.add_header('Last-Modified', modtime)
        if headers.get('if_modified_since') == modtime:
            # The browser cache is up-to-date, send a 304.
            self.status = "304 Not Modified"
            self.data = []
            return

        ct = mimetypes.guess_type(filepath)[0]
        self.content_type = ct if ct else 'text/plain'
        try:
            f = open(filepath, 'rb')
            self.headers['Pragma'] = 'cache'
            self.headers['Cache-Control'] = 'private'
            self.headers['Content-Length'] = str(self.size)
            if self.etag:
                h.add_header('Etag', self.etag)
            if self.expires:
                h.add_header('Expires', self.expires)

            try:
                # Implement 206 partial file support.
                start, end = headers['range'].split('-')
                start = 0 if not start.isdigit() else int(start)
                end = self.size if not end.isdigit() else int(end)
                if self.size < end or start < 0:
                    self.status = "214 Unsatisfiable Range Requested"
                    self.data = FileWrapper(f, CHUNK_SIZE)
                else:
                    f.seek(start)
                    self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end)
                    self.status = "206 Partial Content"
            except:
                self.data = FileWrapper(f, CHUNK_SIZE)
        except IOError:
            self.status = "403 Forbidden"

    def serve_dir(self, pth, rpth):
        def rel_path(path):
            return '/' + path[len(self.root) + 1:] if path.startswith(self.root) else path

        if not self.display_index:
            self.status = '404 File Not Found'
            return b('')
        else:
            self.content_type = 'text/html'
            glob_spec = os.path.normpath('{0}{1}*'.format(pth, os.path.sep))

            globs = [x for x in glob(glob_spec)]
            dirs = [rel_path(x)+'/' for x in globs if os.path.isdir(x)]
            files = [rel_path(x) for x in globs if os.path.isfile(x)]

            self.data = [INDEX_HEADER % dict(path=rpth)]
            self.data += [INDEX_ROW % dict(name=os.path.basename(x[:-1]), size=0, link=rel_path(x)) for x in dirs]
            self.data += [INDEX_ROW % dict(name=os.path.basename(x), size=0, link=rel_path(x)) for x in files]
            self.data += [INDEX_FOOTER]
            self.headers['Content-Length'] = sum([len(x) for x in self.data])
            self.status = '200 OK'

    def run_app(self, conn):
        self.status = "200 OK"
        self.size = 0
        self.expires = None
        self.etag = None
        self.content_type = 'text/plain'
        self.content_length = None

        self.err_log.debug('Getting sock_file')
        # Build our file-like object
        sock_file = conn.makefile('rb',BUF_SIZE)
        request = self.read_request_line(sock_file)
        if request['method'].upper() not in ('GET', ):
            self.status = "501 Not Implemented"

        try:
            # Get our file path
            headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()])
            rpath = request.get('path', '').lstrip('/')
            filepath = os.path.join(self.root, rpath)
            filepath = os.path.abspath(filepath)
            self.err_log.debug('Request for path: %s' % filepath)
            self.closeConnection = headers.get('connection', 'close').lower() == 'close'
            self.headers = Headers([('Date', formatdate(usegmt=True)),
                                    ('Server', HTTP_SERVER_SOFTWARE),
                                    ('Connection', headers.get('connection', 'close')),
                                   ])

            if not filepath.lower().startswith(self.root.lower()):
                # File must be within our root directory
                self.status = "400 Bad Request"
                self.closeConnection = True
            elif not os.path.exists(filepath):
                self.status = "404 File Not Found"
                self.closeConnection = True
            elif os.path.isdir(filepath):
                self.serve_dir(filepath, rpath)
            elif os.path.isfile(filepath):
                self.serve_file(filepath, headers)
            else:
                # It exists but it's not a file or a directory????
                # What is it then?
                self.status = "501 Not Implemented"
                self.closeConnection = True

            h = self.headers
            statcode, statstr = self.status.split(' ', 1)
            statcode = int(statcode)
            if statcode >= 400:
                h.add_header('Content-Type', self.content_type)
                self.data = [statstr]

            # Build our output headers
            header_data = HEADER_RESPONSE % (self.status, str(h))

            # Send the headers
            self.err_log.debug('Sending Headers: %s' % repr(header_data))
            self.conn.sendall(b(header_data))

            for data in self.data:
                self.conn.sendall(b(data))

            if hasattr(self.data, 'close'):
                self.data.close()

        finally:
            self.err_log.debug('Finally closing sock_file')
            sock_file.close()
<commit_msg>Filesystem service does directory indexes properly now.<commit_after># -*- coding: utf-8 -*-

# This file is part of the Rocket Web Server
# Copyright (c) 2010 Timothy Farrell

# Import System Modules
import os
import sys
import time
import socket
import mimetypes
from email.utils import formatdate
from wsgiref.headers import Headers
from wsgiref.util import FileWrapper
# Import Package Modules
from .. import HTTP_SERVER_SOFTWARE, b, u, BUF_SIZE
from ..worker import Worker

# Define Constants
CHUNK_SIZE = 2**16 # 64 Kilobyte chunks
HEADER_RESPONSE = '''HTTP/1.1 %s\r\n%s'''
INDEX_HEADER = '''\

<title>Directory Index: %(path)s</title>

Directory Index: %(path)s

Directories

'''
INDEX_ROW = '''

%(name)s

'''
INDEX_FOOTER = '''
\r\n'''

class LimitingFileWrapper(FileWrapper):
    def __init__(self, limit=None, *args, **kwargs):
        self.limit = limit
        FileWrapper.__init__(self, *args, **kwargs)

    def read(self, amt):
        if amt > self.limit:
            amt = self.limit
        self.limit -= amt
        return FileWrapper.read(self, amt)

class FileSystemWorker(Worker):
    def __init__(self):
        """Builds some instance variables that will last the life of the
        thread."""

        Worker.__init__(self)

        self.root = os.path.abspath(self.app_info['document_root'])
        self.display_index = self.app_info['display_index']

    def serve_file(self, filepath, headers):
        filestat = os.stat(filepath)
        self.size = filestat.st_size
        modtime = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                time.gmtime(filestat.st_mtime))
        self.headers.add_header('Last-Modified', modtime)
        if headers.get('if_modified_since') == modtime:
            # The browser cache is up-to-date, send a 304.
            self.status = "304 Not Modified"
            self.data = []
            return

        ct = mimetypes.guess_type(filepath)[0]
        self.content_type = ct if ct else 'text/plain'
        try:
            f = open(filepath, 'rb')
            self.headers['Pragma'] = 'cache'
            self.headers['Cache-Control'] = 'private'
            self.headers['Content-Length'] = str(self.size)
            if self.etag:
                h.add_header('Etag', self.etag)
            if self.expires:
                h.add_header('Expires', self.expires)

            try:
                # Implement 206 partial file support.
                start, end = headers['range'].split('-')
                start = 0 if not start.isdigit() else int(start)
                end = self.size if not end.isdigit() else int(end)
                if self.size < end or start < 0:
                    self.status = "214 Unsatisfiable Range Requested"
                    self.data = FileWrapper(f, CHUNK_SIZE)
                else:
                    f.seek(start)
                    self.data = LimitingFileWrapper(f, CHUNK_SIZE, limit=end)
                    self.status = "206 Partial Content"
            except:
                self.data = FileWrapper(f, CHUNK_SIZE)
        except IOError:
            self.status = "403 Forbidden"

    def serve_dir(self, pth, rpth):
        def rel_path(path):
            return '/' + path[len(self.root) + 1:] if path.startswith(self.root) else path

        if not self.display_index:
            self.status = '404 File Not Found'
            return b('')
        else:
            self.content_type = 'text/html'

            dir_contents = [os.path.join(rpth, x) for x in os.listdir(os.path.normpath(pth))]
            dir_contents.sort()

            dirs = [rel_path(x)+'/' for x in dir_contents if os.path.isdir(x)]
            files = [rel_path(x) for x in dir_contents if os.path.isfile(x)]

            self.data = [INDEX_HEADER % dict(path=rpth)]
            if rpth:
                self.data += [INDEX_ROW % dict(name='(parent directory)', cls='dir parent', link='/'+'/'.join(rpth[:-1].split('/')[:-1]))]
            self.data += [INDEX_ROW % dict(name=os.path.basename(x[:-1]), link=os.path.join(rpth, os.path.basename(x[:-1])), cls='dir') for x in dirs]
            self.data += ['
Files
']
            self.data += [INDEX_ROW % dict(name=os.path.basename(x), link=os.path.join(rpth, os.path.basename(x)), cls='file') for x in files]
            self.data += [INDEX_FOOTER]
            self.headers['Content-Length'] = sum([len(x) for x in self.data])
            self.status = '200 OK'

    def run_app(self, conn):
        self.status = "200 OK"
        self.size = 0
        self.expires = None
        self.etag = None
        self.content_type = 'text/plain'
        self.content_length = None

        self.err_log.debug('Getting sock_file')
        # Build our file-like object
        sock_file = conn.makefile('rb',BUF_SIZE)
        request = self.read_request_line(sock_file)
        if request['method'].upper() not in ('GET', ):
            self.status = "501 Not Implemented"

        try:
            # Get our file path
            headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()])
            rpath = request.get('path', '').lstrip('/')
            filepath = os.path.join(self.root, rpath)
            filepath = os.path.abspath(filepath)
            self.err_log.debug('Request for path: %s' % filepath)
            self.closeConnection = headers.get('connection', 'close').lower() == 'close'
            self.headers = Headers([('Date', formatdate(usegmt=True)),
                                    ('Server', HTTP_SERVER_SOFTWARE),
                                    ('Connection', headers.get('connection', 'close')),
                                   ])

            if not filepath.lower().startswith(self.root.lower()):
                # File must be within our root directory
                self.status = "400 Bad Request"
                self.closeConnection = True
            elif not os.path.exists(filepath):
                self.status = "404 File Not Found"
                self.closeConnection = True
            elif os.path.isdir(filepath):
                self.serve_dir(filepath, rpath)
            elif os.path.isfile(filepath):
                self.serve_file(filepath, headers)
            else:
                # It exists but it's not a file or a directory????
                # What is it then?
                self.status = "501 Not Implemented"
                self.closeConnection = True

            h = self.headers
            statcode, statstr = self.status.split(' ', 1)
            statcode = int(statcode)
            if statcode >= 400:
                h.add_header('Content-Type', self.content_type)
                self.data = [statstr]

            # Build our output headers
            header_data = HEADER_RESPONSE % (self.status, str(h))

            # Send the headers
            self.err_log.debug('Sending Headers: %s' % repr(header_data))
            self.conn.sendall(b(header_data))

            for data in self.data:
                self.conn.sendall(b(data))

            if hasattr(self.data, 'close'):
                self.data.close()

        finally:
            self.err_log.debug('Finally closing sock_file')
            sock_file.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

'''
https://en.wikipedia.org/wiki/Wikipedia:Getting_to_Philosophy

Script that prints the titles of every article from a start
article on the way to Philosophy.
'''

from bs4 import BeautifulSoup
import urllib2
import wikipedia
import re
import sys

class ExtractLink:
    ''' Class containing subroutines to parse html
    and return the first valid link in a Wikipedia article.'''
    def __init__(self, article_html):
        self.soup = BeautifulSoup(article_html, "html.parser")
        self.soup = self.soup.find(id="mw-content-text")

    def clean_parentheses(self, content):
        ''' Remove content inside parentheses. '''
        return re.sub(r'\([^)]*\)', '', content)

    def is_valid_link(self, href, paragraph):
        ''' A link is valid only if:
            1. It is not an external link or an anchor or a Help link.
            2. It is not within parentheses.
        '''
        if not href or "#" in href or "//" in href or ":" in href \
        or "/wiki/" not in href or href not in paragraph:
            return False

        prefix = paragraph.split(href, 1)[0]

        if prefix.count("(") != prefix.count(")"):
            return False

        return True

    def is_valid_tag(self, tag):
        ''' The only valid tags are <p> and <ul> tags. '''
        return tag.name == 'p' or tag.name == 'ul'

    def get_paragraphs(self):
        ''' Get all valid, top-level paragraphs from the article. '''
        self.paragraphs = self.soup.find_all(self.is_valid_tag, recursive=False)

        return self.paragraphs

    def iterate_links(self):
        ''' Iterate through all links in each paragraph
            and output the first valid link encountered. '''
        for each_paragraph in self.paragraphs:
            for each_link in each_paragraph.find_all("a"):
                href = each_link.get("href")
                if self.is_valid_link(str(href), str(each_paragraph)):
                    return each_link

        return False

    def get_first_link(self):
        ''' Wrapper function to run other subroutines and
            output a dictionary of info about the next article. '''
        self.get_paragraphs()
        first_link = self.iterate_links()

        return {"next_link": first_link.get('href'),
                "next_title": first_link.get('title'),
                "text": first_link.get('text')}

class IterateArticles:
    ''' Class with subroutines to traverse the first links from
    the start article to Philosophy. '''
    def __init__(self, start_page):
        self.start_page = start_page
        self.base_url = "http://en.wikipedia.org/w/index.php?title="

    def get_page(self, address):
        ''' Fetch html of the page from address. '''
        req = urllib2.Request(address)
        data = urllib2.urlopen(req).read()
        return data

    def traverse(self):
        ''' Traverse the first links from the start article,
        recursively until Philosophy is reached. '''
        page_name = self.start_page
        page_url = self.base_url + page_name + "&printable=yes"
        results = []

        while True:
            sys.stdout.write(page_name + ' -> ')
            sys.stdout.flush()

            results.append(page_name)

            article_html = self.get_page(page_url)
            next_link_obj = ExtractLink(article_html).get_first_link()

            page_name = next_link_obj['next_link'][6:]
            page_url = self.base_url + page_name + "&printable=yes"

            if page_name == 'Philosophy':
                print 'Philosophy'
                results.append('Philosophy')
                return results
            elif page_name in results:
                print 'Loop detected.'
                return []

if __name__ == "__main__":
    if len(sys.argv) > 0:
        try:
            results = IterateArticles(sys.argv[1])
            results.traverse()
        except IndexError:
            print 'Format: python tagspace.py <page name>'
    else:
        print 'Format: python tagspace.py <page name>'

<commit_msg>Find common ancestors between two topic paths.<commit_after>#!/usr/bin/env python

'''
https://en.wikipedia.org/wiki/Wikipedia:Getting_to_Philosophy

Script that prints the titles of every article from a start
article on the way to Philosophy.
'''

from bs4 import BeautifulSoup
import urllib2
import wikipedia
import re
import sys

class ExtractLink:
    ''' Class containing subroutines to parse html
    and return the first valid link in a Wikipedia article.'''
    def __init__(self, article_html):
        self.soup = BeautifulSoup(article_html, "html.parser")
        self.soup = self.soup.find(id="mw-content-text")

    def clean_parentheses(self, content):
        ''' Remove content inside parentheses. '''
        return re.sub(r'\([^)]*\)', '', content)

    def is_valid_link(self, href, paragraph):
        ''' A link is valid only if:
            1. It is not an external link or an anchor or a Help link.
            2. It is not within parentheses.
        '''
        if not href or "#" in href or "//" in href or ":" in href \
        or "/wiki/" not in href or href not in paragraph:
            return False

        prefix = paragraph.split(href, 1)[0]

        if prefix.count("(") != prefix.count(")"):
            return False

        return True

    def is_valid_tag(self, tag):
        ''' The only valid tags are <p> and <ul> tags. '''
        return tag.name == 'p' or tag.name == 'ul'

    def get_paragraphs(self):
        ''' Get all valid, top-level paragraphs from the article. '''
        self.paragraphs = self.soup.find_all(self.is_valid_tag, recursive=False)

        return self.paragraphs

    def iterate_links(self):
        ''' Iterate through all links in each paragraph
            and output the first valid link encountered. '''
        for each_paragraph in self.paragraphs:
            for each_link in each_paragraph.find_all("a"):
                href = each_link.get("href")
                if self.is_valid_link(str(href), str(each_paragraph)):
                    return each_link

        return False

    def get_first_link(self):
        ''' Wrapper function to run other subroutines and
            output a dictionary of info about the next article. '''
        self.get_paragraphs()
        first_link = self.iterate_links()

        return {"next_link": first_link.get('href'),
                "next_title": first_link.get('title'),
                "text": first_link.get('text')}

class IterateArticles:
    ''' Class with subroutines to traverse the first links from
    the start article to Philosophy. '''
    def __init__(self, start_page):
        self.start_page = start_page
        self.base_url = "http://en.wikipedia.org/w/index.php?title="

    def get_page(self, address):
        ''' Fetch html of the page from address. '''
        req = urllib2.Request(address)
        data = urllib2.urlopen(req).read()
        return data

    def traverse(self):
        ''' Traverse the first links from the start article,
        recursively until Philosophy is reached. '''
        page_name = self.start_page
        page_url = self.base_url + page_name + "&printable=yes"
        results = []

        while True:
            sys.stdout.write(page_name + ' -> ')
            sys.stdout.flush()

            results.append(page_name)

            article_html = self.get_page(page_url)
            next_link_obj = ExtractLink(article_html).get_first_link()

            page_name = next_link_obj['next_link'][6:]
            page_url = self.base_url + page_name + "&printable=yes"

            if page_name == 'Philosophy':
                print 'Philosophy'
                results.append('Philosophy')
                return results
            elif page_name in results:
                print 'Loop detected.'
                return results

def find_intersection(topic_x, topic_y):
    "Find common ancestors between two topic paths."
    results_x = IterateArticles(topic_x).traverse()
    results_y = IterateArticles(topic_y).traverse()
    return set(results_x).intersection(results_y)

if __name__ == "__main__":
    if len(sys.argv) > 0:
        try:
            results = IterateArticles(sys.argv[1])
            results.traverse()
        except IndexError:
            print 'Format: python tagspace.py <page name>'
    else:
        print 'Format: python tagspace.py <page name>'

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import requests
from datetime import datetime, timedelta
from jobs import AbstractJob

SYMBOL_TABLE = {
    "sleetshowersandthunder": "Sluddbyger og torden",
    "heavyrainandthunder": "Kraftig regn og torden",
    "partlycloudy": "Delvis skyet",
    "rainshowersandthunder": "Regnbyger og torden",
    "lightssnowshowersandthunder": "Lette snøbyger og torden",
    "sleet": "Sludd",
    "lightsleetandthunder": "Lett sludd og torden",
    "lightsleetshowers": "Lette sluddbyger",
    "lightrain": "Lett regn",
    "lightsnow": "Lett snø",
    "lightrainshowers": "Lette regnbyger",
    "heavysnowshowers": "Kraftige snøbyger",
    "lightrainandthunder": "Lett regn og torden",
    "heavysnow": "Kraftig snø",
    "heavysleet": "Kraftig sludd",
    "lightsnowandthunder": "Lett snø og torden",
    "heavysleetshowers": "Kraftige sluddbyger",
    "fair": "Lettskyet",
    "snowshowers": "Snøbyger",
    "heavysnowandthunder": "Kraftig snø og torden",
    "heavyrainshowersandthunder": "Kraftige regnbyger og torden",
    "fog": "Tåke",
    "heavyrain": "Kraftig regn",
    "snowshowersandthunder": "Snøbyger og torden",
    "lightssleetshowersandthunder": "Lette sluddbyger og torden",
    "clearsky": "Klarvær",
    "rainshowers": "Regnbyger",
    "cloudy": "Skyet",
    "heavysnowshowersandthunder": "Kraftige snøbyger og torden",
    "heavysleetandthunder": "Kraftig sludd og torden",
    "snow": "Snø",
    "rainandthunder": "Regn og torden",
    "sleetandthunder": "Sludd og torden",
    "lightsleet": "Lett sludd",
    "lightrainshowersandthunder": "Lette regnbyger og torden",
    "snowandthunder": "Snø og torden",
    "rain": "Regn",
    "heavyrainshowers": "Kraftige regnbyger",
    "lightsnowshowers": "Lette snøbyger",
    "sleetshowers": "Sluddbyger",
    "heavysleetshowersandthunder": "Kraftige sluddbyger og torden",
}

DEFAULT_FORECAST_HOUR = 12

class Yr(AbstractJob):
    def __init__(self, conf):
        self.url = conf["url"]
        self.interval = conf["interval"]
        self.location = conf["location"]
        self.timeout = conf.get("timeout")
        self.forecast_hour = conf.get("forecast_hour", DEFAULT_FORECAST_HOUR)

    def _temperature(self, observation):
        return observation["data"]["instant"]["details"]["air_temperature"]

    def _description(self, observation, short_term=True):
        # The periods returned from the Yr API are unpredicatable, we therefore try all
        # possible periods in order
        if short_term:
            periods = ("next_1_hours", "next_6_hours", "next_12_hours")
        else:
            periods = ("next_6_hours", "next_1_hours", "next_12_hours")
        known_periods = observation["data"].keys()
        period = next((p for p in periods if p in known_periods), None)
        if period is None:
            raise ValueError(
                "Yr API returned periods {}, which does not contain any of {}".format(
                    known_periods, periods
                )
            )
        symbol_code = observation["data"][period]["summary"]["symbol_code"]
        # Remove any trailing time of day identifier: cloudy_night -> cloudy
        symbol_key = symbol_code.split("_")[0]
        return (SYMBOL_TABLE[symbol_key], symbol_code)

    def _baufort(self, wind_speed):  # noqa: C901
        if wind_speed < 0.3:
            return "Stille"
        elif wind_speed < 1.6:
            return "Flau vind"
        elif wind_speed < 3.4:
            return "Svak vind"
        elif wind_speed < 5.5:
            return "Lett bris"
        elif wind_speed < 8.0:
            return "Laber bris"
        elif wind_speed < 10.8:
            return "Frisk bris"
        elif wind_speed < 13.9:
            return "Liten kuling"
        elif wind_speed < 17.2:
            return "Stiv kuling"
        elif wind_speed < 20.8:
            return "Sterk kuling"
        elif wind_speed < 24.5:
            return "Liten storm"
        elif wind_speed < 28.5:
            return "Full storm"
        elif wind_speed < 32.7:
            return "Sterk storm"
        return "Orkan"

    def _direction(self, deg):  # noqa: C901
        if deg <= 22.5 or deg > 337.5:
            return "nord"
        elif deg <= 67.5:
            return "nordøst"
        elif deg <= 112.5:
            return "øst"
        elif deg <= 157.5:
            return "sørøst"
        elif deg <= 202.5:
            return "sør"
        elif deg <= 247.5:
            return "sørvest"
        elif deg <= 292.5:
            return "vest"
        elif deg <= 337.5:
            return "nordvest"
        raise ValueError("Invalid direction {}".format(deg))

    def _day_name(self, dt):
        day = dt.weekday()
        if day == 0:
            return "mandag"
        elif day == 1:
            return "tirsdag"
        elif day == 2:
            return "onsdag"
        elif day == 3:
            return "torsdag"
        elif day == 4:
            return "fredag"
        elif day == 5:
            return "lørdag"
        elif day == 6:
            return "søndag"
        raise ValueError("Invalid day: {}".format(day))

    def _wind(self, observation):
        speed = observation["data"]["instant"]["details"]["wind_speed"]
        direction = self._direction(
            observation["data"]["instant"]["details"]["wind_from_direction"]
        )
        description = self._baufort(speed)
        return {"speed": speed, "direction": direction, "description": description}

    def _find_forecast(self, data, date, hourly=False, count=6):
        if not hourly:
            date = date.replace(hour=self.forecast_hour)
        forecast = []
        for n in range(1, count + 1):
            forecasted_date = date + timedelta(days=n)
            if hourly:
                forecasted_date = date + timedelta(hours=n)
            observation = self._find_observation(data, forecasted_date)
            temperature = self._temperature(observation)
            if not hourly or n == 1:
                # Get the description for next 6 hours if we're forecasting future days
                # or if we're looking at the forecast for the first hour
                _, symbol = self._description(observation, short_term=False)
            dt = datetime.strptime(observation["time"], "%Y-%m-%dT%H:%M:%SZ")
            f = {
                "day": self._day_name(dt),
                "hour": dt.hour,
                "temperature": temperature,
                "symbol": symbol,
            }
            forecast.append(f)
        return forecast

    def _find_observation(self, data, date):
        date_fmt = date.strftime("%Y-%m-%dT%H:00:00Z")
        timeseries = data["properties"]["timeseries"]
        for ts in timeseries:
            if ts["time"] != date_fmt:
                continue
            return ts
        if date.hour != DEFAULT_FORECAST_HOUR:
            # Fall back to default hour if there is no match for given hour
            date = date.replace(hour=DEFAULT_FORECAST_HOUR)
            return self._find_observation(data, date)
        raise ValueError("No observation found for time {}".format(date_fmt))

    def _parse_week(self, data, date):
        forecast = self._find_forecast(data, date)
        return {"forecast": forecast}

    def _parse_day(self, data, date):
        observation = self._find_observation(data, date)
        forecast = self._find_forecast(data, date, hourly=True)
        temperature = self._temperature(observation)
        description, symbol = self._description(observation)
        wind = self._wind(observation)
        return {
            "location": self.location,
            "description": description,
            "symbol": symbol,
            "temperature": temperature,
            "wind": wind,
            "forecast": forecast,
        }

    def _parse(self, data, now=None):
        if now is None:
            now = datetime.now()
        next_day = (now + timedelta(days=1)).replace(hour=self.forecast_hour)
        return {
            "today": self._parse_day(data, now),
            "tomorrow": self._parse_day(data, next_day),
            "week": self._parse_week(data, now),
        }

    def get(self):
        # Yr API requires a unique user agent
        # https://api.met.no/weatherapi/locationforecast/2.0/documentation#AUTHENTICATION
        headers = {"User-Agent": "jarvis2/1.0 (+https://github.com/mpolden/jarvis2)"}
        r = requests.get(self.url, timeout=self.timeout, headers=headers)
        r.raise_for_status()
        return self._parse(r.json())
<commit_msg>yr: Use UTC time when looking up forecast<commit_after># -*- coding: utf-8 -*-

import requests
from datetime import datetime, timedelta
from jobs import AbstractJob

SYMBOL_TABLE = {
    "sleetshowersandthunder": "Sluddbyger og torden",
    "heavyrainandthunder": "Kraftig regn og torden",
    "partlycloudy": "Delvis skyet",
    "rainshowersandthunder": "Regnbyger og torden",
    "lightssnowshowersandthunder": "Lette snøbyger og torden",
    "sleet": "Sludd",
    "lightsleetandthunder": "Lett sludd og torden",
    "lightsleetshowers": "Lette sluddbyger",
    "lightrain": "Lett regn",
    "lightsnow": "Lett snø",
    "lightrainshowers": "Lette regnbyger",
    "heavysnowshowers": "Kraftige snøbyger",
    "lightrainandthunder": "Lett regn og torden",
    "heavysnow": "Kraftig snø",
    "heavysleet": "Kraftig sludd",
    "lightsnowandthunder": "Lett snø og torden",
    "heavysleetshowers": "Kraftige sluddbyger",
    "fair": "Lettskyet",
    "snowshowers": "Snøbyger",
    "heavysnowandthunder": "Kraftig snø og torden",
    "heavyrainshowersandthunder": "Kraftige regnbyger og torden",
    "fog": "Tåke",
    "heavyrain": "Kraftig regn",
    "snowshowersandthunder": "Snøbyger og torden",
    "lightssleetshowersandthunder": "Lette sluddbyger og torden",
    "clearsky": "Klarvær",
    "rainshowers": "Regnbyger",
    "cloudy": "Skyet",
    "heavysnowshowersandthunder": "Kraftige snøbyger og torden",
    "heavysleetandthunder": "Kraftig sludd og torden",
    "snow": "Snø",
    "rainandthunder": "Regn og torden",
    "sleetandthunder": "Sludd og torden",
    "lightsleet": "Lett sludd",
    "lightrainshowersandthunder": "Lette regnbyger og torden",
    "snowandthunder": "Snø og torden",
    "rain": "Regn",
    "heavyrainshowers": "Kraftige regnbyger",
    "lightsnowshowers": "Lette snøbyger",
    "sleetshowers": "Sluddbyger",
    "heavysleetshowersandthunder": "Kraftige sluddbyger og torden",
}

DEFAULT_FORECAST_HOUR = 12

class Yr(AbstractJob):
    def __init__(self, conf):
        self.url = conf["url"]
        self.interval = conf["interval"]
        self.location = conf["location"]
        self.timeout = conf.get("timeout")
        self.forecast_hour = conf.get("forecast_hour", DEFAULT_FORECAST_HOUR)

    def _temperature(self, observation):
        return observation["data"]["instant"]["details"]["air_temperature"]

    def _description(self, observation, short_term=True):
        # The periods returned from the Yr API are unpredicatable, we therefore try all
        # possible periods in order
        if short_term:
            periods = ("next_1_hours", "next_6_hours", "next_12_hours")
        else:
            periods = ("next_6_hours", "next_1_hours", "next_12_hours")
        known_periods = observation["data"].keys()
        period = next((p for p in periods if p in known_periods), None)
        if period is None:
            raise ValueError(
                "Yr API returned periods {}, which does not contain any of {}".format(
                    known_periods, periods
                )
            )
        symbol_code = observation["data"][period]["summary"]["symbol_code"]
        # Remove any trailing time of day identifier: cloudy_night -> cloudy
        symbol_key = symbol_code.split("_")[0]
        return (SYMBOL_TABLE[symbol_key], symbol_code)

    def _baufort(self, wind_speed):  # noqa: C901
        if wind_speed < 0.3:
            return "Stille"
        elif wind_speed < 1.6:
            return "Flau vind"
        elif wind_speed < 3.4:
            return "Svak vind"
        elif wind_speed < 5.5:
            return "Lett bris"
        elif wind_speed < 8.0:
            return "Laber bris"
        elif wind_speed < 10.8:
            return "Frisk bris"
        elif wind_speed < 13.9:
            return "Liten kuling"
        elif wind_speed < 17.2:
            return "Stiv kuling"
        elif wind_speed < 20.8:
            return "Sterk kuling"
        elif wind_speed < 24.5:
            return "Liten storm"
        elif wind_speed < 28.5:
            return "Full storm"
        elif wind_speed < 32.7:
            return "Sterk storm"
        return "Orkan"

    def _direction(self, deg):  # noqa: C901
        if deg <= 22.5 or deg > 337.5:
            return "nord"
        elif deg <= 67.5:
            return "nordøst"
        elif deg <= 112.5:
            return "øst"
        elif deg <= 157.5:
            return "sørøst"
        elif deg <= 202.5:
            return "sør"
        elif deg <= 247.5:
            return "sørvest"
        elif deg <= 292.5:
            return "vest"
        elif deg <= 337.5:
            return "nordvest"
        raise ValueError("Invalid direction {}".format(deg))

    def _day_name(self, dt):
        day = dt.weekday()
        if day == 0:
            return "mandag"
        elif day == 1:
            return "tirsdag"
        elif day == 2:
            return "onsdag"
        elif day == 3:
            return "torsdag"
        elif day == 4:
            return "fredag"
        elif day == 5:
            return "lørdag"
        elif day == 6:
            return "søndag"
        raise ValueError("Invalid day: {}".format(day))

    def _wind(self, observation):
        speed = observation["data"]["instant"]["details"]["wind_speed"]
        direction = self._direction(
            observation["data"]["instant"]["details"]["wind_from_direction"]
        )
        description = self._baufort(speed)
        return {"speed": speed, "direction": direction, "description": description}

    def _find_forecast(self, data, date, hourly=False, count=6):
        if not hourly:
            date = date.replace(hour=self.forecast_hour)
        forecast = []
        for n in range(1, count + 1):
            forecasted_date = date + timedelta(days=n)
            if hourly:
                forecasted_date = date + timedelta(hours=n)
            observation = self._find_observation(data, forecasted_date)
            temperature = self._temperature(observation)
            if not hourly or n == 1:
                # Get the description for next 6 hours if we're forecasting future days
                # or if we're looking at the forecast for the first hour
                _, symbol = self._description(observation, short_term=False)
            dt = datetime.strptime(observation["time"], "%Y-%m-%dT%H:%M:%SZ")
            f = {
                "day": self._day_name(dt),
                "hour": dt.hour,
                "temperature": temperature,
                "symbol": symbol,
            }
            forecast.append(f)
        return forecast

    def _find_observation(self, data, date):
        date_fmt = date.strftime("%Y-%m-%dT%H:00:00Z")
        timeseries = data["properties"]["timeseries"]
        for ts in timeseries:
            if ts["time"] != date_fmt:
                continue
            return ts
        if date.hour != DEFAULT_FORECAST_HOUR:
            # Fall back to default hour if there is no match for given hour
            date = date.replace(hour=DEFAULT_FORECAST_HOUR)
            return self._find_observation(data, date)
        raise ValueError("No observation found for time {}".format(date_fmt))

    def _parse_week(self, data, date):
        forecast = self._find_forecast(data, date)
        return {"forecast": forecast}

    def _parse_day(self, data, date):
        observation = self._find_observation(data, date)
        forecast = self._find_forecast(data, date, hourly=True)
        temperature = self._temperature(observation)
        description, symbol = self._description(observation)
        wind = self._wind(observation)
        return {
            "location": self.location,
            "description": description,
            "symbol": symbol,
            "temperature": temperature,
            "wind": wind,
            "forecast": forecast,
        }

    def _parse(self, data, now=None):
        if now is None:
            now = datetime.utcnow()
        next_day = (now + timedelta(days=1)).replace(hour=self.forecast_hour)
        return {
            "today": self._parse_day(data, now),
            "tomorrow": self._parse_day(data, next_day),
            "week": self._parse_week(data, now),
        }

    def get(self):
        # Yr API requires a unique user agent
        # https://api.met.no/weatherapi/locationforecast/2.0/documentation#AUTHENTICATION
        headers = {"User-Agent": "jarvis2/1.0 (+https://github.com/mpolden/jarvis2)"}
        r = requests.get(self.url, timeout=self.timeout, headers=headers)
        r.raise_for_status()
        return self._parse(r.json())
<|endoftext|>


<commit_before>#The entire taxonomy tree

#A node (assignment of a read by Tango) in such a tree contains:
#@name the name of the species
#@rank its rank (k:Kingdom, p: Phylum, c: Class, o: Order, f: Family, g: Genus, s: Species) (Domain can be added easily if needed)
#@ident an identifier for the node, unique in the tree
#@sampleHitList a list that contains (name of sample,number of assignments to this node in this sample) pairs
#@lineage the list of ancestors ((name,rank) list) of the node (from the father to the root)
#@children the list of children (TaxoTree) of the node ordered by growing @ident
#@paths keeps memory of paths to every node/leaf of the tree (to compute more easily distances between nodes of a TaxoTree)
#Other identifier for a node (ident apart) is (@name,@rank)

#Nodes and leaves are only distinguinshed by children=[]

from parsingTree import parseTree
from parsingMatrix import parseMatrix,associatedData
from misc import containsSpecie,mem,selectPath,compare

class TaxoTree(object):
    #
    #
    def __init__(self,name="Root",rank="R",ident=0,sampleHitList=None,lineage=None,children=None,paths=None):
        self.name = name
        self.rank = rank
        self.ident = ident
        self.sampleHitList = sampleHitList or []
        self.lineage = lineage or []
        self.children = children or []
        self.paths = paths or []
        #
        #
    def __eq__(self,name,rank):
        return (self.name == name) and (self.rank == rank)
    #
    #
    #Returns the tree rooted to the node labelled by name, rank
    def search(self,name,rank,DFS=False):
        if self.__eq__(name,rank):
            return self
        else:
            nodeList = []
            for ch in self.children:
                nodeList.append(ch)
            while nodeList:
                node = nodeList.pop()
                if node.__eq__(name,rank):
                    return node
                if DFS:
                    nodeList = node.children + nodeList
                else:
                    nodeList += node.children
            print "\n/!\ ERROR: Subtree rooted at: (",name,rank,") not found. It probably means that this node is not assigned in the samples you have selected."
            raise ValueError
        #
        #
    def __isMember__(self,name,rank,DFS=False):
        if self.__eq__(name,rank):
            return True
        else:
            nodeList = self.children
            while nodeList:
                node = nodeList.pop()
                if node.__eq__(name,rank):
                    return True
                if DFS:
                    nodeList = node.children + nodeList
                else:
                    nodeList += node.children
            return False
        #
        #
    #Debugging function #1
    def printLineage(self,lineage):
        print "-- LINEAGE of %s %s --"%(self.name,self.rank)
        for x in lineage:
            print "(%s,%s)" %(x[0],x[1])
        print "---"
        #
        #
    #Debugging function #2
    def printChildren(self,children):
        print "-- CHILDREN of %s %s --"%(self.name,self.rank)
        for ch in children:
            print "(%s,%s,%d)" %(ch.name,ch.rank,ch.ident)
        print "---"
    #
    #
    def addNodePreProcess(self,paths,nodesList,samplesList,ranks):
        #Firstly, sorting nodes by decreasing rank: R < K < P < C < O < F < G < S
        sortedNodesList = sorted(nodesList,cmp=lambda x,y: compare(x[1],y[1]))
        nodesNumber = len(sortedNodesList)
        #Secondly, getting paths/lineage, and sampleHitList for every node
        pathsCopy = []
        n = 0
        for path in paths:
            n += 1
            pathsCopy.append(path)
        #@selectionPath is an optimized version of @selectPath for the construction of taxonomic trees
        pathsNodes = [ selectPath(pathsCopy,node[0],node[1],n) for node in sortedNodesList ]
        pathsNodesLength = len(pathsNodes)
        sampleHitListNodes = [ associatedData(nodesList,samplesList,node[0],node[1]) for node in sortedNodesList ]
        #Thirdly, clustering brothers (of same rank) into lists for each rank in @allBrotherList
        #Index of current node in sortedNodesList
        currNodeIdent = 0
        #@allBrotherList will contain lists of lists of nodes having the same father (and having same rank)
        allBrotherList = []
        #@hashBrotherList contains the (n,m) pairs such as if hashBrotherList[i] = (n,m), it means the (i+1)th node in sortedNodesList belongs to the (m+1)th subsublist of the (n+1)th sublist of allBrotherList list
        hashBrotherList = [ None ] * nodesNumber
        #@hashFatherList contains the (n,m) pairs such as if hashBrotherList[i] = (n,m), it means the (i+1)th node is the father of the nodes in the (m+1)th subsublist of the (n+1)th sublist of allBrotherList list
        hashFatherList = [ None ] * nodesNumber
        currN = -1
        for rank in ranks:
            currN += 1
            #@brotherNodesList will contain lists of nodes having the same father
            brotherNodesList = []
            currM = -1
            while (currNodeIdent < nodesNumber) and pathsNodes[currNodeIdent] and (sortedNodesList[currNodeIdent][1] == rank):
                #Father of the current node will be the last node in the lineage
                currFather = pathsNodes[currNodeIdent][-1]
                currFatherIdent = None
                for i in range(nodesNumber):
                    if (sortedNodesList[i] == currFather):
                        currFatherIdent = i
                currM += 1
                #Contains siblings nodes (their ident) and in first position the father
                thisRankNodesList = [ currFatherIdent, currNodeIdent ]
                hashBrotherList[currNodeIdent] = (currN,currM)
                currNodeIdent += 1
                hashFatherList[currFatherIdent] = (currN,currM)
                #Because root have no path
                if (currNodeIdent < pathsNodesLength) and pathsNodes[currNodeIdent]:
                    father = pathsNodes[currNodeIdent][-1]
                #If the rank of node indexed by currNodeIdent+1 is different from that of node indexed by currNodeIdent, the structure of taxonomic tree will ensure the two won't have the same father
                while (father == currFather) and (currNodeIdent < nodesNumber):
                    thisRankNodesList.append(currNodeIdent)
                    if (currNodeIdent < nodesNumber):
                        hashBrotherList[currNodeIdent] = (currN,currM)
                    currNodeIdent += 1
                    #Because root have no path (and no father)
                    if (currNodeIdent < pathsNodesLength) and pathsNodes[currNodeIdent]:
                        father = pathsNodes[currNodeIdent][-1]
                brotherNodesList.append(thisRankNodesList)
            if brotherNodesList:
                allBrotherList.append(brotherNodesList)
        return sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber
    #
    #
    #Starting from the leaves:
    #(loop1) For every rank r (S to R)
    #(1) Get all the nodes identifiers of same rank r
    #(loop2) For every node identifier n of this list:
    #(2) if n has no child (we can check this through @hashFatherList: if hashFatherList[n] is to None, then the corresponding node sortedNodesList[n] does not own any child): then construct its tree and store it in @constructedTree (@constructedTree[i] matches node @sortedNodesList[i])
    #(3) else: get n's children identifiers through @hashFatherList[n], get the children's trees, construct the tree associated to n, and store it in @constructedTree[n]
    #(4) Repeat loop2 until rank R and return tree associated to Root
    def addNodeAux(self,paths,sortedNodesList,pathsNodes,samplesHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber,ranks):
        #Initializing @constructedTree
        constructedTree = [ None ] * nodesNumber
        sLs = sortedNodesList[:: -1]
        ident = 0
        #loop1
        for r in ranks:
            #Step 1:
            sameRankedNodes = []
            name,rank = sLs.pop()
            while sLs and (rank == r):
                sameRankedNodes.append((name,rank,ident))
                ident += 1
                if sLs:
                    name,rank = sLs.pop()
            #Last node to be popped  has not the same rank as other
            #(if rank != r, then it is obvious, and if sLs is empty, it means root has been popped)
            sLs.append((name,rank))
            if (r == "R"):
                #Root is the only node of rank R
                sameRankedNodes.append((name,rank,ident))
            #loop2:
            while sameRankedNodes:
                nm,rk,idt = sameRankedNodes.pop()
                #Step 3:
                if hashFatherList[idt]:
                    n,m = hashFatherList[idt]
                    #First element is the father
                    #Get children identifiers
                    children = allBrotherList[n][m][1:]
                    childrenTrees = [ constructedTree[x] for x in children ]
                    constructedTree[idt] = TaxoTree(nm,rk,idt,samplesHitListNodes[idt],pathsNodes[idt],childrenTrees,paths)
                #Step 2:
                else:
                    constructedTree[idt] = TaxoTree(nm,rk,idt,samplesHitListNodes[idt],pathsNodes[idt],[],paths)
        constructedTree[-1].children = [ child for child in constructedTree[-1].children if child ]
        return constructedTree[-1]
    #
    #
    def addNode(self,paths,nodesList,ranks=["S","G","F","O","C","P","K","R"]):
        from time import time
        start = time()
        sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber = self.addNodePreProcess(paths,nodesList,samplesList,ranks)
        finalTree = self.addNodeAux(paths,sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber,ranks)
        end = time()
        print "TIME:",(end-start),"sec"
        return finalTree
    #
    #
    def iterate(self,f,DFS=False):
        f(self)
        nodeList = []
        for ch in self.children:
            nodeList.append(ch)
        while nodeList:
            child = nodeList.pop()
            f(child)
            if DFS:
                nodeList = child.children + nodeList
            else:
                nodeList = nodeList + child.children

def printTree(tree,sampleOn=False):
    print "ROOT"
    if not tree:
        #Should not happen...
        print "\n/!\ ERROR: Tree turned out to be None."
        raise ValueError
    print "(%s,%s,%d)"%(tree.name,tree.rank,tree.ident)
    if sampleOn:
        print "SAMPLE HIT LIST"
        print tree.sampleHitList
    nodeList = []
    for ch in tree.children:
        nodeList.append(ch)
    n = len(nodeList)
    print "CHILDREN"
    name = tree.name
    rank = tree.rank
    for i in range(n):
        print "**** child #%d of %s %s"%(i,name,rank)
        tree = nodeList[i]
        printTree(tree,sampleOn)
        if tree:
            print "END #%d %s %s of %s %s ****"%(i,tree.name,tree.rank,name,rank)
    print "---"
    return 0

<commit_msg>TaxoTree modification<commit_after>#The entire taxonomy tree
#Slightly modified version of TaxoTree, since sample Hit Lists are not useful anymore!

#A node (assignment of a read by Tango) in such a tree contains:
#@name the name of the species
#@rank its rank (k:Kingdom, p: Phylum, c: Class, o: Order, f: Family, g: Genus, s: Species) (Domain can be added easily if needed)
#@ident an identifier for the node, unique in the tree
#@lineage the list of ancestors ((name,rank) list) of the node (from the father to the root)
#@children the list of children (TaxoTree) of the node ordered by growing @ident
#@paths keeps memory of paths to every node/leaf of the tree (to compute more easily distances between nodes of a TaxoTree)
#Other identifier for a node (ident apart) is (@name,@rank)

#Nodes and leaves are only distinguinshed by children=[]

from parsingTree import parseTree
from parsingMatrix import parseMatrix,associatedData
from misc import containsSpecie,mem,selectPath,compare

class TaxoTree(object):
    #
    #
    def __init__(self,name="Root",rank="R",ident=0,lineage=None,children=None,paths=None):
        self.name = name
        self.rank = rank
        self.ident = ident
        self.lineage = lineage or []
        self.children = children or []
        self.paths = paths or []
        #
        #
    def __eq__(self,name,rank):
        return (self.name == name) and (self.rank == rank)
    #
    #
    #Returns the tree rooted to the node labelled by name, rank
    def search(self,name,rank,DFS=False):
        if self.__eq__(name,rank):
            return self
        else:
            nodeList = []
            for ch in self.children:
                nodeList.append(ch)
            while nodeList:
                node = nodeList.pop()
                if node.__eq__(name,rank):
                    return node
                if DFS:
                    nodeList = node.children + nodeList
                else:
                    nodeList += node.children
            print "\n/!\ ERROR: Subtree rooted at: (",name,rank,") not found. It probably means that this node is not assigned in the samples you have selected."
            raise ValueError
        #
        #
    def __isMember__(self,name,rank,DFS=False):
        if self.__eq__(name,rank):
            return True
        else:
            nodeList = self.children
            while nodeList:
                node = nodeList.pop()
                if node.__eq__(name,rank):
                    return True
                if DFS:
                    nodeList = node.children + nodeList
                else:
                    nodeList += node.children
            return False
        #
        #
    #Debugging function #1
    def printLineage(self,lineage):
        print "-- LINEAGE of %s %s --"%(self.name,self.rank)
        for x in lineage:
            print "(%s,%s)" %(x[0],x[1])
        print "---"
        #
        #
    #Debugging function #2
    def printChildren(self,children):
        print "-- CHILDREN of %s %s --"%(self.name,self.rank)
        for ch in children:
            print "(%s,%s,%d)" %(ch.name,ch.rank,ch.ident)
        print "---"
    #
    #
    def addNodePreProcess(self,paths,nodesList,samplesList,ranks):
        #Firstly, sorting nodes by decreasing rank: R < K < P < C < O < F < G < S
        sortedNodesList = sorted(nodesList,cmp=lambda x,y: compare(x[1],y[1]))
        nodesNumber = len(sortedNodesList)
        #Secondly, getting paths/lineage, and sampleHitList for every node
        pathsCopy = []
        n = 0
        for path in paths:
            n += 1
            pathsCopy.append(path)
        #@selectionPath is an optimized version of @selectPath for the construction of taxonomic trees
        pathsNodes = [ selectPath(pathsCopy,node[0],node[1],n) for node in sortedNodesList ]
        pathsNodesLength = len(pathsNodes)
        sampleHitListNodes = [ associatedData(nodesList,samplesList,node[0],node[1]) for node in sortedNodesList ]
        #Thirdly, clustering brothers (of same rank) into lists for each rank in @allBrotherList
        #Index of current node in sortedNodesList
        currNodeIdent = 0
        #@allBrotherList will contain lists of lists of nodes having the same father (and having same rank)
        allBrotherList = []
        #@hashBrotherList contains the (n,m) pairs such as if hashBrotherList[i] = (n,m), it means the (i+1)th node in sortedNodesList belongs to the (m+1)th subsublist of the (n+1)th sublist of allBrotherList list
        hashBrotherList = [ None ] * nodesNumber
        #@hashFatherList contains the (n,m) pairs such as if hashBrotherList[i] = (n,m), it means the (i+1)th node is the father of the nodes in the (m+1)th subsublist of the (n+1)th sublist of allBrotherList list
        hashFatherList = [ None ] * nodesNumber
        currN = -1
        for rank in ranks:
            currN += 1
            #@brotherNodesList will contain lists of nodes having the same father
            brotherNodesList = []
            currM = -1
            while (currNodeIdent < nodesNumber) and pathsNodes[currNodeIdent] and (sortedNodesList[currNodeIdent][1] == rank):
                #Father of the current node will be the last node in the lineage
                currFather = pathsNodes[currNodeIdent][-1]
                currFatherIdent = None
                for i in range(nodesNumber):
                    if (sortedNodesList[i] == currFather):
                        currFatherIdent = i
                currM += 1
                #Contains siblings nodes (their ident) and in first position the father
                thisRankNodesList = [ currFatherIdent, currNodeIdent ]
                hashBrotherList[currNodeIdent] = (currN,currM)
                currNodeIdent += 1
                hashFatherList[currFatherIdent] = (currN,currM)
                #Because root have no path
                if (currNodeIdent < pathsNodesLength) and pathsNodes[currNodeIdent]:
                    father = pathsNodes[currNodeIdent][-1]
                #If the rank of node indexed by currNodeIdent+1 is different from that of node indexed by currNodeIdent, the structure of taxonomic tree will ensure the two won't have the same father
                while (father == currFather) and (currNodeIdent < nodesNumber):
                    thisRankNodesList.append(currNodeIdent)
                    if (currNodeIdent < nodesNumber):
                        hashBrotherList[currNodeIdent] = (currN,currM)
                    currNodeIdent += 1
                    #Because root have no path (and no father)
                    if (currNodeIdent < pathsNodesLength) and pathsNodes[currNodeIdent]:
                        father = pathsNodes[currNodeIdent][-1]
                brotherNodesList.append(thisRankNodesList)
            if brotherNodesList:
                allBrotherList.append(brotherNodesList)
        return sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber
    #
    #
    #Starting from the leaves:
    #(loop1) For every rank r (S to R)
    #(1) Get all the nodes identifiers of same rank r
    #(loop2) For every node identifier n of this list:
    #(2) if n has no child (we can check this through @hashFatherList: if hashFatherList[n] is to None, then the corresponding node sortedNodesList[n] does not own any child): then construct its tree and store it in @constructedTree (@constructedTree[i] matches node @sortedNodesList[i])
    #(3) else: get n's children identifiers through @hashFatherList[n], get the children's trees, construct the tree associated to n, and store it in @constructedTree[n]
    #(4) Repeat loop2 until rank R and return tree associated to Root
    def addNodeAux(self,paths,sortedNodesList,pathsNodes,samplesHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber,ranks):
        #Initializing @constructedTree
        constructedTree = [ None ] * nodesNumber
        sLs = sortedNodesList[:: -1]
        ident = 0
        #loop1
        for r in ranks:
            #Step 1:
            sameRankedNodes = []
            name,rank = sLs.pop()
            while sLs and (rank == r):
                sameRankedNodes.append((name,rank,ident))
                ident += 1
                if sLs:
                    name,rank = sLs.pop()
            #Last node to be popped  has not the same rank as other
            #(if rank != r, then it is obvious, and if sLs is empty, it means root has been popped)
            sLs.append((name,rank))
            if (r == "R"):
                #Root is the only node of rank R
                sameRankedNodes.append((name,rank,ident))
            #loop2:
            while sameRankedNodes:
                nm,rk,idt = sameRankedNodes.pop()
                #Step 3:
                if hashFatherList[idt]:
                    n,m = hashFatherList[idt]
                    #First element is the father
                    #Get children identifiers
                    children = allBrotherList[n][m][1:]
                    childrenTrees = [ constructedTree[x] for x in children ]
                    constructedTree[idt] = TaxoTree(nm,rk,idt,samplesHitListNodes[idt],pathsNodes[idt],childrenTrees,paths)
                #Step 2:
                else:
                    constructedTree[idt] = TaxoTree(nm,rk,idt,samplesHitListNodes[idt],pathsNodes[idt],[],paths)
        constructedTree[-1].children = [ child for child in constructedTree[-1].children if child ]
        return constructedTree[-1]
    #
    #
    def addNode(self,paths,nodesList,ranks=["S","G","F","O","C","P","K","R"]):
        from time import time
        start = time()
        sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber = self.addNodePreProcess(paths,nodesList,samplesList,ranks)
        finalTree = self.addNodeAux(paths,sortedNodesList,pathsNodes,sampleHitListNodes,allBrotherList,hashBrotherList,hashFatherList,pathsNodesLength,nodesNumber,ranks)
        end = time()
        print "TIME:",(end-start),"sec"
        return finalTree
    #
    #
    def iterate(self,f,DFS=False):
        f(self)
        nodeList = []
        for ch in self.children:
            nodeList.append(ch)
        while nodeList:
            child = nodeList.pop()
            f(child)
            if DFS:
                nodeList = child.children + nodeList
            else:
                nodeList = nodeList + child.children

def printTree(tree,sampleOn=False):
    print "ROOT"
    if not tree:
        #Should not happen...
        print "\n/!\ ERROR: Tree turned out to be None."
        raise ValueError
    print "(%s,%s,%d)"%(tree.name,tree.rank,tree.ident)
    if sampleOn:
        print "SAMPLE HIT LIST"
        print tree.sampleHitList
    nodeList = []
    for ch in tree.children:
        nodeList.append(ch)
    n = len(nodeList)
    print "CHILDREN"
    name = tree.name
    rank = tree.rank
    for i in range(n):
        print "**** child #%d of %s %s"%(i,name,rank)
        tree = nodeList[i]
        printTree(tree,sampleOn)
        if tree:
            print "END #%d %s %s of %s %s ****"%(i,tree.name,tree.rank,name,rank)
    print "---"
    return 0

<|endoftext|>


<commit_before>"""Collection of decorators that caches a function's return value each time they
are called.

If called later with the same arguments, the cached value is returned, and
not re-evaluated. Slow for mutable types.

Taken from U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466320}.

"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2008 Gael Varoquaux
# License: BSD Style, 3 clauses.

import os
import warnings
from shelve import open as sopen
from cPickle import dumps, PicklingError
import inspect
import itertools
import traceback
import logging
from bsddb import _bsddb

def _function_code_hash(func):
    """ Attempts to retrieve a reliable function code hash.

        The reason we don't use inspect.getsource is that it caches the
        source, whereas we want this to be modified on the fly when the
        function is modified.
    """
    try:
        # Try to retrieve the source code.
        source_file = file(func.func_code.co_filename)
        first_line = func.func_code.co_firstlineno
        # All the lines after the function definition:
        source_lines = list(itertools.islice(source_file, first_line-1, None))
        return ''.join(inspect.getblock(source_lines))
    except:
        # If the source code fails, we use the hash. This is fragile and
        # might change from one session to another.
        return func.func_code.__hash__()

################################################################################
class MemoizeFunctor(object):
    """ Functor to decorate a function, caching its return value each time
    it is called. If called later with the same arguments, the cached
    value is returned, and not re-evaluated. Slow for mutable types.

    Cache is a file, hence memory is not lost even computer is shut down! :-)
    The file is syncronized anytime a value is added to it.

    Note
    ----

    Taken from U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466320}.

    """

    __slots__ = ['func', '_cache_filename', '_cache', '_cachedir', '_persist',
                    '_debug']

    def __init__(self, func, cachedir='cache', persist=True, debug=False):
        self.func = func
        self._persist = persist
        self._debug = debug
        if persist is False:
            if not cachedir=='cache':
                warnings.warn('cachedir specified, but persist is False;'
                             'not persisting', stacklevel=2)
            self._cache = dict()
        else:
            self._cachedir = cachedir
            if not os.path.exists(self._cachedir):
                os.makedirs(self._cachedir)
            cache_filename = os.path.join(self._cachedir,
                    '%s.%s.shelve' % (func.__module__, func.func_name))
            cache = self._cache = sopen(cache_filename, 'c', writeback=True)
            self._cache_filename = cache_filename
            # Compare the function code with the previous to see if the
            # function code has changed

            # I cannot use inspect.getsource because it is not
            # reliable when using IPython's magic "%run".
            func_code = _function_code_hash(func)
            try:
                if not ('__func_code__' in cache  and
                            cache['__func_code__'] == func_code):
                    self._cache_clear()
            except _bsddb.DBRunRecoveryError:
                self.warn('Unrecoverable DB error, clearing DB')
                self._cache = None
                os.unlink(self._cache_filename)
                self._cache = sopen(self._cache_filename, 'c')

    def _cache_clear(self):
        if self._debug:
            self.warn("Clearing cache")
        try:
            self._cache.clear()
        except KeyError:
            "DB not found: the db has probably not yet been created."
        self._cache['__func_code__'] = _function_code_hash(self.func)

    def warn(self, msg):
        logging.warn("[memoize]%s (%s line %i): %s" %
            ( self.func.func_name, self.func.__module__,
              self.func.func_code.co_firstlineno, msg)
                )

    def __call__(self, *args, **kwds):
        key = args
        if kwds:
            items = kwds.items()
            key = key + tuple(items)
        try:
            if key in self._cache:
                return self._cache[key]
            if self._debug:
                self.warn("Arguments not in cache.")
                self.print_call(*args, **kwds)
            self._cache[key] = result = self.func(*args, **kwds)
            if self._persist:
                # cache the result to file
                self._cache.sync()
            return result
        except TypeError:
            try:
                dump = dumps(key)
            except PicklingError:
                if self._debug:
                    self.warn("Cannot hash arguments.")
                    self.print_call(*args, **kwds)
                return self.func(*args, **kwds)
            else:
                try:
                    if dump in self._cache:
                        return self._cache[dump]
                    if self._debug:
                        self.warn("Arguments hash not in cache.")
                        self.print_call(*args, **kwds)
                except:
                    if self._debug:
                        self.warn("Error while unpickling for %s." % \
                                        self.func.func_name)
                        traceback.print_exc()
                    self._cache_clear()
                result = self.func(*args, **kwds)
                try:
                    self._cache[dump] = result
                except Exception, e:
                    if isinstance(e, _bsddb.DBRunRecoveryError):
                        self.warn('Unrecoverable DB error, clearing DB')
                        self._cache = None
                        os.unlink(self._cache_filename)
                        self._cache = sopen(self._cache_filename, 'c')
                        self._cache[dump] = result
                if self._persist:
                    # cache the result to file
                    self._cache.sync()
                return result

    def print_call(self, *args, **kwds):
        """ Print a debug statement displaying the function call with the
            arguments.
        """
        self.warn('Calling %s(%s, %s)' % (self.func.func_name,
                                    repr(args)[1:-1],
                                    ', '.join('%s=%s' % (v, i) for v, i
                                    in kwds.iteritems())))

    def clear(self):
        self._cache.clear()
        if self._persist:
            self._cache.sync()

def memoize(persist=True, cachedir='cache', debug=False):
    """ Decorator to cache the return values of a function each time
    it is called. If called later with the same arguments, the cached
    value is returned, and not re-evaluated. If the function code is modified,
    the cache is flushed.

    If persist is True, the cache is a file, hence memory is not lost even
    computer is shut down! :-) The file is syncronized anytime a value is
    added to it. This is slower, especially if mutable types are used.

    If debug is true, the different calls to the function are displayed
    when the function is actually called.
    """
    return lambda func: MemoizeFunctor(func,
                                    persist=persist, cachedir=cachedir,
                                    debug=debug)

<commit_msg>ENH: Make the code independant of bsddb, in case it becomes depreciated.<commit_after>"""Collection of decorators that caches a function's return value each time they
are called.

If called later with the same arguments, the cached value is returned, and
not re-evaluated. Slow for mutable types.

Taken from U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466320}.

"""

# Author: Gael Varoquaux <gael dot varoquaux at normalesup dot org>
# Copyright (c) 2008 Gael Varoquaux
# License: BSD Style, 3 clauses.

import os
import warnings
from shelve import open as sopen
from cPickle import dumps, PicklingError
import inspect
import itertools
import traceback
import logging
try:
    from bsddb import _bsddb
    DBError = _bsddb.DBError
except ImportError:
    # bsddb is not installed on every system.
    DBError = Exception

def _function_code_hash(func):
    """ Attempts to retrieve a reliable function code hash.

        The reason we don't use inspect.getsource is that it caches the
        source, whereas we want this to be modified on the fly when the
        function is modified.
    """
    try:
        # Try to retrieve the source code.
        source_file = file(func.func_code.co_filename)
        first_line = func.func_code.co_firstlineno
        # All the lines after the function definition:
        source_lines = list(itertools.islice(source_file, first_line-1, None))
        return ''.join(inspect.getblock(source_lines))
    except:
        # If the source code fails, we use the hash. This is fragile and
        # might change from one session to another.
        return func.func_code.__hash__()

################################################################################
class MemoizeFunctor(object):
    """ Functor to decorate a function, caching its return value each time
    it is called. If called later with the same arguments, the cached
    value is returned, and not re-evaluated. Slow for mutable types.

    Cache is a file, hence memory is not lost even computer is shut down! :-)
    The file is syncronized anytime a value is added to it.

    Note
    ----

    Taken from U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/466320}.

    """

    __slots__ = ['func', '_cache_filename', '_cache', '_cachedir', '_persist',
                    '_debug']

    def __init__(self, func, cachedir='cache', persist=True, debug=False):
        self.func = func
        self._persist = persist
        self._debug = debug
        if persist is False:
            if not cachedir=='cache':
                warnings.warn('cachedir specified, but persist is False;'
                             'not persisting', stacklevel=2)
            self._cache = dict()
        else:
            self._cachedir = cachedir
            if not os.path.exists(self._cachedir):
                os.makedirs(self._cachedir)
            cache_filename = os.path.join(self._cachedir,
                    '%s.%s.shelve' % (func.__module__, func.func_name))
            cache = self._cache = sopen(cache_filename, 'c', writeback=True)
            self._cache_filename = cache_filename
            # Compare the function code with the previous to see if the
            # function code has changed

            # I cannot use inspect.getsource because it is not
            # reliable when using IPython's magic "%run".
            func_code = _function_code_hash(func)
            try:
                if not ('__func_code__' in cache  and
                            cache['__func_code__'] == func_code):
                    self._cache_clear()
            except DBError:
                self.warn('Unrecoverable DB error, clearing DB')
                self._cache = None
                os.unlink(self._cache_filename)
                self._cache = sopen(self._cache_filename, 'c')

    def _cache_clear(self):
        if self._debug:
            self.warn("Clearing cache")
        try:
            self._cache.clear()
        except KeyError:
            "DB not found: the db has probably not yet been created."
        self._cache['__func_code__'] = _function_code_hash(self.func)

    def warn(self, msg):
        logging.warn("[memoize]%s (%s line %i): %s" %
            ( self.func.func_name, self.func.__module__,
              self.func.func_code.co_firstlineno, msg)
                )

    def __call__(self, *args, **kwds):
        key = args
        if kwds:
            items = kwds.items()
            key = key + tuple(items)
        try:
            if key in self._cache:
                return self._cache[key]
            if self._debug:
                self.warn("Arguments not in cache.")
                self.print_call(*args, **kwds)
            self._cache[key] = result = self.func(*args, **kwds)
            if self._persist:
                # cache the result to file
                self._cache.sync()
            return result
        except TypeError:
            try:
                dump = dumps(key)
            except PicklingError:
                if self._debug:
                    self.warn("Cannot hash arguments.")
                    self.print_call(*args, **kwds)
                return self.func(*args, **kwds)
            else:
                try:
                    if dump in self._cache:
                        return self._cache[dump]
                    if self._debug:
                        self.warn("Arguments hash not in cache.")
                        self.print_call(*args, **kwds)
                except:
                    if self._debug:
                        self.warn("Error while unpickling for %s." % \
                                        self.func.func_name)
                        traceback.print_exc()
                    self._cache_clear()
                result = self.func(*args, **kwds)
                try:
                    self._cache[dump] = result
                except Exception, e:
                    if isinstance(e, DBError):
                        self.warn('Unrecoverable DB error, clearing DB')
                        self._cache = None
                        os.unlink(self._cache_filename)
                        self._cache = sopen(self._cache_filename, 'c')
                        self._cache[dump] = result
                if self._persist:
                    # cache the result to file
                    self._cache.sync()
                return result

    def print_call(self, *args, **kwds):
        """ Print a debug statement displaying the function call with the
            arguments.
        """
        self.warn('Calling %s(%s, %s)' % (self.func.func_name,
                                    repr(args)[1:-1],
                                    ', '.join('%s=%s' % (v, i) for v, i
                                    in kwds.iteritems())))

    def clear(self):
        self._cache.clear()
        if self._persist:
            self._cache.sync()

def memoize(persist=True, cachedir='cache', debug=False):
    """ Decorator to cache the return values of a function each time
    it is called. If called later with the same arguments, the cached
    value is returned, and not re-evaluated. If the function code is modified,
    the cache is flushed.

    If persist is True, the cache is a file, hence memory is not lost even
    computer is shut down! :-) The file is syncronized anytime a value is
    added to it. This is slower, especially if mutable types are used.

    If debug is true, the different calls to the function are displayed
    when the function is actually called.
    """
    return lambda func: MemoizeFunctor(func,
                                    persist=persist, cachedir=cachedir,
                                    debug=debug)

<|endoftext|>