<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# maxminddb documentation build configuration file, created by
# sphinx-quickstart on Tue Apr  9 13:34:57 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration -----------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
              'sphinx.ext.intersphinx', 'sphinx.ext.coverage']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'maxminddb'
copyright = '2013, Gregory Oschwald'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = maxminddb.__version__
# The full version, including alpha/beta/rc tags.
release = maxminddb.__version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'maxminddbdoc'

# -- Options for LaTeX output --------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('index', 'maxminddb.tex', 'maxminddb Documentation',
     'Gregory Oschwald', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'maxminddb', 'maxminddb Documentation',
     ['Gregory Oschwald'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'maxminddb', 'maxminddb Documentation',
     'Gregory Oschwald', 'maxminddb', 'MaxMind DB Reader',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
<commit_msg>Manually set version in conf.py for now<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# maxminddb documentation build configuration file, created by
# sphinx-quickstart on Tue Apr  9 13:34:57 2013.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os

__version__ = 0.1.0

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('..'))

# -- General configuration -----------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest',
              'sphinx.ext.intersphinx', 'sphinx.ext.coverage']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'maxminddb'
copyright = '2013, Gregory Oschwald'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = __version__
# The full version, including alpha/beta/rc tags.
release = __version__

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'sphinxdoc'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'maxminddbdoc'

# -- Options for LaTeX output --------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
    ('index', 'maxminddb.tex', 'maxminddb Documentation',
     'Gregory Oschwald', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output --------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'maxminddb', 'maxminddb Documentation',
     ['Gregory Oschwald'], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output ------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'maxminddb', 'maxminddb Documentation',
     'Gregory Oschwald', 'maxminddb', 'MaxMind DB Reader',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'http://docs.python.org/': None}
<|endoftext|>


<commit_before>import threading
from datetime import datetime
from functools import lru_cache
from typing import Any

import zmq
from zmq.backend.cython.constants import NOBLOCK

from .common import HEARTBEAT_TOPIC, HEARTBEAT_TOLERANCE

class RemoteException(Exception):
    """
    RPC remote exception
    """

    def __init__(self, value: Any) -> None:
        """
        Constructor
        """
        self._value = value

    def __str__(self) -> str:
        """
        Output error message
        """
        return self._value

class RpcClient:
    """"""

    def __init__(self) -> None:
        """Constructor"""
        # zmq port related
        self._context: zmq.Context = zmq.Context()

        # Request socket (Request–reply pattern)
        self._socket_req: zmq.Socket = self._context.socket(zmq.REQ)

        # Subscribe socket (Publish–subscribe pattern)
        self._socket_sub: zmq.Socket = self._context.socket(zmq.SUB)

        # Worker thread relate, used to process data pushed from server
        self._active: bool = False                 # RpcClient status
        self._thread: threading.Thread = None      # RpcClient thread
        self._lock: threading.Lock = threading.Lock()

        self._last_received_ping: datetime = datetime.utcnow()

    @lru_cache(100)
    def __getattr__(self, name: str) -> Any:
        """
        Realize remote call function
        """

        # Perform remote call task
        def dorpc(*args, **kwargs):
            # Get timeout value from kwargs, default value is 30 seconds
            if "timeout" in kwargs:
                timeout = kwargs.pop("timeout")
            else:
                timeout = 30000

            # Generate request
            req: list = [name, args, kwargs]

            # Send request and wait for response
            with self._lock:
                self._socket_req.send_pyobj(req)

                # Timeout reached without any data
                n: int = self._socket_req.poll(timeout)
                if not n:
                    msg: str = f"Timeout of {timeout}ms reached for {req}"
                    raise RemoteException(msg)

                rep = self._socket_req.recv_pyobj()

            # Return response if successed; Trigger exception if failed
            if rep[0]:
                return rep[1]
            else:
                raise RemoteException(rep[1])

        return dorpc

    def start(
        self,
        req_address: str,
        sub_address: str
    ) -> None:
        """
        Start RpcClient
        """
        if self._active:
            return

        # Connect zmq port
        self._socket_req.connect(req_address)
        self._socket_sub.connect(sub_address)

        # Start RpcClient status
        self._active = True

        # Start RpcClient thread
        self._thread = threading.Thread(target=self.run)
        self._thread.start()

        self._last_received_ping = datetime.utcnow()

    def stop(self) -> None:
        """
        Stop RpcClient
        """
        if not self._active:
            return

        # Stop RpcClient status
        self._active = False

    def join(self) -> None:
        # Wait for RpcClient thread to exit
        if self._thread and self._thread.is_alive():
            self._thread.join()
        self._thread = None

    def run(self) -> None:
        """
        Run RpcClient function
        """
        pull_tolerance: int = HEARTBEAT_TOLERANCE * 1000

        while self._active:
            if not self._socket_sub.poll(pull_tolerance):
                self.on_disconnected()
                continue

            # Receive data from subscribe socket
            topic, data = self._socket_sub.recv_pyobj(flags=NOBLOCK)

            if topic == HEARTBEAT_TOPIC:
                self._last_received_ping = data
            else:
                # Process data by callable function
                self.callback(topic, data)

        # Close socket
        self._socket_req.close()
        self._socket_sub.close()

    def callback(self, topic: str, data: Any) -> None:
        """
        Callable function
        """
        raise NotImplementedError

    def subscribe_topic(self, topic: str) -> None:
        """
        Subscribe data
        """
        self._socket_sub.setsockopt_string(zmq.SUBSCRIBE, topic)

    def on_disconnected(self):
        """
        Callback when heartbeat is lost.
        """
        msg: str = f"RpcServer has no response over {HEARTBEAT_TOLERANCE} seconds, please check you connection."
        print(msg)
<commit_msg>pyzmq23及以上版本报错的解决办法<commit_after>import threading
from datetime import datetime
from functools import lru_cache
from typing import Any

import zmq
#from zmq.backend.cython.constants import NOBLOCK     pyzmq23以及以后的版本会报错
from zmq import NOBLOCK    #适用于pyzmq23以及以上的版本
from .common import HEARTBEAT_TOPIC, HEARTBEAT_TOLERANCE

class RemoteException(Exception):
    """
    RPC remote exception
    """

    def __init__(self, value: Any) -> None:
        """
        Constructor
        """
        self._value = value

    def __str__(self) -> str:
        """
        Output error message
        """
        return self._value

class RpcClient:
    """"""

    def __init__(self) -> None:
        """Constructor"""
        # zmq port related
        self._context: zmq.Context = zmq.Context()

        # Request socket (Request–reply pattern)
        self._socket_req: zmq.Socket = self._context.socket(zmq.REQ)

        # Subscribe socket (Publish–subscribe pattern)
        self._socket_sub: zmq.Socket = self._context.socket(zmq.SUB)

        # Worker thread relate, used to process data pushed from server
        self._active: bool = False                 # RpcClient status
        self._thread: threading.Thread = None      # RpcClient thread
        self._lock: threading.Lock = threading.Lock()

        self._last_received_ping: datetime = datetime.utcnow()

    @lru_cache(100)
    def __getattr__(self, name: str) -> Any:
        """
        Realize remote call function
        """

        # Perform remote call task
        def dorpc(*args, **kwargs):
            # Get timeout value from kwargs, default value is 30 seconds
            if "timeout" in kwargs:
                timeout = kwargs.pop("timeout")
            else:
                timeout = 30000

            # Generate request
            req: list = [name, args, kwargs]

            # Send request and wait for response
            with self._lock:
                self._socket_req.send_pyobj(req)

                # Timeout reached without any data
                n: int = self._socket_req.poll(timeout)
                if not n:
                    msg: str = f"Timeout of {timeout}ms reached for {req}"
                    raise RemoteException(msg)

                rep = self._socket_req.recv_pyobj()

            # Return response if successed; Trigger exception if failed
            if rep[0]:
                return rep[1]
            else:
                raise RemoteException(rep[1])

        return dorpc

    def start(
        self,
        req_address: str,
        sub_address: str
    ) -> None:
        """
        Start RpcClient
        """
        if self._active:
            return

        # Connect zmq port
        self._socket_req.connect(req_address)
        self._socket_sub.connect(sub_address)

        # Start RpcClient status
        self._active = True

        # Start RpcClient thread
        self._thread = threading.Thread(target=self.run)
        self._thread.start()

        self._last_received_ping = datetime.utcnow()

    def stop(self) -> None:
        """
        Stop RpcClient
        """
        if not self._active:
            return

        # Stop RpcClient status
        self._active = False

    def join(self) -> None:
        # Wait for RpcClient thread to exit
        if self._thread and self._thread.is_alive():
            self._thread.join()
        self._thread = None

    def run(self) -> None:
        """
        Run RpcClient function
        """
        pull_tolerance: int = HEARTBEAT_TOLERANCE * 1000

        while self._active:
            if not self._socket_sub.poll(pull_tolerance):
                self.on_disconnected()
                continue

            # Receive data from subscribe socket
            topic, data = self._socket_sub.recv_pyobj(flags=NOBLOCK)

            if topic == HEARTBEAT_TOPIC:
                self._last_received_ping = data
            else:
                # Process data by callable function
                self.callback(topic, data)

        # Close socket
        self._socket_req.close()
        self._socket_sub.close()

    def callback(self, topic: str, data: Any) -> None:
        """
        Callable function
        """
        raise NotImplementedError

    def subscribe_topic(self, topic: str) -> None:
        """
        Subscribe data
        """
        self._socket_sub.setsockopt_string(zmq.SUBSCRIBE, topic)

    def on_disconnected(self):
        """
        Callback when heartbeat is lost.
        """
        msg: str = f"RpcServer has no response over {HEARTBEAT_TOLERANCE} seconds, please check you connection."
        print(msg)
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-12-28 21:12
from hanlp_common.constant import HANLP_URL

SIGHAN2005_PKU_CONVSEG = HANLP_URL + 'tok/sighan2005-pku-convseg_20200110_153722.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on sighan2005 pku dataset.'
SIGHAN2005_MSR_CONVSEG = HANLP_URL + 'tok/convseg-msr-nocrf-noembed_20200110_153524.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on sighan2005 msr dataset.'
CTB6_CONVSEG = HANLP_URL + 'tok/ctb6_convseg_nowe_nocrf_20200110_004046.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on CTB6 dataset.'
PKU_NAME_MERGED_SIX_MONTHS_CONVSEG = HANLP_URL + 'tok/pku98_6m_conv_ngram_20200110_134736.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on pku98 six months dataset with familiy name and given name merged into one unit.'
LARGE_ALBERT_BASE = HANLP_URL + 'tok/large_corpus_cws_albert_base_20211228_160926.zip'
'ALBERT model (:cite:`Lan2020ALBERT:`) trained on the largest CWS dataset in the world.'
SIGHAN2005_PKU_BERT_BASE_ZH = HANLP_URL + 'tok/sighan2005_pku_bert_base_zh_20201231_141130.zip'
'BERT model (:cite:`devlin-etal-2019-bert`) trained on sighan2005 pku dataset.'
COARSE_ELECTRA_SMALL_ZH = HANLP_URL + 'tok/coarse_electra_small_zh_20210603_112321.zip'
'Electra (:cite:`clark2020electra`) small model trained on coarse-grained CWS corpora. Its performance is P=97.08% R=96.94% F1=97.01% which is ' \
'much higher than that of MTL model '

# Will be filled up during runtime
ALL = {}
<commit_msg>Release a CTB9 tok model<commit_after># -*- coding:utf-8 -*-
# Author: hankcs
# Date: 2019-12-28 21:12
from hanlp_common.constant import HANLP_URL

SIGHAN2005_PKU_CONVSEG = HANLP_URL + 'tok/sighan2005-pku-convseg_20200110_153722.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on sighan2005 pku dataset.'
SIGHAN2005_MSR_CONVSEG = HANLP_URL + 'tok/convseg-msr-nocrf-noembed_20200110_153524.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on sighan2005 msr dataset.'
CTB6_CONVSEG = HANLP_URL + 'tok/ctb6_convseg_nowe_nocrf_20200110_004046.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on CTB6 dataset.'
PKU_NAME_MERGED_SIX_MONTHS_CONVSEG = HANLP_URL + 'tok/pku98_6m_conv_ngram_20200110_134736.zip'
'Conv model (:cite:`wang-xu-2017-convolutional`) trained on pku98 six months dataset with familiy name and given name merged into one unit.'
LARGE_ALBERT_BASE = HANLP_URL + 'tok/large_corpus_cws_albert_base_20211228_160926.zip'
'ALBERT model (:cite:`Lan2020ALBERT:`) trained on the largest CWS dataset in the world.'
SIGHAN2005_PKU_BERT_BASE_ZH = HANLP_URL + 'tok/sighan2005_pku_bert_base_zh_20201231_141130.zip'
'BERT model (:cite:`devlin-etal-2019-bert`) trained on sighan2005 pku dataset.'
COARSE_ELECTRA_SMALL_ZH = HANLP_URL + 'tok/coarse_electra_small_zh_20210603_112321.zip'
'Electra (:cite:`clark2020electra`) small model trained on coarse-grained CWS corpora. Its performance is P=97.08% R=96.94% F1=97.01% which is ' \
'much higher than that of MTL model '
CTB9_TOK_ELECTRA_SMALL = HANLP_URL + 'tok/ctb9_electra_small_20220204_184813.zip'
'Electra (:cite:`clark2020electra`) small model trained on CTB9. Its performance is  P=97.23% R=97.38% F1=97.31% which is ' \
'much higher than that of MTL model '

# Will be filled up during runtime
ALL = {}
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from flask import current_app as app, json, g
from eve_elastic.elastic import set_filters
from copy import deepcopy

import superdesk

from superdesk import get_resource_service
from superdesk.metadata.item import CONTENT_STATE, ITEM_STATE, get_schema
from superdesk.metadata.utils import aggregations as common_aggregations, item_url, get_elastic_highlight_query
from apps.archive.archive import SOURCE as ARCHIVE
from superdesk.resource import build_custom_hateoas
from apps.publish.published_item import published_item_fields
from superdesk import es_utils

class SearchService(superdesk.Service):
    """Federated search service.

    It can search against different collections like Ingest, Production, Archived etc.. at the same time.
    """

    repos = None
    aggregations = deepcopy(common_aggregations)

    @property
    def elastic(self):
        return app.data.elastic

    def __init__(self, datasource, backend):
        super().__init__(datasource=datasource, backend=backend)

    def get_ingest_filters(self):
        """
        Returns Ingest filters
        """
        return [{'term': {'_type': 'ingest'}}]

    def get_archive_filters(self):
        """
        Returns Archive filters filters

        If the content state is draft, it must be from the current user
        """
        user_id = (g.get('user') or {}).get('_id')
        return [
            {'exists': {'field': 'task.desk'}},
            {
                'bool': {
                    'should': [
                        {
                            'and': [
                                {'term': {ITEM_STATE: CONTENT_STATE.DRAFT}},
                                {'term': {'task.user': str(user_id)}}
                            ]
                        },
                        {
                            'terms': {
                                ITEM_STATE: [
                                    CONTENT_STATE.FETCHED,
                                    CONTENT_STATE.ROUTED,
                                    CONTENT_STATE.PROGRESS,
                                    CONTENT_STATE.SUBMITTED,
                                    CONTENT_STATE.SPIKED
                                ]
                            }
                        }
                    ],
                    'must_not': {'term': {'version': 0}},
                    'minimum_should_match': 1,
                }
            }
        ]

    def get_published_filters(self):
        """
        Returns published filters
        """
        return [
            {'term': {'_type': 'published'}},
            {'terms': {
                ITEM_STATE: [
                    CONTENT_STATE.SCHEDULED,
                    CONTENT_STATE.PUBLISHED,
                    CONTENT_STATE.KILLED,
                    CONTENT_STATE.RECALLED,
                    CONTENT_STATE.CORRECTED,
                    CONTENT_STATE.UNPUBLISHED,
                ]
            }
            }
        ]

    def get_archived_filters(self):
        """
        Returns archived filters
        """
        return [{'term': {'_type': 'archived'}}]

    def _get_private_filters(self, repo, invisible_stages):
        query = {'and': []}

        if repo == 'ingest':
            query['and'].extend(self.get_ingest_filters())
        elif repo == 'archive':
            query['and'].extend(self.get_archive_filters())
        elif repo == 'published':
            query['and'].extend(self.get_published_filters())
        elif repo == 'archived':
            query['and'].extend(self.get_archived_filters())

        if invisible_stages and repo != 'ingest':
            query['and'].append({'not': {'terms': {'task.stage': invisible_stages}}})

        return query

    def _get_query(self, req):
        """Get elastic query."""
        args = getattr(req, 'args', {})
        query = json.loads(args.get('source')) if args.get('source') else {'query': {'filtered': {}}}
        if app.data.elastic.should_aggregate(req):
            query['aggs'] = self.aggregations

        if app.data.elastic.should_highlight(req):
            highlight_query = get_elastic_highlight_query(self._get_highlight_query_string(req))
            if highlight_query:
                query['highlight'] = highlight_query
        return query

    def _get_highlight_query_string(self, req):
        args = getattr(req, 'args', {})
        source = json.loads(args.get('source')) if args.get('source') else {'query': {'filtered': {}}}
        query_string = source.get('query', {}).get('filtered', {}).get('query', {}).get('query_string')
        return query_string

    def _get_projected_fields(self, req):
        """Get elastic projected fields."""
        if app.data.elastic.should_project(req):
            return app.data.elastic.get_projected_fields(req)

    def _get_types(self, req):
        """Get document types for the given query."""
        args = getattr(req, 'args', {})
        repos = args.get('repo')
        return es_utils.get_doc_types(repos, self.repos)

    def _get_filters(self, repos, invisible_stages):
        """
        Gets filters for the passed repos.
        """
        filters = []

        for repo in repos:
            filters.append(self._get_private_filters(repo, invisible_stages))

        return [{'or': filters}] if filters else []

    def get_stages_to_exclude(self):
        """
        Returns the list of the current users invisible stages
        """
        user = g.get('user', {})
        if 'invisible_stages' in user:
            stages = user.get('invisible_stages')
        else:
            stages = get_resource_service('users').get_invisible_stages_ids(user.get('_id'))

        return stages

    def get(self, req, lookup):
        """
        Runs elastic search on multiple doc types.
        """

        query = self._get_query(req)
        fields = self._get_projected_fields(req)
        types = self._get_types(req)
        excluded_stages = self.get_stages_to_exclude()
        filters = self._get_filters(types, excluded_stages)

        # if the system has a setting value for the maximum search depth then apply the filter
        if not app.settings['MAX_SEARCH_DEPTH'] == -1:
            query['terminate_after'] = app.settings['MAX_SEARCH_DEPTH']

        if filters:
            set_filters(query, filters)

        params = {}
        if fields:
            params['_source'] = fields

        docs = self.elastic.search(query, types, params)

        for resource in types:
            response = {app.config['ITEMS']: [doc for doc in docs if doc['_type'] == resource]}
            getattr(app, 'on_fetched_resource')(resource, response)
            getattr(app, 'on_fetched_resource_%s' % resource)(response)

        return docs

    def _get_docs(self, hits):
        """Parse hits from elastic and return only docs.

        This will remove some extra metadata from elastic.

        :param hits: elastic hits dictionary
        """
        return self.elastic._parse_hits(hits, 'ingest')  # any resource with item schema will do

    def find_one(self, req, **lookup):
        """Find item by id in all collections."""
        _id = lookup['_id']
        for resource in self._get_types(req):
            id_field = 'item_id' if resource == 'published' else '_id'
            resource_lookup = {id_field: _id}
            item = get_resource_service(resource).find_one(req=req, **resource_lookup)
            if item:
                return item

    def on_fetched(self, doc):
        """
        Overriding to add HATEOS for each individual item in the response.

        :param doc: response doc
        :type doc: dict
        """

        docs = doc[app.config['ITEMS']]
        for item in docs:
            build_custom_hateoas({'self': {'title': item['_type'], 'href': '/{}/{{_id}}'.format(item['_type'])}}, item)

    def get_available_indexes(self):
        """Returns a set of the configured indexes

        :return:
        """
        return set(es_utils.get_index().split(','))

class SearchResource(superdesk.Resource):
    resource_methods = ['GET']
    item_methods = ['GET']
    item_url = item_url
    schema = get_schema(versioning=True)
    schema.update(published_item_fields)

def init_app(app):
    search_service = SearchService(ARCHIVE, backend=superdesk.get_backend())
    SearchResource('search', app=app, service=search_service)

    # Set the start of week config for use in both server and client
    app.client_config['start_of_week'] = app.config.get('START_OF_WEEK') or 0
<commit_msg>Include all schema items in default search projection + archive_item (#1913)<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license

from flask import current_app as app, json, g
from eve_elastic.elastic import set_filters
from copy import deepcopy

import superdesk

from superdesk import get_resource_service
from superdesk.metadata.item import CONTENT_STATE, ITEM_STATE, get_schema
from superdesk.metadata.utils import aggregations as common_aggregations, item_url, get_elastic_highlight_query
from apps.archive.archive import SOURCE as ARCHIVE
from superdesk.resource import build_custom_hateoas
from apps.publish.published_item import published_item_fields
from superdesk import es_utils

class SearchService(superdesk.Service):
    """Federated search service.

    It can search against different collections like Ingest, Production, Archived etc.. at the same time.
    """

    repos = None
    aggregations = deepcopy(common_aggregations)

    @property
    def elastic(self):
        return app.data.elastic

    def __init__(self, datasource, backend):
        super().__init__(datasource=datasource, backend=backend)

    def get_ingest_filters(self):
        """
        Returns Ingest filters
        """
        return [{'term': {'_type': 'ingest'}}]

    def get_archive_filters(self):
        """
        Returns Archive filters filters

        If the content state is draft, it must be from the current user
        """
        user_id = (g.get('user') or {}).get('_id')
        return [
            {'exists': {'field': 'task.desk'}},
            {
                'bool': {
                    'should': [
                        {
                            'and': [
                                {'term': {ITEM_STATE: CONTENT_STATE.DRAFT}},
                                {'term': {'task.user': str(user_id)}}
                            ]
                        },
                        {
                            'terms': {
                                ITEM_STATE: [
                                    CONTENT_STATE.FETCHED,
                                    CONTENT_STATE.ROUTED,
                                    CONTENT_STATE.PROGRESS,
                                    CONTENT_STATE.SUBMITTED,
                                    CONTENT_STATE.SPIKED
                                ]
                            }
                        }
                    ],
                    'must_not': {'term': {'version': 0}},
                    'minimum_should_match': 1,
                }
            }
        ]

    def get_published_filters(self):
        """
        Returns published filters
        """
        return [
            {'term': {'_type': 'published'}},
            {'terms': {
                ITEM_STATE: [
                    CONTENT_STATE.SCHEDULED,
                    CONTENT_STATE.PUBLISHED,
                    CONTENT_STATE.KILLED,
                    CONTENT_STATE.RECALLED,
                    CONTENT_STATE.CORRECTED,
                    CONTENT_STATE.UNPUBLISHED,
                ]
            }
            }
        ]

    def get_archived_filters(self):
        """
        Returns archived filters
        """
        return [{'term': {'_type': 'archived'}}]

    def _get_private_filters(self, repo, invisible_stages):
        query = {'and': []}

        if repo == 'ingest':
            query['and'].extend(self.get_ingest_filters())
        elif repo == 'archive':
            query['and'].extend(self.get_archive_filters())
        elif repo == 'published':
            query['and'].extend(self.get_published_filters())
        elif repo == 'archived':
            query['and'].extend(self.get_archived_filters())

        if invisible_stages and repo != 'ingest':
            query['and'].append({'not': {'terms': {'task.stage': invisible_stages}}})

        return query

    def _get_query(self, req):
        """Get elastic query."""
        args = getattr(req, 'args', {})
        query = json.loads(args.get('source')) if args.get('source') else {'query': {'filtered': {}}}
        if app.data.elastic.should_aggregate(req):
            query['aggs'] = self.aggregations

        if app.data.elastic.should_highlight(req):
            highlight_query = get_elastic_highlight_query(self._get_highlight_query_string(req))
            if highlight_query:
                query['highlight'] = highlight_query
        return query

    def _get_highlight_query_string(self, req):
        args = getattr(req, 'args', {})
        source = json.loads(args.get('source')) if args.get('source') else {'query': {'filtered': {}}}
        query_string = source.get('query', {}).get('filtered', {}).get('query', {}).get('query_string')
        return query_string

    def _get_projected_fields(self, req):
        """Get elastic projected fields."""
        if app.data.elastic.should_project(req):
            return app.data.elastic.get_projected_fields(req)

    def _get_types(self, req):
        """Get document types for the given query."""
        args = getattr(req, 'args', {})
        repos = args.get('repo')
        return es_utils.get_doc_types(repos, self.repos)

    def _get_filters(self, repos, invisible_stages):
        """
        Gets filters for the passed repos.
        """
        filters = []

        for repo in repos:
            filters.append(self._get_private_filters(repo, invisible_stages))

        return [{'or': filters}] if filters else []

    def get_stages_to_exclude(self):
        """
        Returns the list of the current users invisible stages
        """
        user = g.get('user', {})
        if 'invisible_stages' in user:
            stages = user.get('invisible_stages')
        else:
            stages = get_resource_service('users').get_invisible_stages_ids(user.get('_id'))

        return stages

    def get(self, req, lookup):
        """
        Runs elastic search on multiple doc types.
        """

        query = self._get_query(req)
        fields = self._get_projected_fields(req)
        types = self._get_types(req)
        excluded_stages = self.get_stages_to_exclude()
        filters = self._get_filters(types, excluded_stages)

        # if the system has a setting value for the maximum search depth then apply the filter
        if not app.settings['MAX_SEARCH_DEPTH'] == -1:
            query['terminate_after'] = app.settings['MAX_SEARCH_DEPTH']

        if filters:
            set_filters(query, filters)

        params = {}
        if fields:
            params['_source'] = fields

        docs = self.elastic.search(query, types, params)

        for resource in types:
            response = {app.config['ITEMS']: [doc for doc in docs if doc['_type'] == resource]}
            getattr(app, 'on_fetched_resource')(resource, response)
            getattr(app, 'on_fetched_resource_%s' % resource)(response)

        return docs

    def _get_docs(self, hits):
        """Parse hits from elastic and return only docs.

        This will remove some extra metadata from elastic.

        :param hits: elastic hits dictionary
        """
        return self.elastic._parse_hits(hits, 'ingest')  # any resource with item schema will do

    def find_one(self, req, **lookup):
        """Find item by id in all collections."""
        _id = lookup['_id']
        for resource in self._get_types(req):
            id_field = 'item_id' if resource == 'published' else '_id'
            resource_lookup = {id_field: _id}
            item = get_resource_service(resource).find_one(req=req, **resource_lookup)
            if item:
                return item

    def on_fetched(self, doc):
        """
        Overriding to add HATEOS for each individual item in the response.

        :param doc: response doc
        :type doc: dict
        """

        docs = doc[app.config['ITEMS']]
        for item in docs:
            build_custom_hateoas({'self': {'title': item['_type'], 'href': '/{}/{{_id}}'.format(item['_type'])}}, item)

    def get_available_indexes(self):
        """Returns a set of the configured indexes

        :return:
        """
        return set(es_utils.get_index().split(','))

class SearchResource(superdesk.Resource):
    resource_methods = ['GET']
    item_methods = ['GET']
    item_url = item_url
    schema = get_schema(versioning=True)
    schema.update(published_item_fields)
    datasource = {
        'projection': {field: 1 for field in list(schema.keys()) + ['archive_item']}
    }

def init_app(app):
    search_service = SearchService(ARCHIVE, backend=superdesk.get_backend())
    SearchResource('search', app=app, service=search_service)

    # Set the start of week config for use in both server and client
    app.client_config['start_of_week'] = app.config.get('START_OF_WEEK') or 0
<|endoftext|>


<commit_before>"""\
Move files from "uploading" to "in progress".

Example:

    %(prog)s --username ACCESS_KEY_ID --password SECRET_ACCESS_KEY \\
        --input check_files.log https://www.encodeproject.org
"""

import json
import requests
import sys
try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

EPILOG = __doc__

HEADERS = {
    'Accept': 'application/json',
    'Content-Type': 'application/json',
}

def run(fp, url, username, password):
    for line in fp.readlines():
        item, result, errors = json.loads(line)
        if errors:
            print('skipped %s: %s' % (item['accession'], errors))
            continue
        item_url = urljoin(url, item['@id'])
        r = requests.get(
            item_url + '?frame=object&datastore=database',
            auth=(username, password),
            headers=HEADERS,
        )
        if r.status_code != 200:
            print('ERROR retrieving %s: %s %s' % (item['@id'], r.status_code, r.reason))
            print(r.text)
            continue
        if r.json()['status'] != 'uploading':
            print('skipped %s: status %r is not "uploading"' % (item['@id'], r.json()['status']))
            continue
        r = requests.patch(
            item_url,
            data=json.dumps({
                'status': 'in progress',
                'file_size': result['file_size'],
                'content_md5sum': result['content_md5sum']
            }),
            auth=(username, password),
            headers=HEADERS,
        )
        if r.status_code != 200:
            print('ERROR patching %s: %s %s' % (item['@id'], r.status_code, r.reason))
            print(r.text)
            continue
        print('patched %s' % item['@id'])

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update file status", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--username', '-u', default='', help="HTTP username (access_key_id)")
    parser.add_argument('--password', '-p', default='', help="HTTP password (secret_access_key)")
    parser.add_argument(
        '--input', '-i', type=argparse.FileType('r'), default=sys.stdin, help="Input file.")
    parser.add_argument('url', help="server to post to")
    args = parser.parse_args()
    run(args.input, args.url, args.username, args.password)

if __name__ == '__main__':
    main()
<commit_msg>Only post content_md5sum when when it's there.<commit_after>"""\
Move files from "uploading" to "in progress".

Example:

    %(prog)s --username ACCESS_KEY_ID --password SECRET_ACCESS_KEY \\
        --input check_files.log https://www.encodeproject.org
"""

import json
import requests
import sys
try:
    from urllib.parse import urljoin
except ImportError:
    from urlparse import urljoin

EPILOG = __doc__

HEADERS = {
    'Accept': 'application/json',
    'Content-Type': 'application/json',
}

def run(fp, url, username, password):
    for line in fp.readlines():
        item, result, errors = json.loads(line)
        if errors:
            print('skipped %s: %s' % (item['accession'], errors))
            continue
        item_url = urljoin(url, item['@id'])
        r = requests.get(
            item_url + '?frame=object&datastore=database',
            auth=(username, password),
            headers=HEADERS,
        )
        if r.status_code != 200:
            print('ERROR retrieving %s: %s %s' % (item['@id'], r.status_code, r.reason))
            print(r.text)
            continue
        if r.json()['status'] != 'uploading':
            print('skipped %s: status %r is not "uploading"' % (item['@id'], r.json()['status']))
            continue
        data = {
            'status': 'in progress',
            'file_size': result['file_size'],
        }
        if 'content_md5sum' in result:
            data['content_md5sum'] = result['content_md5sum']
        r = requests.patch(
            item_url,
            data=json.dumps(data),
            auth=(username, password),
            headers=HEADERS,
        )
        if r.status_code != 200:
            print('ERROR patching %s: %s %s' % (item['@id'], r.status_code, r.reason))
            print(r.text)
            continue
        print('patched %s' % item['@id'])

def main():
    import argparse
    parser = argparse.ArgumentParser(
        description="Update file status", epilog=EPILOG,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    parser.add_argument('--username', '-u', default='', help="HTTP username (access_key_id)")
    parser.add_argument('--password', '-p', default='', help="HTTP password (secret_access_key)")
    parser.add_argument(
        '--input', '-i', type=argparse.FileType('r'), default=sys.stdin, help="Input file.")
    parser.add_argument('url', help="server to post to")
    args = parser.parse_args()
    run(args.input, args.url, args.username, args.password)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Views for builds app."""

import logging
import textwrap
from urllib.parse import urlparse

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseForbidden,
    HttpResponseRedirect,
)
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import DetailView, ListView
from requests.utils import quote

from readthedocs.builds.models import Build, Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.utils import trigger_build
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.projects.models import Project

log = logging.getLogger(__name__)

class BuildBase:

    model = Build

    def get_queryset(self):
        self.project_slug = self.kwargs.get('project_slug', None)
        self.project = get_object_or_404(
            Project.objects.public(self.request.user),
            slug=self.project_slug,
        )
        queryset = Build.objects.public(
            user=self.request.user,
            project=self.project,
        ).select_related('project', 'version')

        return queryset

class BuildTriggerMixin:

    @method_decorator(login_required)
    def post(self, request, project_slug):
        project = get_object_or_404(Project, slug=project_slug)

        if not AdminPermission.is_admin(request.user, project):
            return HttpResponseForbidden()

        version_slug = request.POST.get('version_slug')
        build_pk = request.POST.get('build_pk')

        if build_pk:
            # Filter over external versions only when re-triggering a specific build
            version = get_object_or_404(
                Version.external.public(self.request.user),
                slug=version_slug,
                project=project,
            )

            build = get_object_or_404(
                Build.objects.all(),
                pk=build_pk,
                version=version,
            )
            if build != Build.objects.filter(version=version).first():
                messages.add_message(
                    request,
                    messages.ERROR,
                    "This build can't be re-triggered because it's not the latest build for this version.",
                )
                return HttpResponseRedirect(request.path)
        else:
            # Use generic query when triggering a normal build
            version = get_object_or_404(
                self._get_versions(project),
                slug=version_slug,
            )

        # Set either the build to re-trigger it or None
        build_to_retrigger = version.builds.filter(pk=build_pk).first()
        commit_to_retrigger = None
        if build_to_retrigger:
            commit_to_retrigger = build_to_retrigger.commit
            log.info(
                'Re-triggering build. project=%s version=%s commit=%s build=%s',
                project.slug,
                version.slug,
                build_to_retrigger.commit,
                build_to_retrigger.pk
            )

        update_docs_task, build = trigger_build(
            project=project,
            version=version,
            commit=commit_to_retrigger,
        )
        if (update_docs_task, build) == (None, None):
            # Build was skipped
            messages.add_message(
                request,
                messages.WARNING,
                "This project is currently disabled and can't trigger new builds.",
            )
            return HttpResponseRedirect(
                reverse('builds_project_list', args=[project.slug]),
            )

        return HttpResponseRedirect(
            reverse('builds_detail', args=[project.slug, build.pk]),
        )

    def _get_versions(self, project):
        return Version.internal.public(
            user=self.request.user,
            project=project,
        )

class BuildList(BuildBase, BuildTriggerMixin, ListView):

    def get_context_data(self, **kwargs):  # pylint: disable=arguments-differ
        context = super().get_context_data(**kwargs)

        active_builds = self.get_queryset().exclude(
            state='finished',
        ).values('id')

        context['project'] = self.project
        context['active_builds'] = active_builds
        context['versions'] = self._get_versions(self.project)
        context['build_qs'] = self.get_queryset()

        return context

class BuildDetail(BuildBase, DetailView):

    pk_url_kwarg = 'build_pk'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['project'] = self.project

        build = self.get_object()
        context['is_latest_build'] = (
            build == Build.objects.filter(
                project=build.project,
                version=build.version,
            ).first()
        )

        if build.error != BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
            # Do not suggest to open an issue if the error is not generic
            return context

        scheme = (
            'https://github.com/rtfd/readthedocs.org/issues/new'
            '?title={title}{build_id}'
            '&body={body}'
        )

        # TODO: we could use ``.github/ISSUE_TEMPLATE.md`` here, but we would
        # need to add some variables to it which could impact in the UX when
        # filling an issue from the web
        body = """
        ## Details:

        * Project URL: https://readthedocs.org/projects/{project_slug}/
        * Build URL(if applicable): https://readthedocs.org{build_path}
        * Read the Docs username(if applicable): {username}

        ## Expected Result

        *A description of what you wanted to happen*

        ## Actual Result

        *A description of what actually happened*""".format(
            project_slug=self.project,
            build_path=self.request.path,
            username=self.request.user,
        )

        scheme_dict = {
            'title': quote('Build error with build id #'),
            'build_id': context['build'].id,
            'body': quote(textwrap.dedent(body)),
        }

        issue_url = scheme.format(**scheme_dict)
        issue_url = urlparse(issue_url).geturl()
        context['issue_url'] = issue_url
        return context
<commit_msg>Move "build_to_retrigger" block behind the "if build_pk" conditional<commit_after>"""Views for builds app."""

import logging
import textwrap
from urllib.parse import urlparse

from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.http import (
    HttpResponseForbidden,
    HttpResponseRedirect,
)
from django.shortcuts import get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import DetailView, ListView
from requests.utils import quote

from readthedocs.builds.models import Build, Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.utils import trigger_build
from readthedocs.doc_builder.exceptions import BuildEnvironmentError
from readthedocs.projects.models import Project

log = logging.getLogger(__name__)

class BuildBase:

    model = Build

    def get_queryset(self):
        self.project_slug = self.kwargs.get('project_slug', None)
        self.project = get_object_or_404(
            Project.objects.public(self.request.user),
            slug=self.project_slug,
        )
        queryset = Build.objects.public(
            user=self.request.user,
            project=self.project,
        ).select_related('project', 'version')

        return queryset

class BuildTriggerMixin:

    @method_decorator(login_required)
    def post(self, request, project_slug):
        commit_to_retrigger = None
        project = get_object_or_404(Project, slug=project_slug)

        if not AdminPermission.is_admin(request.user, project):
            return HttpResponseForbidden()

        version_slug = request.POST.get('version_slug')
        build_pk = request.POST.get('build_pk')

        if build_pk:
            # Filter over external versions only when re-triggering a specific build
            version = get_object_or_404(
                Version.external.public(self.request.user),
                slug=version_slug,
                project=project,
            )

            build = get_object_or_404(
                Build.objects.all(),
                pk=build_pk,
                version=version,
            )
            if build != Build.objects.filter(version=version).first():
                messages.add_message(
                    request,
                    messages.ERROR,
                    "This build can't be re-triggered because it's not the latest build for this version.",
                )
                return HttpResponseRedirect(request.path)

            # Set either the build to re-trigger it or None
            build_to_retrigger = version.builds.filter(pk=build_pk).first()
            if build_to_retrigger:
                commit_to_retrigger = build_to_retrigger.commit
                log.info(
                    'Re-triggering build. project=%s version=%s commit=%s build=%s',
                    project.slug,
                    version.slug,
                    build_to_retrigger.commit,
                    build_to_retrigger.pk
                )
        else:
            # Use generic query when triggering a normal build
            version = get_object_or_404(
                self._get_versions(project),
                slug=version_slug,
            )

        update_docs_task, build = trigger_build(
            project=project,
            version=version,
            commit=commit_to_retrigger,
        )
        if (update_docs_task, build) == (None, None):
            # Build was skipped
            messages.add_message(
                request,
                messages.WARNING,
                "This project is currently disabled and can't trigger new builds.",
            )
            return HttpResponseRedirect(
                reverse('builds_project_list', args=[project.slug]),
            )

        return HttpResponseRedirect(
            reverse('builds_detail', args=[project.slug, build.pk]),
        )

    def _get_versions(self, project):
        return Version.internal.public(
            user=self.request.user,
            project=project,
        )

class BuildList(BuildBase, BuildTriggerMixin, ListView):

    def get_context_data(self, **kwargs):  # pylint: disable=arguments-differ
        context = super().get_context_data(**kwargs)

        active_builds = self.get_queryset().exclude(
            state='finished',
        ).values('id')

        context['project'] = self.project
        context['active_builds'] = active_builds
        context['versions'] = self._get_versions(self.project)
        context['build_qs'] = self.get_queryset()

        return context

class BuildDetail(BuildBase, DetailView):

    pk_url_kwarg = 'build_pk'

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        context['project'] = self.project

        build = self.get_object()
        context['is_latest_build'] = (
            build == Build.objects.filter(
                project=build.project,
                version=build.version,
            ).first()
        )

        if build.error != BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format(build_id=build.pk):
            # Do not suggest to open an issue if the error is not generic
            return context

        scheme = (
            'https://github.com/rtfd/readthedocs.org/issues/new'
            '?title={title}{build_id}'
            '&body={body}'
        )

        # TODO: we could use ``.github/ISSUE_TEMPLATE.md`` here, but we would
        # need to add some variables to it which could impact in the UX when
        # filling an issue from the web
        body = """
        ## Details:

        * Project URL: https://readthedocs.org/projects/{project_slug}/
        * Build URL(if applicable): https://readthedocs.org{build_path}
        * Read the Docs username(if applicable): {username}

        ## Expected Result

        *A description of what you wanted to happen*

        ## Actual Result

        *A description of what actually happened*""".format(
            project_slug=self.project,
            build_path=self.request.path,
            username=self.request.user,
        )

        scheme_dict = {
            'title': quote('Build error with build id #'),
            'build_id': context['build'].id,
            'body': quote(textwrap.dedent(body)),
        }

        issue_url = scheme.format(**scheme_dict)
        issue_url = urlparse(issue_url).geturl()
        context['issue_url'] = issue_url
        return context
<|endoftext|>


<commit_before>from __future__ import division
from textwrap import dedent
import colorsys
import numpy as np
from scipy import stats
import pandas as pd
from pandas.core.series import remove_na
import matplotlib as mpl
from matplotlib.collections import PatchCollection
import matplotlib.patches as Patches
import matplotlib.pyplot as plt
import warnings

import matplotlib.pyplot as plt
from matplotlib import offsetbox
import numpy as np
import pandas as pd
from scipy.spatial import distance
import seaborn as sns

class _ReducedPlotter(object):
    """Generic object for plotting reduced representations of high-dimensional data on 2d space"""

    def establish_reducer(self, reducer, n_components, smusher_kws):
        smusher_kws.setdefault('n_components', n_components)

        if reducer is None:
            # Create a brand new dimensionality reducer (matrix decomposer/manifold learner)
            self.reducer = reducer(**smusher_kws)
        else:
            # We're using a pre-existing, user-supplied reducer
            self.reducer = reducer

    def establish_variables(self, data):
        """Convert input specification into a common representation."""

        # Option 1a:
        # The input data is a Pandas DataFrame
        # ------------------------------------
        if isinstance(data, pd.DataFrame):
            high_dimensional_data = data
            group_label = data.index.name
            value_label = data.columns.name

        # Option 1b:
        # The input data is an array or list
        # ----------------------------------
        else:
            # The input data is an array
            if hasattr(data, "shape"):
                if len(data.shape) == 2:
                    nr, nc = data.shape
                    if nr == 1 or nc == 1:
                        error = ("Input `data` must have "
                                 "exactly 2 dimensions, where both "
                                 "have at least 2 components")
                        raise ValueError(error)
                    else:
                        high_dimensional_data = pd.DataFrame(data)
                else:
                    error = ("Input `data` must have "
                             "exactly 2 dimensions")
                    raise ValueError(error)

            # Check if `data` is None to let us bail out here (for testing)
            elif data is None:
                high_dimensional_data = pd.DataFrame([[]])
            group_label = None
            value_label = None

        # Assign object attributes
        # ------------------------
        self.high_dimensional_data = high_dimensional_data
        self.group_label = group_label
        self.value_label = value_label

    def establish_colors(self, color, palette, saturation, marker, marker_order, text, text_order, plot_kws):
        """Get a list of colors for the main component of the plots."""
        if self.hue_names is None:
            n_colors = len(self.reduced_data.index)
        else:
            n_colors = len(self.hue_names)

        # Determine the main colors
        if color is None and palette is None:
            # Determine whether the current palette will have enough values
            # If not, we'll default to the husl palette so each is distinct
            current_palette = sns.utils.get_color_cycle()
            if n_colors <= len(current_palette):
                colors = sns.color_palette(n_colors=n_colors)
            else:
                colors = sns.husl_palette(n_colors, l=.7)

        elif palette is None:
            # When passing a specific color, the interpretation depends
            # on whether there is a hue variable or not.
            # If so, we will make a blend palette so that the different
            # levels have some amount of variation.
            if self.hue_names is None:
                colors = [color] * n_colors
            else:
                colors = sns.light_palette(color, n_colors)
        else:

            # Let `palette` be a dict mapping level to color
            if isinstance(palette, dict):
                if self.hue_names is None:
                    levels = self.group_names
                else:
                    levels = self.hue_names
                palette = [palette[l] for l in levels]

            colors = sns.color_palette(palette, n_colors)

        # Desaturate a bit because these are patches
        if saturation < 1:
            colors = sns.color_palette(colors, desat=saturation)

        # Conver the colors to a common representations
        rgb_colors = sns.color_palette(colors)

        # Determine the gray color to use for the lines framing the plot
        light_vals = [colorsys.rgb_to_hls(*c)[1] for c in rgb_colors]
        l = min(light_vals) * .6
        gray = mpl.colors.rgb2hex((l, l, l))

        # Assign object attributes
        self.colors = rgb_colors
        self.gray = gray
    #
    # def establish_markers(self, marker=None, marker_order=None,
    #                       text=None, text_order=None, plot_kws=None):
    #     if text is None:
    #         # Use matplotlib's plotting function to plot each sample
    #         markerplotter = plt.plot
    #     else:
    #         if marker is not None:
    #             raise ValueError('Only one of "marker" or "text" can be '
    #                              'specified')
    #
    #         # Plot each sample "point" as text instead of a plotting symbol
    #         markerplotter = plt.text
    #     self.markerplotter = markerplotter

    def markerplotter(self, xs, ys, marker, text, **kwargs):
        if text:
            for x, y in zip(xs, ys):
                plt.text(x, y, marker, **kwargs)
        else:
            plt.plot(xs, ys, marker, **kwargs)

    def annotate_axes(self, ax):
        """Add descriptive labels to an Axes object."""
        if self.orient == "v":
            xlabel, ylabel = self.group_label, self.value_label
        else:
            xlabel, ylabel = self.value_label, self.group_label

        if xlabel is not None:
            ax.set_xlabel(xlabel)
        if ylabel is not None:
            ax.set_ylabel(ylabel)

        if self.orient == "v":
            ax.set_xticks(np.arange(len(self.reduced_data)))
            ax.set_xticklabels(self.group_names)
        else:
            ax.set_yticks(np.arange(len(self.reduced_data)))
            ax.set_yticklabels(self.group_names)

        if self.orient == "v":
            ax.xaxis.grid(False)
            ax.set_xlim(-.5, len(self.reduced_data) - .5)
        else:
            ax.yaxis.grid(False)
            ax.set_ylim(-.5, len(self.reduced_data) - .5)

        if self.hue_names is not None:
            leg = ax.legend(loc="best")
            if self.hue_title is not None:
                leg.set_title(self.hue_title)

                # Set the title size a roundabout way to maintain
                # compatability with matplotlib 1.1
                try:
                    title_size = mpl.rcParams["axes.labelsize"] * .85
                except TypeError:  # labelsize is something like "large"
                    title_size = mpl.rcParams["axes.labelsize"]
                prop = mpl.font_manager.FontProperties(size=title_size)
                leg._legend_title_box._text.set_font_properties(prop)

    def add_legend_data(self, ax, color, label):
        """Add a dummy patch object so we can get legend data."""
        rect = plt.Rectangle([0, 0], 0, 0,
                             linewidth=self.linewidth / 2,
                             edgecolor=self.gray,
                             facecolor=color,
                             label=label)
        ax.add_patch(rect)

    def draw_markers(self, ax, text, x=0, y=1):
        if isinstance(text, bool):
            if text:
                # Label each point with its sample id (row id)
                pass
        for hue, hue_df in self.reduced_data.groupby(self.hue):
            for marker, marker_df in hue_df.groupby(self.marker):
                self.markerplotter(marker_df[:, x], marker_df[:, y], text=text,
                                   marker=marker, color=hue, ax=ax)

    def draw_images(self, ax, images):
        if hasattr(offsetbox, 'AnnotationBbox'):
            # only print thumbnails with matplotlib > 1.0
            shown_images = np.array([[1., 1.]])  # just something big
            for i in range(self.reduced_data.shape[0]):
                dist = np.sum((self.reduced_data[i] - shown_images) ** 2, 1)
                if np.min(dist) < 5e-4 * np.product(images[0].shape):
                    # don't show points that are too close
                    continue
                shown_images = np.r_[shown_images, [self.reduced_data[i]]]
                imagebox = offsetbox.AnnotationBbox(
                    offsetbox.OffsetImage(images[i], cmap=plt.cm.gray),
                    self.reduced_data[i])
                ax.add_artist(imagebox)

    def compute_reduction(self):
        """Use reducer to smush high-dimensional data into a smaller space"""
        matrix = self.reducer.fit_transform(self.high_dimensional_data)
        columns = np.arange(matrix.shape[1], dtype=int) + 1
        self.reduced_data = pd.DataFrame(
            matrix, columns=columns, index=self.high_dimensional_data.index)
<commit_msg>Wrote initial code for establishing plotting symbols<commit_after>"""
Internal abstract base class for all dimensionality reduction plots.
Not intended to be user-facing.
"""
from __future__ import division
from textwrap import dedent
import colorsys
import numpy as np
from scipy import stats
import pandas as pd
from pandas.core.series import remove_na
import matplotlib as mpl
from matplotlib.collections import PatchCollection
import matplotlib.patches as Patches
import matplotlib.pyplot as plt
import warnings

import matplotlib.pyplot as plt
from matplotlib import offsetbox
import numpy as np
import pandas as pd
from scipy.spatial import distance
import seaborn as sns

class _ReducedPlotter(object):
    """Generic object for plotting reduced representations of high-dimensional data on 2d space"""

    def establish_reducer(self, reducer, n_components, smusher_kws):
        smusher_kws.setdefault('n_components', n_components)

        if reducer is None:
            # Create a brand new dimensionality reducer (matrix decomposer/manifold learner)
            self.reducer = reducer(**smusher_kws)
        else:
            # We're using a pre-existing, user-supplied reducer
            self.reducer = reducer

    def establish_variables(self, data):
        """Convert input specification into a common representation."""

        # Option 1a:
        # The input data is a Pandas DataFrame
        # ------------------------------------
        if isinstance(data, pd.DataFrame):
            high_dimensional_data = data
            group_label = data.index.name
            value_label = data.columns.name

        # Option 1b:
        # The input data is an array or list
        # ----------------------------------
        else:
                if len(data.shape) == 2:
                    nr, nc = data.shape
                    if nr == 1 or nc == 1:
                        error = ("Input `data` must have "
                                 "exactly 2 dimensions, where both "
                                 "have at least 2 components")
                        raise ValueError(error)
                    else:
                        high_dimensional_data = pd.DataFrame(data)
                else:
                    error = ("Input `data` must have "
                             "exactly 2 dimensions")
                    raise ValueError(error)

            # Check if `data` is None to let us bail out here (for testing)
            elif data is None:
                high_dimensional_data = pd.DataFrame([[]])
            group_label = None
            value_label = None

        # Assign object attributes
        # ------------------------
        self.high_dimensional_data = high_dimensional_data
        self.group_label = group_label
        self.value_label = value_label

    def establish_colors(self, color, palette, saturation):
        """Get a list of colors for the main component of the plots."""
        if self.hue_names is None:
            n_colors = len(self.reduced_data.index)
        else:
            n_colors = len(self.hue_names)

        # Determine the main colors
        if color is None and palette is None:
            # Determine whether the current palette will have enough values
            # If not, we'll default to the husl palette so each is distinct
            current_palette = sns.utils.get_color_cycle()
            if n_colors <= len(current_palette):
                colors = sns.color_palette(n_colors=n_colors)
            else:
                colors = sns.husl_palette(n_colors, l=.7)

        elif palette is None:
            # When passing a specific color, the interpretation depends
            # on whether there is a hue variable or not.
            # If so, we will make a blend palette so that the different
            # levels have some amount of variation.
            if self.hue_names is None:
                colors = [color] * n_colors
            else:
                colors = sns.light_palette(color, n_colors)
        else:

            # Let `palette` be a dict mapping level to color
            if isinstance(palette, dict):
                if self.hue_names is None:
                    levels = self.group_names
                else:
                    levels = self.hue_names
                palette = [palette[l] for l in levels]

            colors = sns.color_palette(palette, n_colors)

        # Desaturate a bit because these are patches
        if saturation < 1:
            colors = sns.color_palette(colors, desat=saturation)

        # Conver the colors to a common representations
        rgb_colors = sns.color_palette(colors)

        # Determine the gray color to use for the lines framing the plot
        light_vals = [colorsys.rgb_to_hls(*c)[1] for c in rgb_colors]
        l = min(light_vals) * .6
        gray = mpl.colors.rgb2hex((l, l, l))

        # Assign object attributes
        self.colors = rgb_colors
        self.gray = gray

    def establish_symbols(self, marker, marker_order, text, text_order):
        """Figure out what to place on the axes for each data point"""
        if isinstance(text, bool):
            self.text = text

            if self.text:
                # Use the sample names of data as the plotting symbol
                symbol = sns.utils.categorical_order(
                    self.high_dimensional_data.index)
            else:
                symbol = sns.utils.categorical_order(marker, marker_order)
        else:
            # Assume "text" is a mapping from row names (sample ids) of the
            # data to text labels
            if marker is not None:
                symbol = sns.utils.categorical_order(text, text_order)
            else:
                error = 'Cannot specify both "marker" and "text'
                raise ValueError(error)

            # Turn text into a boolean
            text = True

        self.symbol = symbol
        self.text = text

    def markerplotter(self, xs, ys, symbol, text, **kwargs):
        # If both the x- and y- positions don't have data, don't do anything
        if xs.empty and ys.empty:
            return

        if text:
            # Plot each (x, y) position as text
            for x, y in zip(xs, ys):
                plt.text(x, y, symbol, **kwargs)
        else:
            # use plt.plot instead of plt.scatter for speed, since plotting all
            # the same marker shape and color and linestyle
            plt.plot(xs, ys, 'o', marker=symbol, **kwargs)

    def annotate_axes(self, ax):
        """Add descriptive labels to an Axes object."""
        if self.orient == "v":
            xlabel, ylabel = self.group_label, self.value_label
        else:
            xlabel, ylabel = self.value_label, self.group_label

        if xlabel is not None:
            ax.set_xlabel(xlabel)
        if ylabel is not None:
            ax.set_ylabel(ylabel)

        if self.orient == "v":
            ax.set_xticks(np.arange(len(self.reduced_data)))
            ax.set_xticklabels(self.group_names)
        else:
            ax.set_yticks(np.arange(len(self.reduced_data)))
            ax.set_yticklabels(self.group_names)

        if self.orient == "v":
            ax.xaxis.grid(False)
            ax.set_xlim(-.5, len(self.reduced_data) - .5)
        else:
            ax.yaxis.grid(False)
            ax.set_ylim(-.5, len(self.reduced_data) - .5)

        if self.hue_names is not None:
            leg = ax.legend(loc="best")
            if self.hue_title is not None:
                leg.set_title(self.hue_title)

                # Set the title size a roundabout way to maintain
                # compatability with matplotlib 1.1
                try:
                    title_size = mpl.rcParams["axes.labelsize"] * .85
                except TypeError:  # labelsize is something like "large"
                    title_size = mpl.rcParams["axes.labelsize"]
                prop = mpl.font_manager.FontProperties(size=title_size)
                leg._legend_title_box._text.set_font_properties(prop)

    def add_legend_data(self, ax, color, label):
        """Add a dummy patch object so we can get legend data."""
        rect = plt.Rectangle([0, 0], 0, 0,
                             linewidth=self.linewidth / 2,
                             edgecolor=self.gray,
                             facecolor=color,
                             label=label)
        ax.add_patch(rect)

    def draw_markers(self, ax, text, x=0, y=1):
        if isinstance(text, bool):
            if text:
                # Label each point with its sample id (row id)
                pass
        for hue, hue_df in self.reduced_data.groupby(self.hue):
            for symbol, symbol_df in hue_df.groupby(self.symbol):
                self.markerplotter(symbol_df[:, x], symbol_df[:, y], text=text,
                                   symbol=symbol, color=hue, ax=ax)

    def draw_images(self, ax, images):
        if hasattr(offsetbox, 'AnnotationBbox'):
            # only print thumbnails with matplotlib > 1.0
            shown_images = np.array([[1., 1.]])  # just something big
            for i in range(self.reduced_data.shape[0]):
                dist = np.sum((self.reduced_data[i] - shown_images) ** 2, 1)
                if np.min(dist) < 5e-4 * np.product(images[0].shape):
                    # don't show points that are too close
                    continue
                shown_images = np.r_[shown_images, [self.reduced_data[i]]]
                imagebox = offsetbox.AnnotationBbox(
                    offsetbox.OffsetImage(images[i], cmap=plt.cm.gray),
                    self.reduced_data[i])
                ax.add_artist(imagebox)

    def compute_reduction(self):
        """Use reducer to smush high-dimensional data into a smaller space"""
        matrix = self.reducer.fit_transform(self.high_dimensional_data)
        columns = np.arange(matrix.shape[1], dtype=int) + 1
        self.reduced_data = pd.DataFrame(
            matrix, columns=columns, index=self.high_dimensional_data.index)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import json
from functools import wraps

from bottle import request, HTTPError

# Functions & classes =========================================================
def decode_json_body():
    """
    Decode ``bottle.request.body`` to JSON.

    Returns:
        obj: Structure decoded by ``json.loads()``.

    Raises:
        HTTPError: 400 in case the data was malformed.
    """
    raw_data = request.body.read()

    try:
        return json.loads(raw_data)
    except ValueError as e:
        raise HTTPError(400, e.message)

def handle_type_error(fn):
    """
    Convert ``TypeError`` to ``bottle.HTTPError`` with ``400`` code and message
    about wrong parameters.

    Raises:
        HTTPError: 400 in case too many/too little function parameters were \
                   given.
    """
    @wraps(fn)
    def handle_type_error_wrapper(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except TypeError as e:
            msg = e.message
            if fn.__name__ in msg and \
               ("got an unexpected" in msg or "takes exactly" in msg):
                raise HTTPError(400, msg)

            raise  # This will cause 500: Internal server error

    return handle_type_error_wrapper

def json_to_params(fn=None, return_json=True):
    """
    Convert JSON in the body of the request to the parameters for the wrapped
    function.

    If the JSON is list, add it to ``*args``.

    If dict, add it to ``**kwargs`` in non-rewrite mode (no key in ``**kwargs``
    will be overwritten).

    If single value, add it to ``*args``.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def json_to_params_decorator(fn):
        @wraps(fn)
        @handle_type_error
        def json_to_params_wrapper(*args, **kwargs):
            data = decode_json_body()

            if type(data) in [tuple, list]:
                args = list(args) + data
            elif type(data) == dict:
                # transport only items that are not already in kwargs
                allowed_keys = set(data.keys()) - set(kwargs.keys())
                for key in allowed_keys:
                    kwargs[key] = data[key]
            elif type(data) in [bool, int, float, long, str, unicode]:
                args = list(args)
                args.append(data)

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return json_to_params_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return json_to_params_decorator(fn)

    return json_to_params_decorator

def json_to_data(fn=None, return_json=True):
    """
    Decode JSON from the request and add it as ``data`` parameter for wrapped
    function.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def json_to_data_decorator(fn):
        @wraps(fn)
        @handle_type_error
        def get_data_wrapper(*args, **kwargs):
            kwargs["data"] = decode_json_body()

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return get_data_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return json_to_data_decorator(fn)

    return json_to_data_decorator

def form_to_params(fn=None, return_json=True):
    """
    Convert bottle forms request to parameters for the wrapped function.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def forms_to_params_decorator(fn):
        @wraps(fn)
        @handle_type_error
        def forms_to_params_wrapper(*args, **kwargs):
            kwargs.update(
                dict(request.forms)
            )

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return forms_to_params_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return forms_to_params_decorator(fn)

    return forms_to_params_decorator
<commit_msg>Fixed nasty bug found only thanks to unittests.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Interpreter version: python 2.7
#
# Imports =====================================================================
import json
from functools import wraps

from bottle import request, HTTPError

# Functions & classes =========================================================
def decode_json_body():
    """
    Decode ``bottle.request.body`` to JSON.

    Returns:
        obj: Structure decoded by ``json.loads()``.

    Raises:
        HTTPError: 400 in case the data was malformed.
    """
    raw_data = request.body.read()

    try:
        return json.loads(raw_data)
    except ValueError as e:
        raise HTTPError(400, e.message)

def handle_type_error(fn):
    """
    Convert ``TypeError`` to ``bottle.HTTPError`` with ``400`` code and message
    about wrong parameters.

    Raises:
        HTTPError: 400 in case too many/too little function parameters were \
                   given.
    """
    def any_match(string_list, obj):
        return filter(lambda x: x in obj, string_list)

    @wraps(fn)
    def handle_type_error_wrapper(*args, **kwargs):
        try:
            return fn(*args, **kwargs)
        except TypeError as e:
            str_list = [
                "takes exactly",
                "got an unexpected",
                "takes no argument",
            ]
            if fn.__name__ in e.message and any_match(str_list, e.message):
                raise HTTPError(400, e.message)

            raise  # This will cause 500: Internal server error

    return handle_type_error_wrapper

def json_to_params(fn=None, return_json=True):
    """
    Convert JSON in the body of the request to the parameters for the wrapped
    function.

    If the JSON is list, add it to ``*args``.

    If dict, add it to ``**kwargs`` in non-rewrite mode (no key in ``**kwargs``
    will be overwritten).

    If single value, add it to ``*args``.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def json_to_params_decorator(fn):
        @handle_type_error
        @wraps(fn)
        def json_to_params_wrapper(*args, **kwargs):
            data = decode_json_body()

            if type(data) in [tuple, list]:
                args = list(args) + data
            elif type(data) == dict:
                # transport only items that are not already in kwargs
                allowed_keys = set(data.keys()) - set(kwargs.keys())
                for key in allowed_keys:
                    kwargs[key] = data[key]
            elif type(data) in [bool, int, float, long, str, unicode]:
                args = list(args)
                args.append(data)

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return json_to_params_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return json_to_params_decorator(fn)

    return json_to_params_decorator

def json_to_data(fn=None, return_json=True):
    """
    Decode JSON from the request and add it as ``data`` parameter for wrapped
    function.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def json_to_data_decorator(fn):
        @handle_type_error
        @wraps(fn)
        def get_data_wrapper(*args, **kwargs):
            kwargs["data"] = decode_json_body()

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return get_data_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return json_to_data_decorator(fn)

    return json_to_data_decorator

def form_to_params(fn=None, return_json=True):
    """
    Convert bottle forms request to parameters for the wrapped function.

    Args:
        return_json (bool, default True): Should the decorator automatically
                    convert returned value to JSON?
    """
    def forms_to_params_decorator(fn):
        @handle_type_error
        @wraps(fn)
        def forms_to_params_wrapper(*args, **kwargs):
            kwargs.update(
                dict(request.forms)
            )

            if not return_json:
                return fn(*args, **kwargs)

            return json.dumps(
                fn(*args, **kwargs)
            )

        return forms_to_params_wrapper

    if fn:  # python decorator with optional parameters bukkake
        return forms_to_params_decorator(fn)

    return forms_to_params_decorator
<|endoftext|>


<commit_before>#! /usr/bin/env python2
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2014 Ciel <imwithye@gmail.com>
#
# Distributed under terms of the MIT license.

import os, sys

userpath = os.path.expanduser('~') + '/.git-ignore-templates/'
syspath = sys.path[0] + "/system-templates/"
githubpath = sys.path[0] + "/github-templates/"

# get file list in three search paths
def filelist():
	try:
		usertemplates = os.listdir(userpath)
	except OSError:
		os.mkdir(userpath)
		usertemplates = os.listdir(userpath)
	try:
		systemplates = os.listdir(syspath)
	except OSError:
		os.mkdir(syspath)
		systemplates = os.listdir(syspath)
	try:
		githubtemplates = os.listdir(githubpath)
	except OSError:
		githubtemplates = []

	return (usertemplates, systemplates, githubtemplates)

# search file
def searchfile(language, templates):
	language = language.lower() + ".gitignore"
	for template in templates:
		if language == template.lower():
			return template
	return ""

# create ignore file path list
def ignorelist(languages):
	user, system, github = filelist()
	ignorelist = []
	for language in languages:
		ignorefile = ""

		ignorefile = searchfile(language, user)
		if ignorefile != "":
			ignorelist.append(userpath + ignorefile)
			continue

		ignorefile = searchfile(language, system)
		if ignorefile != "":
			ignorelist.append(syspath + ignorefile)
			continue

		ignorefile = searchfile(language, github)
		if ignorefile != "":
			ignorelist.append(githubpath + ignorefile)
			continue
	return ignorelist

def readfile(filepath):
	content = ""
	try:
		ignorefile = open(filepath, 'r')
		content = ignorefile.read()
	except:
		content = ""
	finally:
		ignorefile.close()
		return content
<commit_msg>rm usertemplates.<commit_after>#! /usr/bin/env python2
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2014 Ciel <imwithye@gmail.com>
#
# Distributed under terms of the MIT license.

import os, sys

userpath = os.path.expanduser('~') + '/.git-ignores/'
syspath = sys.path[0] + "/system-templates/"
githubpath = sys.path[0] + "/github-templates/"

# get file list in three search paths
def filelist():
	try:
		usertemplates = os.listdir(userpath)
	except OSError:
		usertemplates = []
	try:
		systemplates = os.listdir(syspath)
	except OSError:
		os.mkdir(syspath)
		systemplates = os.listdir(syspath)
	try:
		githubtemplates = os.listdir(githubpath)
	except OSError:
		githubtemplates = []

	return (usertemplates, systemplates, githubtemplates)

# search file
def searchfile(language, templates):
	language = language.lower() + ".gitignore"
	for template in templates:
		if language == template.lower():
			return template
	return ""

# create ignore file path list
def ignorelist(languages):
	user, system, github = filelist()
	ignorelist = []
	for language in languages:
		ignorefile = ""

		ignorefile = searchfile(language, user)
		if ignorefile != "":
			ignorelist.append(userpath + ignorefile)
			continue

		ignorefile = searchfile(language, system)
		if ignorefile != "":
			ignorelist.append(syspath + ignorefile)
			continue

		ignorefile = searchfile(language, github)
		if ignorefile != "":
			ignorelist.append(githubpath + ignorefile)
			continue
	return ignorelist

def readfile(filepath):
	content = ""
	try:
		ignorefile = open(filepath, 'r')
		content = ignorefile.read()
	except:
		content = ""
	finally:
		ignorefile.close()
		return content
<|endoftext|>