<commit_before>"""
Copyright (c) 2015, 2019 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

import os

DOCKER_SOCKET_PATH = '/var/run/docker.sock'
DOCKERFILE_FILENAME = 'Dockerfile'
CACHITO_ENV_FILENAME = 'cachito.env'
CACHITO_ENV_ARG_ALIAS = 'CACHITO_ENV_FILE'
BUILD_JSON = 'build.json'
BUILD_JSON_ENV = 'BUILD_JSON'
RESULTS_JSON = 'results.json'
METADATA_TAG = "_metadata_"

WORKSPACE_BASE_PATH = '/workspace'
REACTOR_CONFIG_WORKSPACE = 'ws-reactor-config-map'
REACTOR_CONFIG_BASE_NAME = 'config.yaml'
REACTOR_CONFIG_FULL_PATH = os.path.join(
    WORKSPACE_BASE_PATH, REACTOR_CONFIG_WORKSPACE, REACTOR_CONFIG_BASE_NAME
)
REACTOR_CONFIG_ENV_NAME = 'REACTOR_CONFIG'

CONTAINER_SHARE_PATH = '/run/share/'
CONTAINER_SHARE_SOURCE_SUBDIR = 'source'
CONTAINER_SECRET_PATH = ''
CONTAINER_BUILD_JSON_PATH = os.path.join(CONTAINER_SHARE_PATH, BUILD_JSON)
CONTAINER_RESULTS_JSON_PATH = os.path.join(CONTAINER_SHARE_PATH, RESULTS_JSON)
CONTAINER_DOCKERFILE_PATH = os.path.join(CONTAINER_SHARE_PATH, DOCKERFILE_FILENAME)

HOST_SECRET_PATH = ''

EXPORTED_SQUASHED_IMAGE_NAME = 'image.tar'
EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE = 'compressed.tar.{0}'

YUM_REPOS_DIR = '/etc/yum.repos.d/'
RELATIVE_REPOS_PATH = "atomic-reactor-repos/"
DEFAULT_YUM_REPOFILE_NAME = 'atomic-reactor-injected.repo'

# key in dictionary returned by "docker inspect" that holds the image
# configuration (such as labels)
INSPECT_CONFIG = "Config"
# key that holds the RootFS
INSPECT_ROOTFS = "RootFS"
# key that holds the layer diff_ids
INSPECT_ROOTFS_LAYERS = 'Layers'

# docs constants

DESCRIPTION = "Python library with command line interface for building container images."
HOMEPAGE = "https://github.com/containerbuildsystem/atomic-reactor"
PROG = "atomic-reactor"
MANPAGE_AUTHORS = "Jiri Popelka <jpopelka@redhat.com>, " \
                  "Martin Milata <mmilata@redhat.com>, " \
                  "Slavek Kabrda <slavek@redhat.com>, " \
                  "Tim Waugh <twaugh@redhat.com>, " \
                  "Tomas Tomecek <ttomecek@redhat.com>"
MANPAGE_SECTION = 1

# debug print of tools reactor uses

TOOLS_USED = (
    {"pkg_name": "docker", "display_name": "docker-py"},
    {"pkg_name": "atomic_reactor"},
    {"pkg_name": "osbs", "display_name": "osbs-client"},
)

DEFAULT_DOWNLOAD_BLOCK_SIZE = 10 * 1024 * 1024  # 10Mb

IMAGE_TYPE_DOCKER_ARCHIVE = 'docker-archive'
IMAGE_TYPE_OCI = 'oci'
IMAGE_TYPE_OCI_TAR = 'oci-tar'

PLUGIN_KOJI_PROMOTE_PLUGIN_KEY = 'koji_promote'
PLUGIN_KOJI_IMPORT_PLUGIN_KEY = 'koji_import'
PLUGIN_KOJI_IMPORT_SOURCE_CONTAINER_PLUGIN_KEY = 'koji_import_source_container'
PLUGIN_KOJI_UPLOAD_PLUGIN_KEY = 'koji_upload'
PLUGIN_KOJI_TAG_BUILD_KEY = 'koji_tag_build'
PLUGIN_ADD_FILESYSTEM_KEY = 'add_filesystem'
PLUGIN_BUMP_RELEASE_KEY = 'bump_release'
PLUGIN_DISTGIT_FETCH_KEY = 'distgit_fetch_artefacts'
PLUGIN_FETCH_MAVEN_KEY = 'fetch_maven_artifacts'
PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY = 'maven_url_sources_metadata'
PLUGIN_GATHER_BUILDS_METADATA_KEY = 'gather_builds_metadata'
PLUGIN_GROUP_MANIFESTS_KEY = 'group_manifests'
PLUGIN_INJECT_PARENT_IMAGE_KEY = 'inject_parent_image'
PLUGIN_KOJI_PARENT_KEY = 'koji_parent'
PLUGIN_COMPARE_COMPONENTS_KEY = 'compare_components'
PLUGIN_CHECK_AND_SET_PLATFORMS_KEY = 'check_and_set_platforms'
PLUGIN_CHECK_USER_SETTINGS = 'check_user_settings'
PLUGIN_REMOVE_WORKER_METADATA_KEY = 'remove_worker_metadata'
PLUGIN_RESOLVE_COMPOSES_KEY = 'resolve_composes'
PLUGIN_RESOLVE_REMOTE_SOURCE = 'resolve_remote_source'
PLUGIN_SENDMAIL_KEY = 'sendmail'
PLUGIN_VERIFY_MEDIA_KEY = 'verify_media'
PLUGIN_PIN_OPERATOR_DIGESTS_KEY = 'pin_operator_digest'
PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY = 'export_operator_manifests'
PLUGIN_PUSH_OPERATOR_MANIFESTS_KEY = 'push_operator_manifests'
PLUGIN_SOURCE_CONTAINER_KEY = 'source_container'
PLUGIN_FETCH_SOURCES_KEY = 'fetch_sources'
PLUGIN_KOJI_DELEGATE_KEY = 'koji_delegate'
PLUGIN_PUSH_FLOATING_TAGS_KEY = 'push_floating_tags'
PLUGIN_ADD_IMAGE_CONTENT_MANIFEST = 'add_image_content_manifest'
PLUGIN_CANCEL_BUILD_RESERVATION = 'cancel_build_reservation'
PLUGIN_FLATPAK_CREATE_OCI = 'flatpak_create_oci'

# some shared dict keys for build metadata that gets recorded with koji.
# for consistency of metadata in historical builds, these values basically cannot change.
# however constant names could change to more accurately reflect current semantics.
BASE_IMAGE_KOJI_BUILD = 'parent-image-koji-build'  # from when the base image was the only parent
PARENT_IMAGES_KOJI_BUILDS = 'parent-images-koji-builds'
BASE_IMAGE_BUILD_ID_KEY = 'parent_build_id'  # from when the base image was the only parent
PARENT_IMAGE_BUILDS_KEY = 'parent_image_builds'
PARENT_IMAGES_KEY = 'parent_images'
SCRATCH_FROM = 'scratch'

# max retries for docker requests
DOCKER_MAX_RETRIES = 3
# how many seconds should wait before another try of docker request
DOCKER_BACKOFF_FACTOR = 5
# docker retries statuses
DOCKER_CLIENT_STATUS_RETRY = (408, 500, 502, 503, 504)
# max retries for docker push
DOCKER_PUSH_MAX_RETRIES = 6
# how many seconds should wait before another try of docker push
DOCKER_PUSH_BACKOFF_FACTOR = 5
# max retries for http requests
HTTP_MAX_RETRIES = 10
# how many seconds should wait before another try of http request
HTTP_BACKOFF_FACTOR = 5
# http retries statuses
HTTP_CLIENT_STATUS_RETRY = (408, 429, 500, 502, 503, 504)
# requests timeout in seconds
HTTP_REQUEST_TIMEOUT = 600
# max retries for git clone
GIT_MAX_RETRIES = 3
# how many seconds should wait before another try of git clone
GIT_BACKOFF_FACTOR = 5
# max retries for reserving koji builds
KOJI_RESERVE_MAX_RETRIES = 20
# wait for 2sec (usual time of bump_release with reserve)
KOJI_RESERVE_RETRY_DELAY = 2
KOJI_MAX_RETRIES = 120
KOJI_RETRY_INTERVAL = 60
KOJI_OFFLINE_RETRY_INTERVAL = 120
# max retries for subprocesses (see utils.retries.run_cmd())
SUBPROCESS_MAX_RETRIES = 5
# the factor for the exponential backoff series - 5, 10, 20, 40, 80 seconds of waiting
SUBPROCESS_BACKOFF_FACTOR = 5

# Media types
MEDIA_TYPE_DOCKER_V2_SCHEMA1 = "application/vnd.docker.distribution.manifest.v1+json"
MEDIA_TYPE_DOCKER_V2_SCHEMA2 = "application/vnd.docker.distribution.manifest.v2+json"
MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
MEDIA_TYPE_OCI_V1 = "application/vnd.oci.image.manifest.v1+json"
MEDIA_TYPE_OCI_V1_INDEX = "application/vnd.oci.image.index.v1+json"

# PNC related constants
PNC_SYSTEM_USER = "newcastle"

REPO_CONTAINER_CONFIG = 'container.yaml'
REPO_CONTENT_SETS_CONFIG = 'content_sets.yml'
REPO_FETCH_ARTIFACTS_URL = 'fetch-artifacts-url.yaml'
REPO_FETCH_ARTIFACTS_PNC = 'fetch-artifacts-pnc.yaml'
REPO_FETCH_ARTIFACTS_KOJI = 'fetch-artifacts-koji.yaml'

DOCKERIGNORE = '.dockerignore'

# Operator manifest constants
OPERATOR_MANIFESTS_ARCHIVE = 'operator_manifests.zip'

KOJI_BTYPE_IMAGE = 'image'
KOJI_BTYPE_OPERATOR_MANIFESTS = 'operator-manifests'
KOJI_BTYPE_REMOTE_SOURCES = 'remote-sources'
KOJI_BTYPE_REMOTE_SOURCE_FILE = 'remote-source-file'

# Path to where the remote source bundle is copied to during the build process
REMOTE_SOURCE_DIR = '/remote-source'

# Name of downloaded remote sources tarball
REMOTE_SOURCE_TARBALL_FILENAME = 'remote-source.tar.gz'
REMOTE_SOURCE_JSON_FILENAME = 'remote-source.json'

# koji osbs_build metadata
KOJI_KIND_IMAGE_BUILD = 'container_build'
KOJI_KIND_IMAGE_SOURCE_BUILD = 'source_container_build'
KOJI_SUBTYPE_OP_APPREGISTRY = 'operator_appregistry'
KOJI_SUBTYPE_OP_BUNDLE = 'operator_bundle'
KOJI_SOURCE_ENGINE = 'bsi'

# Storage names as defined in skopeo
DOCKER_STORAGE_TRANSPORT_NAME = 'docker-daemon'

# location for build info directory in the image
IMAGE_BUILD_INFO_DIR = '/root/buildinfo/'

USER_CONFIG_FILES = {
    # filename: json schema file
    REPO_FETCH_ARTIFACTS_URL: 'schemas/fetch-artifacts-url.json',
    REPO_FETCH_ARTIFACTS_PNC: 'schemas/fetch-artifacts-pnc.json',
    REPO_FETCH_ARTIFACTS_KOJI: 'schemas/fetch-artifacts-nvr.json',
    REPO_CONTENT_SETS_CONFIG: 'schemas/content_sets.json',
}
<commit_msg>remove docker from list of tools we are printing<commit_after>"""
Copyright (c) 2015, 2019 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

import os

DOCKER_SOCKET_PATH = '/var/run/docker.sock'
DOCKERFILE_FILENAME = 'Dockerfile'
CACHITO_ENV_FILENAME = 'cachito.env'
CACHITO_ENV_ARG_ALIAS = 'CACHITO_ENV_FILE'
BUILD_JSON = 'build.json'
BUILD_JSON_ENV = 'BUILD_JSON'
RESULTS_JSON = 'results.json'
METADATA_TAG = "_metadata_"

WORKSPACE_BASE_PATH = '/workspace'
REACTOR_CONFIG_WORKSPACE = 'ws-reactor-config-map'
REACTOR_CONFIG_BASE_NAME = 'config.yaml'
REACTOR_CONFIG_FULL_PATH = os.path.join(
    WORKSPACE_BASE_PATH, REACTOR_CONFIG_WORKSPACE, REACTOR_CONFIG_BASE_NAME
)
REACTOR_CONFIG_ENV_NAME = 'REACTOR_CONFIG'

CONTAINER_SHARE_PATH = '/run/share/'
CONTAINER_SHARE_SOURCE_SUBDIR = 'source'
CONTAINER_SECRET_PATH = ''
CONTAINER_BUILD_JSON_PATH = os.path.join(CONTAINER_SHARE_PATH, BUILD_JSON)
CONTAINER_RESULTS_JSON_PATH = os.path.join(CONTAINER_SHARE_PATH, RESULTS_JSON)
CONTAINER_DOCKERFILE_PATH = os.path.join(CONTAINER_SHARE_PATH, DOCKERFILE_FILENAME)

HOST_SECRET_PATH = ''

EXPORTED_SQUASHED_IMAGE_NAME = 'image.tar'
EXPORTED_COMPRESSED_IMAGE_NAME_TEMPLATE = 'compressed.tar.{0}'

YUM_REPOS_DIR = '/etc/yum.repos.d/'
RELATIVE_REPOS_PATH = "atomic-reactor-repos/"
DEFAULT_YUM_REPOFILE_NAME = 'atomic-reactor-injected.repo'

# key in dictionary returned by "docker inspect" that holds the image
# configuration (such as labels)
INSPECT_CONFIG = "Config"
# key that holds the RootFS
INSPECT_ROOTFS = "RootFS"
# key that holds the layer diff_ids
INSPECT_ROOTFS_LAYERS = 'Layers'

# docs constants

DESCRIPTION = "Python library with command line interface for building container images."
HOMEPAGE = "https://github.com/containerbuildsystem/atomic-reactor"
PROG = "atomic-reactor"
MANPAGE_AUTHORS = "Jiri Popelka <jpopelka@redhat.com>, " \
                  "Martin Milata <mmilata@redhat.com>, " \
                  "Slavek Kabrda <slavek@redhat.com>, " \
                  "Tim Waugh <twaugh@redhat.com>, " \
                  "Tomas Tomecek <ttomecek@redhat.com>"
MANPAGE_SECTION = 1

# debug print of tools reactor uses

TOOLS_USED = (
    {"pkg_name": "atomic_reactor"},
    {"pkg_name": "osbs", "display_name": "osbs-client"},
)

DEFAULT_DOWNLOAD_BLOCK_SIZE = 10 * 1024 * 1024  # 10Mb

IMAGE_TYPE_DOCKER_ARCHIVE = 'docker-archive'
IMAGE_TYPE_OCI = 'oci'
IMAGE_TYPE_OCI_TAR = 'oci-tar'

PLUGIN_KOJI_PROMOTE_PLUGIN_KEY = 'koji_promote'
PLUGIN_KOJI_IMPORT_PLUGIN_KEY = 'koji_import'
PLUGIN_KOJI_IMPORT_SOURCE_CONTAINER_PLUGIN_KEY = 'koji_import_source_container'
PLUGIN_KOJI_UPLOAD_PLUGIN_KEY = 'koji_upload'
PLUGIN_KOJI_TAG_BUILD_KEY = 'koji_tag_build'
PLUGIN_ADD_FILESYSTEM_KEY = 'add_filesystem'
PLUGIN_BUMP_RELEASE_KEY = 'bump_release'
PLUGIN_DISTGIT_FETCH_KEY = 'distgit_fetch_artefacts'
PLUGIN_FETCH_MAVEN_KEY = 'fetch_maven_artifacts'
PLUGIN_MAVEN_URL_SOURCES_METADATA_KEY = 'maven_url_sources_metadata'
PLUGIN_GATHER_BUILDS_METADATA_KEY = 'gather_builds_metadata'
PLUGIN_GROUP_MANIFESTS_KEY = 'group_manifests'
PLUGIN_INJECT_PARENT_IMAGE_KEY = 'inject_parent_image'
PLUGIN_KOJI_PARENT_KEY = 'koji_parent'
PLUGIN_COMPARE_COMPONENTS_KEY = 'compare_components'
PLUGIN_CHECK_AND_SET_PLATFORMS_KEY = 'check_and_set_platforms'
PLUGIN_CHECK_USER_SETTINGS = 'check_user_settings'
PLUGIN_REMOVE_WORKER_METADATA_KEY = 'remove_worker_metadata'
PLUGIN_RESOLVE_COMPOSES_KEY = 'resolve_composes'
PLUGIN_RESOLVE_REMOTE_SOURCE = 'resolve_remote_source'
PLUGIN_SENDMAIL_KEY = 'sendmail'
PLUGIN_VERIFY_MEDIA_KEY = 'verify_media'
PLUGIN_PIN_OPERATOR_DIGESTS_KEY = 'pin_operator_digest'
PLUGIN_EXPORT_OPERATOR_MANIFESTS_KEY = 'export_operator_manifests'
PLUGIN_PUSH_OPERATOR_MANIFESTS_KEY = 'push_operator_manifests'
PLUGIN_SOURCE_CONTAINER_KEY = 'source_container'
PLUGIN_FETCH_SOURCES_KEY = 'fetch_sources'
PLUGIN_KOJI_DELEGATE_KEY = 'koji_delegate'
PLUGIN_PUSH_FLOATING_TAGS_KEY = 'push_floating_tags'
PLUGIN_ADD_IMAGE_CONTENT_MANIFEST = 'add_image_content_manifest'
PLUGIN_CANCEL_BUILD_RESERVATION = 'cancel_build_reservation'
PLUGIN_FLATPAK_CREATE_OCI = 'flatpak_create_oci'

# some shared dict keys for build metadata that gets recorded with koji.
# for consistency of metadata in historical builds, these values basically cannot change.
# however constant names could change to more accurately reflect current semantics.
BASE_IMAGE_KOJI_BUILD = 'parent-image-koji-build'  # from when the base image was the only parent
PARENT_IMAGES_KOJI_BUILDS = 'parent-images-koji-builds'
BASE_IMAGE_BUILD_ID_KEY = 'parent_build_id'  # from when the base image was the only parent
PARENT_IMAGE_BUILDS_KEY = 'parent_image_builds'
PARENT_IMAGES_KEY = 'parent_images'
SCRATCH_FROM = 'scratch'

# max retries for docker requests
DOCKER_MAX_RETRIES = 3
# how many seconds should wait before another try of docker request
DOCKER_BACKOFF_FACTOR = 5
# docker retries statuses
DOCKER_CLIENT_STATUS_RETRY = (408, 500, 502, 503, 504)
# max retries for docker push
DOCKER_PUSH_MAX_RETRIES = 6
# how many seconds should wait before another try of docker push
DOCKER_PUSH_BACKOFF_FACTOR = 5
# max retries for http requests
HTTP_MAX_RETRIES = 10
# how many seconds should wait before another try of http request
HTTP_BACKOFF_FACTOR = 5
# http retries statuses
HTTP_CLIENT_STATUS_RETRY = (408, 429, 500, 502, 503, 504)
# requests timeout in seconds
HTTP_REQUEST_TIMEOUT = 600
# max retries for git clone
GIT_MAX_RETRIES = 3
# how many seconds should wait before another try of git clone
GIT_BACKOFF_FACTOR = 5
# max retries for reserving koji builds
KOJI_RESERVE_MAX_RETRIES = 20
# wait for 2sec (usual time of bump_release with reserve)
KOJI_RESERVE_RETRY_DELAY = 2
KOJI_MAX_RETRIES = 120
KOJI_RETRY_INTERVAL = 60
KOJI_OFFLINE_RETRY_INTERVAL = 120
# max retries for subprocesses (see utils.retries.run_cmd())
SUBPROCESS_MAX_RETRIES = 5
# the factor for the exponential backoff series - 5, 10, 20, 40, 80 seconds of waiting
SUBPROCESS_BACKOFF_FACTOR = 5

# Media types
MEDIA_TYPE_DOCKER_V2_SCHEMA1 = "application/vnd.docker.distribution.manifest.v1+json"
MEDIA_TYPE_DOCKER_V2_SCHEMA2 = "application/vnd.docker.distribution.manifest.v2+json"
MEDIA_TYPE_DOCKER_V2_MANIFEST_LIST = "application/vnd.docker.distribution.manifest.list.v2+json"
MEDIA_TYPE_OCI_V1 = "application/vnd.oci.image.manifest.v1+json"
MEDIA_TYPE_OCI_V1_INDEX = "application/vnd.oci.image.index.v1+json"

# PNC related constants
PNC_SYSTEM_USER = "newcastle"

REPO_CONTAINER_CONFIG = 'container.yaml'
REPO_CONTENT_SETS_CONFIG = 'content_sets.yml'
REPO_FETCH_ARTIFACTS_URL = 'fetch-artifacts-url.yaml'
REPO_FETCH_ARTIFACTS_PNC = 'fetch-artifacts-pnc.yaml'
REPO_FETCH_ARTIFACTS_KOJI = 'fetch-artifacts-koji.yaml'

DOCKERIGNORE = '.dockerignore'

# Operator manifest constants
OPERATOR_MANIFESTS_ARCHIVE = 'operator_manifests.zip'

KOJI_BTYPE_IMAGE = 'image'
KOJI_BTYPE_OPERATOR_MANIFESTS = 'operator-manifests'
KOJI_BTYPE_REMOTE_SOURCES = 'remote-sources'
KOJI_BTYPE_REMOTE_SOURCE_FILE = 'remote-source-file'

# Path to where the remote source bundle is copied to during the build process
REMOTE_SOURCE_DIR = '/remote-source'

# Name of downloaded remote sources tarball
REMOTE_SOURCE_TARBALL_FILENAME = 'remote-source.tar.gz'
REMOTE_SOURCE_JSON_FILENAME = 'remote-source.json'

# koji osbs_build metadata
KOJI_KIND_IMAGE_BUILD = 'container_build'
KOJI_KIND_IMAGE_SOURCE_BUILD = 'source_container_build'
KOJI_SUBTYPE_OP_APPREGISTRY = 'operator_appregistry'
KOJI_SUBTYPE_OP_BUNDLE = 'operator_bundle'
KOJI_SOURCE_ENGINE = 'bsi'

# Storage names as defined in skopeo
DOCKER_STORAGE_TRANSPORT_NAME = 'docker-daemon'

# location for build info directory in the image
IMAGE_BUILD_INFO_DIR = '/root/buildinfo/'

USER_CONFIG_FILES = {
    # filename: json schema file
    REPO_FETCH_ARTIFACTS_URL: 'schemas/fetch-artifacts-url.json',
    REPO_FETCH_ARTIFACTS_PNC: 'schemas/fetch-artifacts-pnc.json',
    REPO_FETCH_ARTIFACTS_KOJI: 'schemas/fetch-artifacts-nvr.json',
    REPO_CONTENT_SETS_CONFIG: 'schemas/content_sets.json',
}
<|endoftext|>


<commit_before>
__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

import datetime
import user_metric as um

class BytesAdded(um.UserMetric):
    """
        Produces a float value that reflects the rate of edit behaviour

            `https://meta.wikimedia.org/wiki/Research:Metrics/edit_rate`

            usage e.g.: ::

                >>> import classes.Metrics as M
                >>> M.BytesAdded(date_start='2012-07-30 00:00:00', raw_count=False, mode=1).process(123456)
                5
                1200
    """

    @staticmethod
    def __doc__():
        return 'process-**kwargs:\n"log_frequency" frequency to log to stdout.' \
               '\n"log_progress" log metrics gathering.'

    def __init__(self,
                 date_start='2001-01-01 00:00:00',
                 date_end=datetime.datetime.now(),
                 project='enwiki',
                 **kwargs):

        """
            - Parameters:
                - **date_start**: string or datetime.datetime. start date of edit interval
                - **date_end**: string or datetime.datetime. end date of edit interval
                - **raw_count**: Boolean. Flag that when set to True returns one total count for all users.  Count by user otherwise.

            - Return:
                - Empty.
        """
        self._start_ts_ = self._get_timestamp(date_start)
        self._end_ts_ = self._get_timestamp(date_end)
        um.UserMetric.__init__(self, project=project, **kwargs)

    @staticmethod
    def header(): return ['user_id', 'bytes_added_net', 'bytes_added_absolute', 'bytes_added_pos', 'bytes_added_neg', 'edit_count']

    def process(self, user_handle=None, is_id=True, **kwargs):

        """
            Determine the bytes added over a number of revisions for user(s).  The parameter *user_handle* can be either a string or an integer or a
            list of these types.  When the *user_handle* type is integer it is interpreted as a user id, and as a user_name for string input.  If a list
            of users is passed to the *process* method then a dict object with edit rates keyed by user handles is returned.

            The flow of the request is as follows:

                #. Get all revisions for the specified users in the given timeframe
                #. For each parent revision get its length
                #. Compute the difference in length between each revision and its parent
                #. Record edit count, raw bytes added (with sign and absolute), amount of positive bytes added, amount of negative bytes added

            - Parameters:
                - **user_handle** - String or Integer (optionally lists).  Value or list of values representing user handle(s).
                - **is_id** - Boolean.  Flag indicating whether user_handle stores user names or user ids

            - Return:
                - Dictionary. key(string): user handle, value(Float): edit counts
        """

        freq = 0
        is_log = False
        if 'log_progress' in kwargs: is_log = bool(kwargs['log_progress'])
        if 'log_frequency' in kwargs: freq = int(kwargs['log_frequency'])

        bytes_added = dict()
        ts_condition  = 'rev_timestamp >= "%s" and rev_timestamp < "%s"' % (self._start_ts_, self._end_ts_)

        # determine the format field
        field_name = ['rev_user_text','rev_user'][is_id]

        # build the user set for inclusion into the query
        if not user_handle is None:
            user_handle = self._escape_var(user_handle) # Escape user_handle for SQL injection
            if not hasattr(user_handle, '__iter__'): user_handle = [user_handle] # ensure the handles are iterable
            if is_id:
                user_set = um.dl.DataLoader().format_comma_separated_list(user_handle, include_quotes=False)
            else:
                user_set = um.dl.DataLoader().format_comma_separated_list(user_handle, include_quotes=True)
            where_clause = '%(field_name)s in (%(user_set)s) and %(ts_condition)s' % {
            'field_name' : field_name, 'user_set' : user_set, 'ts_condition' : ts_condition}
        else:
            where_clause = '%(ts_condition)s' % {'ts_condition' : ts_condition}

        sql = """
                select
                    %(field_name)s,
                    rev_len,
                    rev_parent_id
                from %(project)s.revision
                where %(where_clause)s
            """ % {
                'field_name' : field_name,
                'where_clause' : where_clause,
                'project' : self._project_}
        sql = " ".join(sql.strip().split())

        if is_log:
            print str(datetime.datetime.now()) + ' - Querying revisions...'
        try:
            cur_1 = self._data_source_._db_.cursor()
            cur_1.execute(sql)
        except um.MySQLdb.ProgrammingError:
            raise self.UserMetricError(message=str(self.__class__()) +
                                               '::Could not get bytes added for specified users(s) - Query Failed.')

        # Get the difference for each revision length from the parent to compute bytes added
        cur_2 = self._data_source_._db_.cursor() # Get a new cursor for rev length queries
        row_count = 1
        missed_records = 0
        total_rows = len(cur_1._rows)

        print str(datetime.datetime.now()) + ' - Processing revision data by user...'
        for row in cur_1.fetchall():
            try:
                user = str(row[0])
                rev_len_total = int(row[1])
                parent_rev_id = row[2]

            except IndexError:
                missed_records += 1
                continue
            except TypeError:
                missed_records += 1
                continue

            # Produce the revision length of the parent
            if parent_rev_id == 0: # In case of a new article, parent_rev_id = 0, no record in the db
                parent_rev_len = 0
            else:
                sql = 'select rev_len from enwiki.revision where rev_id = %(parent_rev_id)s' % {
                      'parent_rev_id' : parent_rev_id}
                try:
                    cur_2.execute(sql)
                    parent_rev_len = int(cur_2.fetchall()[0][0])
                except TypeError:
                    missed_records += 1
                    continue
                except um.MySQLdb.ProgrammingError:
                    raise um.UserMetric.UserMetricError(message=str(self.__class__()) +
                                '::Could not produce rev diff for %s on rev_id %s.' % (user, str(parent_rev_id)))

            # Update the bytes added hash
            bytes_added_bit = rev_len_total - parent_rev_len

            try: # Exception where the user does not exist.  Handle this by creating the key
                bytes_added[user][0] += bytes_added_bit
            except KeyError:
                bytes_added[user] = [0] * 5
                bytes_added[user][0] += bytes_added_bit
                pass

            bytes_added[user][1] += abs(bytes_added_bit)
            if bytes_added_bit > 0:
                bytes_added[user][2] += bytes_added_bit
            else:
                bytes_added[user][3] += bytes_added_bit
            bytes_added[user][4] += 1

            if freq and row_count % freq == 0 and is_log:
                s = ' - Processed %s of %s records.' % (row_count,total_rows)
                print str(datetime.datetime.now()) + s

            row_count += 1

        self._results = [[user] + bytes_added[user] for user in bytes_added]
        if is_log:
            print str(datetime.datetime.now()) + ' - processed %s out of %s records.' % (total_rows
                                                                                - missed_records,total_rows)
        return self<commit_msg>some cleanup<commit_after>
__author__ = "Ryan Faulkner"
__date__ = "July 27th, 2012"
__license__ = "GPL (version 2 or later)"

import datetime
import user_metric as um

class BytesAdded(um.UserMetric):
    """
        Produces a float value that reflects the rate of edit behaviour

            `https://meta.wikimedia.org/wiki/Research:Metrics/edit_rate`

            usage e.g.: ::

                >>> import classes.Metrics as M
                >>> M.BytesAdded(date_start='2012-07-30 00:00:00', raw_count=False, mode=1).process(123456)
                5
                1200
    """

    @staticmethod
    def __doc__():
        return 'process-**kwargs:\n"log_frequency" frequency to log to stdout.' \
               '\n"log_progress" log metrics gathering.'

    def __init__(self,
                 date_start='2001-01-01 00:00:00',
                 date_end=datetime.datetime.now(),
                 project='enwiki',
                 **kwargs):

        """
            - Parameters:
                - **date_start**: string or datetime.datetime. start date of edit interval
                - **date_end**: string or datetime.datetime. end date of edit interval
                - **raw_count**: Boolean. Flag that when set to True returns one total count for all users.
                                            Count by user otherwise.

            - Return:
                - Empty.
        """
        self._start_ts_ = self._get_timestamp(date_start)
        self._end_ts_ = self._get_timestamp(date_end)
        um.UserMetric.__init__(self, project=project, **kwargs)

    @staticmethod
    def header(): return ['user_id', 'bytes_added_net', 'bytes_added_absolute',
                          'bytes_added_pos', 'bytes_added_neg', 'edit_count']

    def process(self, user_handle=None, is_id=True, **kwargs):

        """
            Determine the bytes added over a number of revisions for user(s).  The parameter *user_handle* can
            be either a string or an integer or a list of these types.  When the *user_handle* type is integer it is
            interpreted as a user id, and as a user_name for string input.  If a list of users is passed to the
            *process* method then a dict object with edit rates keyed by user handles is returned.

            The flow of the request is as follows:

                #. Get all revisions for the specified users in the given timeframe
                #. For each parent revision get its length
                #. Compute the difference in length between each revision and its parent
                #. Record edit count, raw bytes added (with sign and absolute), amount of positive bytes added,
                amount of negative bytes added

            - Parameters:
                - **user_handle** - String or Integer (optionally lists).  Value or list of values representing
                                    user handle(s).
                - **is_id** - Boolean.  Flag indicating whether user_handle stores user names or user ids

            - Return:
                - Dictionary. key(string): user handle, value(Float): edit counts
        """

        freq = 0
        is_log = False
        if 'log_progress' in kwargs: is_log = bool(kwargs['log_progress'])
        if 'log_frequency' in kwargs: freq = int(kwargs['log_frequency'])

        bytes_added = dict()
        ts_condition  = 'rev_timestamp >= "%s" and rev_timestamp < "%s"' % (self._start_ts_, self._end_ts_)

        # determine the format field
        field_name = ['rev_user_text','rev_user'][is_id]

        # build the user set for inclusion into the query
        if not user_handle is None:
            user_handle = self._escape_var(user_handle) # Escape user_handle for SQL injection
            if not hasattr(user_handle, '__iter__'): user_handle = [user_handle] # ensure the handles are iterable
            if is_id:
                user_set = um.dl.DataLoader().format_comma_separated_list(user_handle, include_quotes=False)
            else:
                user_set = um.dl.DataLoader().format_comma_separated_list(user_handle, include_quotes=True)
            where_clause = '%(field_name)s in (%(user_set)s) and %(ts_condition)s' % {
            'field_name' : field_name, 'user_set' : user_set, 'ts_condition' : ts_condition}
        else:
            where_clause = '%(ts_condition)s' % {'ts_condition' : ts_condition}

        sql = """
                select
                    %(field_name)s,
                    rev_len,
                    rev_parent_id
                from %(project)s.revision
                where %(where_clause)s
            """ % {
                'field_name' : field_name,
                'where_clause' : where_clause,
                'project' : self._project_}
        sql = " ".join(sql.strip().split())

        if is_log:
            print str(datetime.datetime.now()) + ' - Querying revisions...'
        try:
            cur_1 = self._data_source_._db_.cursor()
            cur_1.execute(sql)
        except um.MySQLdb.ProgrammingError:
            raise self.UserMetricError(message=str(self.__class__()) +
                                               '::Could not get bytes added for specified users(s) - Query Failed.')

        # Get the difference for each revision length from the parent to compute bytes added
        cur_2 = self._data_source_._db_.cursor() # Get a new cursor for rev length queries
        row_count = 1
        missed_records = 0
        total_rows = len(cur_1._rows)

        print str(datetime.datetime.now()) + ' - Processing revision data by user...'
        for row in cur_1.fetchall():
            try:
                user = str(row[0])
                rev_len_total = int(row[1])
                parent_rev_id = row[2]

            except IndexError:
                missed_records += 1
                continue
            except TypeError:
                missed_records += 1
                continue

            # Produce the revision length of the parent
            if parent_rev_id == 0: # In case of a new article, parent_rev_id = 0, no record in the db
                parent_rev_len = 0
            else:
                sql = 'select rev_len from enwiki.revision where rev_id = %(parent_rev_id)s' % {
                      'parent_rev_id' : parent_rev_id}
                try:
                    cur_2.execute(sql)
                    parent_rev_len = int(cur_2.fetchall()[0][0])
                except TypeError:
                    missed_records += 1
                    continue
                except um.MySQLdb.ProgrammingError:
                    raise um.UserMetric.UserMetricError(message=str(self.__class__()) +
                                '::Could not produce rev diff for %s on rev_id %s.' % (user, str(parent_rev_id)))

            # Update the bytes added hash
            bytes_added_bit = rev_len_total - parent_rev_len

            try: # Exception where the user does not exist.  Handle this by creating the key
                bytes_added[user][0] += bytes_added_bit
            except KeyError:
                bytes_added[user] = [0] * 5
                bytes_added[user][0] += bytes_added_bit
                pass

            bytes_added[user][1] += abs(bytes_added_bit)
            if bytes_added_bit > 0:
                bytes_added[user][2] += bytes_added_bit
            else:
                bytes_added[user][3] += bytes_added_bit
            bytes_added[user][4] += 1

            if freq and row_count % freq == 0 and is_log:
                s = ' - Processed %s of %s records.' % (row_count,total_rows)
                print str(datetime.datetime.now()) + s

            row_count += 1

        self._results = [[user] + bytes_added[user] for user in bytes_added]
        if is_log:
            print str(datetime.datetime.now()) + ' - processed %s out of %s records.' % (total_rows
                                                                                - missed_records,total_rows)
        return self<|endoftext|>


<commit_before>"""
Page parsing

Parsing of web pages for extraction task.
"""
from collections import defaultdict
from numpy import array

from scrapy.utils.py26 import json

from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
        TemplatePage, ExtractionPage, AnnotationText, TokenDict)

def parse_strings(template_html, extraction_html):
    """Create a template and extraction page from raw strings

    this is useful for testing purposes
    """
    t = TokenDict()
    template_page = HtmlPage(body=template_html)
    extraction_page = HtmlPage(body=extraction_html)
    return (parse_template(t, template_page),
            parse_extraction_page(t, extraction_page))

def parse_template(token_dict, template_html):
    """Create an TemplatePage object by parsing the annotated html"""
    parser = TemplatePageParser(token_dict)
    parser.feed(template_html)
    return parser.to_template()

def parse_extraction_page(token_dict, page_html):
    """Create an ExtractionPage object by parsing the html"""
    parser = ExtractionPageParser(token_dict)
    parser.feed(page_html)
    return parser.to_extraction_page()

class InstanceLearningParser(object):
    """Base parser for instance based learning algorithm

    This does not require correct HTML and the parsing method should not alter
    the original tag order. It is important that parsing results do not vary.
    """
    def __init__(self, token_dict):
        self.token_dict = token_dict
        self.token_list = []

    def _add_token(self, token, token_type, start, end):
        tid = self.token_dict.tokenid(token, token_type)
        self.token_list.append(tid)

    def feed(self, html_page):
        self.html_page = html_page
        self.previous_element_class = None
        for data in html_page.parsed_body:
            if isinstance(data, HtmlTag):
                self._add_token(data.tag, data.tag_type, data.start, data.end)
                self.handle_tag(data)
            else:
                self.handle_data(data)
            self.previous_element_class = data.__class__

    def handle_data(self, html_data_fragment):
        pass

    def handle_tag(self, html_tag):
        pass

_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]

class TemplatePageParser(InstanceLearningParser):
    """Template parsing for instance based learning algorithm"""

    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.annotations = []
        self.ignored_regions = []
        self.extra_required_attrs = []
        self.ignored_tag_stacks = defaultdict(list)
        # tag names that have not been completed
        self.labelled_tag_stacks = defaultdict(list)
        self.replacement_stacks = defaultdict(list)
        self.unpairedtag_stack = []
        self.variant_stack = []
        self.prev_data = None
        self.last_text_region = None
        self.next_tag_index = 0

    def handle_tag(self, html_tag):
        if self.last_text_region:
            self._process_text('')

        if html_tag.tag_type == HtmlTagType.OPEN_TAG:
            self._handle_open_tag(html_tag)
        elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
            self._handle_close_tag(html_tag)
        else:
            # the tag is not paired, it can contain only attribute annotations
            self._handle_unpaired_tag(html_tag)

    @staticmethod
    def _read_template_annotation(html_tag):
        template_attr = html_tag.attributes.get('data-scrapy-annotate')
        if template_attr is None:
            return None
        unescaped = template_attr.replace('"', '"')
        return json.loads(unescaped)

    @staticmethod
    def _read_bool_template_attribute(html_tag, attribute):
        return html_tag.attributes.get("data-scrapy-" + attribute) == "true"

    def _close_unpaired_tag(self):
        self.unpairedtag_stack[0].end_index = self.next_tag_index
        self.unpairedtag_stack = []

    def _handle_unpaired_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
        elif self._read_bool_template_attribute(html_tag, "ignore-beneath") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, None))
        jannotation = self._read_template_annotation(html_tag)
        if jannotation:
            if self.unpairedtag_stack:
                self._close_unpaired_tag()

            annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
            attribute_annotations = jannotation.pop('annotations', {}).items()
            for extract_attribute, tag_value in attribute_annotations:
                if extract_attribute == 'content':
                    annotation.surrounds_attribute = tag_value
                    self.unpairedtag_stack.append(annotation)
                else:
                    annotation.tag_attributes.append((extract_attribute, tag_value))
            self.annotations.append(annotation)
            if jannotation.pop('common_prefix', False):
                annotation.match_common_prefix = True

            self.extra_required_attrs.extend(jannotation.pop('required', []))
            annotation.metadata = jannotation

        self.next_tag_index += 1

    def _handle_open_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore"):
            if html_tag.tag == "img":
                self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
            else:
                self.ignored_regions.append((self.next_tag_index, None))
                self.ignored_tag_stacks[html_tag.tag].append(html_tag)

        elif self.ignored_tag_stacks.get(html_tag.tag):
            self.ignored_tag_stacks[html_tag.tag].append(None)
        if self._read_bool_template_attribute(html_tag, "ignore-beneath"):
            self.ignored_regions.append((self.next_tag_index, None))

        replacement = html_tag.attributes.pop("data-scrapy-replacement", None)
        if replacement:
            self.token_list.pop()
            self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
            self.replacement_stacks[html_tag.tag].append(replacement)
        elif html_tag.tag in self.replacement_stacks:
            self.replacement_stacks[html_tag.tag].append(None)

        if self.unpairedtag_stack:
            if html_tag.tag in _END_UNPAIREDTAG_TAGS:
                self._close_unpaired_tag()
            else:
                self.unpairedtag_stack.append(html_tag.tag)

        # can't be a p inside another p. Also, an open p element closes
        # a previous open p element.
        if html_tag.tag == "p" and html_tag.tag in self.labelled_tag_stacks:
            annotation = self.labelled_tag_stacks.pop(html_tag.tag)[0]
            annotation.end_index = self.next_tag_index
            self.annotations.append(annotation)

        jannotation = self._read_template_annotation(html_tag)
        if not jannotation:
            if html_tag.tag in self.labelled_tag_stacks:
                # add this tag to the stack to match correct end tag
                self.labelled_tag_stacks[html_tag.tag].append(None)
            self.next_tag_index += 1
            return

        annotation = AnnotationTag(self.next_tag_index, None)
        if jannotation.pop('generated', False):
            self.token_list.pop()
            annotation.start_index -= 1
            if self.previous_element_class == HtmlTag:
                annotation.annotation_text = AnnotationText('')
            else:
                annotation.annotation_text = AnnotationText(self.prev_data)
            if self._read_bool_template_attribute(html_tag, "ignore") \
                    or self._read_bool_template_attribute(html_tag, "ignore-beneath"):
                ignored = self.ignored_regions.pop()
                self.ignored_regions.append((ignored[0]-1, ignored[1]))

        if jannotation.pop('common_prefix', False):
            annotation.match_common_prefix = True

        self.extra_required_attrs.extend(jannotation.pop('required', []))

        variant_id = jannotation.pop('variant', 0)
        if variant_id > 0:
            self.variant_stack.append(variant_id)
            annotation.surrounds_variant = variant_id
        attribute_annotations = jannotation.pop('annotations', {}).items()
        for extract_attribute, tag_value in attribute_annotations:
            if extract_attribute == 'content':
                annotation.surrounds_attribute = tag_value
            else:
                annotation.tag_attributes.append((extract_attribute, tag_value))

        annotation.metadata = jannotation

        if annotation.annotation_text is None:
            self.next_tag_index += 1
        if self.variant_stack:
            variant_id = self.variant_stack[-1]
            if variant_id == '0':
                variant_id = None
            annotation.variant_id = variant_id

        # look for a closing tag if the content is important
        if annotation.surrounds_attribute or annotation.surrounds_variant:
            self.labelled_tag_stacks[html_tag.tag].append(annotation)
        else:
            annotation.end_index = annotation.start_index + 1
            self.annotations.append(annotation)

    def _handle_close_tag(self, html_tag):

        if self.unpairedtag_stack:
            if html_tag.tag == self.unpairedtag_stack[-1]:
                self.unpairedtag_stack.pop()
            else:
                self._close_unpaired_tag()
        ignored_tags = self.ignored_tag_stacks.get(html_tag.tag)
        if ignored_tags is not None:
            tag = ignored_tags.pop()
            if isinstance(tag, HtmlTag):
                for i in range(-1, -len(self.ignored_regions) - 1, -1):
                    if self.ignored_regions[i][1] is None:
                        self.ignored_regions[i] = (self.ignored_regions[i][0], self.next_tag_index)
                        break
            if len(ignored_tags) == 0:
                del self.ignored_tag_stacks[html_tag.tag]

        if html_tag.tag in self.replacement_stacks:
            replacement = self.replacement_stacks[html_tag.tag].pop()
            if replacement:
                self.token_list.pop()
                self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
            if len(self.replacement_stacks[html_tag.tag]) == 0:
                del self.replacement_stacks[html_tag.tag]

        labelled_tags = self.labelled_tag_stacks.get(html_tag.tag)
        if labelled_tags is None:
            self.next_tag_index += 1
            return
        annotation = labelled_tags.pop()
        if annotation is None:
            self.next_tag_index += 1
        else:
            annotation.end_index = self.next_tag_index
            self.annotations.append(annotation)
            if annotation.annotation_text is not None:
                self.token_list.pop()
                self.last_text_region = annotation
            else:
                self.next_tag_index += 1
            if len(labelled_tags) == 0:
                del self.labelled_tag_stacks[html_tag.tag]
            if annotation.surrounds_variant and self.variant_stack:
                prev = self.variant_stack.pop()
                if prev != annotation.surrounds_variant:
                    raise ValueError("unbalanced variant annotation tags")

    def handle_data(self, html_data_fragment):
        fragment_text = self.html_page.fragment_data(html_data_fragment)
        self._process_text(fragment_text)

    def _process_text(self, text):
        if self.last_text_region is not None:
            self.last_text_region.annotation_text.follow_text = text
            self.last_text_region = None
        self.prev_data = text

    def to_template(self):
        """create a TemplatePage from the data fed to this parser"""
        return TemplatePage(self.token_dict, self.token_list, self.annotations,
                self.html_page.page_id, self.ignored_regions, self.extra_required_attrs)

class ExtractionPageParser(InstanceLearningParser):
    """Parse an HTML page for extraction using the instance based learning
    algorithm

    This needs to extract the tokens in a similar way to LabelledPageParser,
    it needs to also maintain a mapping from token index to the original content
    so that once regions are identified, the original content can be extracted.
    """
    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.page_data = []
        self.token_start_index = []
        self.token_follow_index = []
        self.tag_attrs = {}

    def _add_token(self, token, token_type, start, end):
        InstanceLearningParser._add_token(self, token, token_type, start, end)
        self.token_start_index.append(start)
        self.token_follow_index.append(end)

    def handle_tag(self, html_tag):
        if html_tag.attributes:
            self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes

    def to_extraction_page(self):
        return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list),
                self.token_start_index, self.token_follow_index, self.tag_attrs)
<commit_msg>Automated merge with ssh://hg@hg.scrapy.org:2222/scrapy<commit_after>"""
Page parsing

Parsing of web pages for extraction task.
"""
from collections import defaultdict
from numpy import array

from scrapy.utils.py26 import json

from scrapy.contrib.ibl.htmlpage import HtmlTagType, HtmlTag, HtmlPage
from scrapy.contrib.ibl.extraction.pageobjects import (AnnotationTag,
        TemplatePage, ExtractionPage, AnnotationText, TokenDict)

def parse_strings(template_html, extraction_html):
    """Create a template and extraction page from raw strings

    this is useful for testing purposes
    """
    t = TokenDict()
    template_page = HtmlPage(body=template_html)
    extraction_page = HtmlPage(body=extraction_html)
    return (parse_template(t, template_page),
            parse_extraction_page(t, extraction_page))

def parse_template(token_dict, template_html):
    """Create an TemplatePage object by parsing the annotated html"""
    parser = TemplatePageParser(token_dict)
    parser.feed(template_html)
    return parser.to_template()

def parse_extraction_page(token_dict, page_html):
    """Create an ExtractionPage object by parsing the html"""
    parser = ExtractionPageParser(token_dict)
    parser.feed(page_html)
    return parser.to_extraction_page()

class InstanceLearningParser(object):
    """Base parser for instance based learning algorithm

    This does not require correct HTML and the parsing method should not alter
    the original tag order. It is important that parsing results do not vary.
    """
    def __init__(self, token_dict):
        self.token_dict = token_dict
        self.token_list = []

    def _add_token(self, token, token_type, start, end):
        tid = self.token_dict.tokenid(token, token_type)
        self.token_list.append(tid)

    def feed(self, html_page):
        self.html_page = html_page
        self.previous_element_class = None
        for data in html_page.parsed_body:
            if isinstance(data, HtmlTag):
                self._add_token(data.tag, data.tag_type, data.start, data.end)
                self.handle_tag(data)
            else:
                self.handle_data(data)
            self.previous_element_class = data.__class__

    def handle_data(self, html_data_fragment):
        pass

    def handle_tag(self, html_tag):
        pass

_END_UNPAIREDTAG_TAGS = ["form", "div", "p", "table", "tr", "td"]

class TemplatePageParser(InstanceLearningParser):
    """Template parsing for instance based learning algorithm"""

    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.annotations = []
        self.ignored_regions = []
        self.extra_required_attrs = []
        self.ignored_tag_stacks = defaultdict(list)
        # tag names that have not been completed
        self.labelled_tag_stacks = defaultdict(list)
        self.replacement_stacks = defaultdict(list)
        self.unpairedtag_stack = []
        self.variant_stack = []
        self.prev_data = None
        self.last_text_region = None
        self.next_tag_index = 0

    def handle_tag(self, html_tag):
        if self.last_text_region:
            self._process_text('')

        if html_tag.tag_type == HtmlTagType.OPEN_TAG:
            self._handle_open_tag(html_tag)
        elif html_tag.tag_type == HtmlTagType.CLOSE_TAG:
            self._handle_close_tag(html_tag)
        else:
            # the tag is not paired, it can contain only attribute annotations
            self._handle_unpaired_tag(html_tag)

    @staticmethod
    def _read_template_annotation(html_tag):
        template_attr = html_tag.attributes.get('data-scrapy-annotate')
        if template_attr is None:
            return None
        unescaped = template_attr.replace('"', '"')
        return json.loads(unescaped)

    @staticmethod
    def _read_bool_template_attribute(html_tag, attribute):
        return html_tag.attributes.get("data-scrapy-" + attribute) == "true"

    def _close_unpaired_tag(self):
        self.unpairedtag_stack[0].end_index = self.next_tag_index
        self.unpairedtag_stack = []

    def _handle_unpaired_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore") and html_tag.tag == "img":
            self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
        elif self._read_bool_template_attribute(html_tag, "ignore-beneath"):
            self.ignored_regions.append((self.next_tag_index, None))
        jannotation = self._read_template_annotation(html_tag)
        if jannotation:
            if self.unpairedtag_stack:
                self._close_unpaired_tag()

            annotation = AnnotationTag(self.next_tag_index, self.next_tag_index + 1)
            attribute_annotations = jannotation.pop('annotations', {}).items()
            for extract_attribute, tag_value in attribute_annotations:
                if extract_attribute == 'content':
                    annotation.surrounds_attribute = tag_value
                    self.unpairedtag_stack.append(annotation)
                else:
                    annotation.tag_attributes.append((extract_attribute, tag_value))
            self.annotations.append(annotation)
            if jannotation.pop('common_prefix', False):
                annotation.match_common_prefix = True

            self.extra_required_attrs.extend(jannotation.pop('required', []))
            annotation.metadata = jannotation

        self.next_tag_index += 1

    def _handle_open_tag(self, html_tag):
        if self._read_bool_template_attribute(html_tag, "ignore"):
            if html_tag.tag == "img":
                self.ignored_regions.append((self.next_tag_index, self.next_tag_index + 1))
            else:
                self.ignored_regions.append((self.next_tag_index, None))
                self.ignored_tag_stacks[html_tag.tag].append(html_tag)

        elif self.ignored_tag_stacks.get(html_tag.tag):
            self.ignored_tag_stacks[html_tag.tag].append(None)
        if self._read_bool_template_attribute(html_tag, "ignore-beneath"):
            self.ignored_regions.append((self.next_tag_index, None))

        replacement = html_tag.attributes.pop("data-scrapy-replacement", None)
        if replacement:
            self.token_list.pop()
            self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
            self.replacement_stacks[html_tag.tag].append(replacement)
        elif html_tag.tag in self.replacement_stacks:
            self.replacement_stacks[html_tag.tag].append(None)

        if self.unpairedtag_stack:
            if html_tag.tag in _END_UNPAIREDTAG_TAGS:
                self._close_unpaired_tag()
            else:
                self.unpairedtag_stack.append(html_tag.tag)

        # can't be a p inside another p. Also, an open p element closes
        # a previous open p element.
        if html_tag.tag == "p" and html_tag.tag in self.labelled_tag_stacks:
            annotation = self.labelled_tag_stacks.pop(html_tag.tag)[0]
            annotation.end_index = self.next_tag_index
            self.annotations.append(annotation)

        jannotation = self._read_template_annotation(html_tag)
        if not jannotation:
            if html_tag.tag in self.labelled_tag_stacks:
                # add this tag to the stack to match correct end tag
                self.labelled_tag_stacks[html_tag.tag].append(None)
            self.next_tag_index += 1
            return

        annotation = AnnotationTag(self.next_tag_index, None)
        if jannotation.pop('generated', False):
            self.token_list.pop()
            annotation.start_index -= 1
            if self.previous_element_class == HtmlTag:
                annotation.annotation_text = AnnotationText('')
            else:
                annotation.annotation_text = AnnotationText(self.prev_data)
            if self._read_bool_template_attribute(html_tag, "ignore") \
                    or self._read_bool_template_attribute(html_tag, "ignore-beneath"):
                ignored = self.ignored_regions.pop()
                self.ignored_regions.append((ignored[0]-1, ignored[1]))

        if jannotation.pop('common_prefix', False):
            annotation.match_common_prefix = True

        self.extra_required_attrs.extend(jannotation.pop('required', []))

        variant_id = jannotation.pop('variant', 0)
        if variant_id > 0:
            self.variant_stack.append(variant_id)
            annotation.surrounds_variant = variant_id
        attribute_annotations = jannotation.pop('annotations', {}).items()
        for extract_attribute, tag_value in attribute_annotations:
            if extract_attribute == 'content':
                annotation.surrounds_attribute = tag_value
            else:
                annotation.tag_attributes.append((extract_attribute, tag_value))

        annotation.metadata = jannotation

        if annotation.annotation_text is None:
            self.next_tag_index += 1
        if self.variant_stack:
            variant_id = self.variant_stack[-1]
            if variant_id == '0':
                variant_id = None
            annotation.variant_id = variant_id

        # look for a closing tag if the content is important
        if annotation.surrounds_attribute or annotation.surrounds_variant:
            self.labelled_tag_stacks[html_tag.tag].append(annotation)
        else:
            annotation.end_index = annotation.start_index + 1
            self.annotations.append(annotation)

    def _handle_close_tag(self, html_tag):

        if self.unpairedtag_stack:
            if html_tag.tag == self.unpairedtag_stack[-1]:
                self.unpairedtag_stack.pop()
            else:
                self._close_unpaired_tag()
        ignored_tags = self.ignored_tag_stacks.get(html_tag.tag)
        if ignored_tags is not None:
            tag = ignored_tags.pop()
            if isinstance(tag, HtmlTag):
                for i in range(-1, -len(self.ignored_regions) - 1, -1):
                    if self.ignored_regions[i][1] is None:
                        self.ignored_regions[i] = (self.ignored_regions[i][0], self.next_tag_index)
                        break
            if len(ignored_tags) == 0:
                del self.ignored_tag_stacks[html_tag.tag]

        if html_tag.tag in self.replacement_stacks:
            replacement = self.replacement_stacks[html_tag.tag].pop()
            if replacement:
                self.token_list.pop()
                self._add_token(replacement, html_tag.tag_type, html_tag.start, html_tag.end)
            if len(self.replacement_stacks[html_tag.tag]) == 0:
                del self.replacement_stacks[html_tag.tag]

        labelled_tags = self.labelled_tag_stacks.get(html_tag.tag)
        if labelled_tags is None:
            self.next_tag_index += 1
            return
        annotation = labelled_tags.pop()
        if annotation is None:
            self.next_tag_index += 1
        else:
            annotation.end_index = self.next_tag_index
            self.annotations.append(annotation)
            if annotation.annotation_text is not None:
                self.token_list.pop()
                self.last_text_region = annotation
            else:
                self.next_tag_index += 1
            if len(labelled_tags) == 0:
                del self.labelled_tag_stacks[html_tag.tag]
            if annotation.surrounds_variant and self.variant_stack:
                prev = self.variant_stack.pop()
                if prev != annotation.surrounds_variant:
                    raise ValueError("unbalanced variant annotation tags")

    def handle_data(self, html_data_fragment):
        fragment_text = self.html_page.fragment_data(html_data_fragment)
        self._process_text(fragment_text)

    def _process_text(self, text):
        if self.last_text_region is not None:
            self.last_text_region.annotation_text.follow_text = text
            self.last_text_region = None
        self.prev_data = text

    def to_template(self):
        """create a TemplatePage from the data fed to this parser"""
        return TemplatePage(self.token_dict, self.token_list, self.annotations,
                self.html_page.page_id, self.ignored_regions, self.extra_required_attrs)

class ExtractionPageParser(InstanceLearningParser):
    """Parse an HTML page for extraction using the instance based learning
    algorithm

    This needs to extract the tokens in a similar way to LabelledPageParser,
    it needs to also maintain a mapping from token index to the original content
    so that once regions are identified, the original content can be extracted.
    """
    def __init__(self, token_dict):
        InstanceLearningParser.__init__(self, token_dict)
        self.page_data = []
        self.token_start_index = []
        self.token_follow_index = []
        self.tag_attrs = {}

    def _add_token(self, token, token_type, start, end):
        InstanceLearningParser._add_token(self, token, token_type, start, end)
        self.token_start_index.append(start)
        self.token_follow_index.append(end)

    def handle_tag(self, html_tag):
        if html_tag.attributes:
            self.tag_attrs[len(self.token_list) - 1] = html_tag.attributes

    def to_extraction_page(self):
        return ExtractionPage(self.html_page.body, self.token_dict, array(self.token_list),
                self.token_start_index, self.token_follow_index, self.tag_attrs)
<|endoftext|>


<commit_before>import numpy
import pytest

import cupy
from cupy import testing
import cupyx
import cupyx.scipy.stats  # NOQA

try:
    import scipy.stats
except ImportError:
    pass

atol = {'default': 1e-6, cupy.float64: 1e-14}
rtol = {'default': 1e-6, cupy.float64: 1e-14}

@testing.with_requires('scipy')
class TestTrim:

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(
        scipy_name='scp', rtol=1e-6, contiguous_check=False)
    @pytest.mark.parametrize('shape', [(24,), (6, 4), (6, 4, 3), (4, 6)])
    def test_base(self, xp, scp, dtype, order, shape):
        a = testing.shaped_random(
            shape, xp=xp, dtype=dtype, order=order, scale=100)
        return scp.stats.trim_mean(a, 2 / 6.)

    @testing.for_all_dtypes()
    def test_zero_dim(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            a = xp.array(0, dtype=dtype)
            with pytest.raises(IndexError):
                return scp.stats.trim_mean(a, 2 / 6.)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zero_dim_axis_none(self, xp, scp, dtype):
        a = xp.array(0, dtype=dtype)
        return scp.stats.trim_mean(a, 2 / 6., axis=None)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(scipy_name='scp')
    @pytest.mark.parametrize('propotiontocut', [0.0, 0.6])
    def test_empty(self, xp, scp, dtype, propotiontocut):
        a = xp.array([])
        return scp.stats.trim_mean(a, 2 / 6., propotiontocut)

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(
        scipy_name='scp', rtol=1e-6, contiguous_check=False)
    @pytest.mark.parametrize('axis', [0, 1, 2, 3, -1, None])
    def test_axis(self, xp, scp, dtype, order, axis):
        a = testing.shaped_random(
            (5, 6, 4, 7), xp=xp, dtype=dtype, order=order, scale=100)
        return scp.stats.trim_mean(a, 2 / 6., axis=axis)

    def test_propotion_too_big(self):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            a = xp.array([4, 8, 2, 0, 9, 5, 10, 1, 7, 3, 6])
            with pytest.raises(ValueError):
                scp.stats.trim_mean(a, 0.6)

@testing.with_requires('scipy')
class TestZmap:

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_1dim(self, xp, scp, dtype):
        x = testing.shaped_random((10,), xp, dtype=dtype)
        y = testing.shaped_random((8,), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_2dim(self, xp, scp, dtype):
        x = testing.shaped_random((2, 6), xp, dtype=dtype)
        y = testing.shaped_random((2, 1), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_multi_dim(self, xp, scp, dtype):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
        y = testing.shaped_random((3, 4, 1, 1), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_multi_dim_2(self, xp, scp, dtype):
        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zmap_multi_dim_2_float16(self, xp, scp):
        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_with_axis(self, xp, scp, dtype):
        x = testing.shaped_random((2, 3), xp, dtype=dtype)
        y = testing.shaped_random((1, 3), xp, dtype=dtype)
        return scp.stats.zmap(x, y, axis=1)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_with_axis_ddof(self, xp, scp, dtype):
        x = testing.shaped_random((4, 5), xp, dtype=dtype)
        y = testing.shaped_random((1, 5), xp, dtype=dtype)
        return scp.stats.zmap(x, y, axis=1, ddof=2)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_empty(self, xp, scp, dtype):
        x = xp.array([], dtype=dtype)
        y = xp.array([1, 3, 5], dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_propagate(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        return scp.stats.zmap(x, y, nan_policy='propagate')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_omit(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        return scp.stats.zmap(x, y, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        return scp.stats.zmap(x, y, axis=0, ddof=1, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    def test_zmap_nan_policy_raise(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            x = xp.array([1, 2, 3], dtype=dtype)
            y = xp.array([8, -4, xp.nan, 4], dtype=dtype)
            with pytest.raises(ValueError):
                scp.stats.zmap(x, y, nan_policy='raise')

@testing.with_requires('scipy')
class TestZscore:

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=rtol)
    def test_zscore_1dim(self, xp, scp, dtype):
        x = testing.shaped_random((10,), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
    def test_zscore_1dim_float16(self, xp, scp):
        x = testing.shaped_random((10,), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_2dim(self, xp, scp, dtype):
        x = testing.shaped_random((5, 3), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zscore_2dim_float16(self, xp, scp):
        x = testing.shaped_random((5, 3), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_multi_dim(self, xp, scp, dtype):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zscore_multi_dim_float16(self, xp, scp):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=atol)
    def test_zscore_with_axis(self, xp, scp, dtype):
        x = testing.shaped_random((5, 6), xp, dtype=dtype)
        return scp.stats.zscore(x, axis=1)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_with_axis_ddof(self, xp, scp, dtype):
        x = testing.shaped_random((2, 3, 8), xp, dtype=dtype)
        return scp.stats.zscore(x, axis=2, ddof=2)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_empty(self, xp, scp, dtype):
        x = xp.array([], dtype=dtype)
        return scp.stats.zscore(x)

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_propagate(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        return scp.stats.zscore(x, nan_policy='propagate')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_omit(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        return scp.stats.zscore(x, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        return scp.stats.zscore(x, axis=0, ddof=1, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    def test_zscore_nan_policy_raise(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            x = xp.array([1, 2, 3, xp.nan], dtype=dtype)
            with pytest.raises(ValueError):
                scp.stats.zscore(x, nan_policy='raise')
<commit_msg>Ignore numpy's warning on an invalid value<commit_after>import numpy
import pytest

import cupy
from cupy import testing
import cupyx
import cupyx.scipy.stats  # NOQA

try:
    import scipy.stats
except ImportError:
    pass

atol = {'default': 1e-6, cupy.float64: 1e-14}
rtol = {'default': 1e-6, cupy.float64: 1e-14}

@testing.with_requires('scipy')
class TestTrim:

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(
        scipy_name='scp', rtol=1e-6, contiguous_check=False)
    @pytest.mark.parametrize('shape', [(24,), (6, 4), (6, 4, 3), (4, 6)])
    def test_base(self, xp, scp, dtype, order, shape):
        a = testing.shaped_random(
            shape, xp=xp, dtype=dtype, order=order, scale=100)
        return scp.stats.trim_mean(a, 2 / 6.)

    @testing.for_all_dtypes()
    def test_zero_dim(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            a = xp.array(0, dtype=dtype)
            with pytest.raises(IndexError):
                return scp.stats.trim_mean(a, 2 / 6.)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zero_dim_axis_none(self, xp, scp, dtype):
        a = xp.array(0, dtype=dtype)
        return scp.stats.trim_mean(a, 2 / 6., axis=None)

    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(scipy_name='scp')
    @pytest.mark.parametrize('propotiontocut', [0.0, 0.6])
    def test_empty(self, xp, scp, dtype, propotiontocut):
        a = xp.array([])
        return scp.stats.trim_mean(a, 2 / 6., propotiontocut)

    @testing.for_CF_orders()
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(
        scipy_name='scp', rtol=1e-6, contiguous_check=False)
    @pytest.mark.parametrize('axis', [0, 1, 2, 3, -1, None])
    def test_axis(self, xp, scp, dtype, order, axis):
        a = testing.shaped_random(
            (5, 6, 4, 7), xp=xp, dtype=dtype, order=order, scale=100)
        return scp.stats.trim_mean(a, 2 / 6., axis=axis)

    def test_propotion_too_big(self):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            a = xp.array([4, 8, 2, 0, 9, 5, 10, 1, 7, 3, 6])
            with pytest.raises(ValueError):
                scp.stats.trim_mean(a, 0.6)

@testing.with_requires('scipy')
class TestZmap:

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_1dim(self, xp, scp, dtype):
        x = testing.shaped_random((10,), xp, dtype=dtype)
        y = testing.shaped_random((8,), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_2dim(self, xp, scp, dtype):
        x = testing.shaped_random((2, 6), xp, dtype=dtype)
        y = testing.shaped_random((2, 1), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_multi_dim(self, xp, scp, dtype):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
        y = testing.shaped_random((3, 4, 1, 1), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_multi_dim_2(self, xp, scp, dtype):
        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zmap_multi_dim_2_float16(self, xp, scp):
        x = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
        y = testing.shaped_random((4, 4, 5, 6, 2), xp, dtype=xp.float16)
        return scp.stats.zmap(x, y)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_with_axis(self, xp, scp, dtype):
        x = testing.shaped_random((2, 3), xp, dtype=dtype)
        y = testing.shaped_random((1, 3), xp, dtype=dtype)
        return scp.stats.zmap(x, y, axis=1)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zmap_with_axis_ddof(self, xp, scp, dtype):
        x = testing.shaped_random((4, 5), xp, dtype=dtype)
        y = testing.shaped_random((1, 5), xp, dtype=dtype)
        return scp.stats.zmap(x, y, axis=1, ddof=2)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_empty(self, xp, scp, dtype):
        x = xp.array([], dtype=dtype)
        y = xp.array([1, 3, 5], dtype=dtype)
        return scp.stats.zmap(x, y)

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_propagate(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        with numpy.errstate(invalid='ignore'):  # numpy warns with complex
            return scp.stats.zmap(x, y, nan_policy='propagate')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_omit(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        return scp.stats.zmap(x, y, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zmap_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        y = xp.array([xp.nan, -4.0, -1.0, -5.0], dtype=dtype)
        return scp.stats.zmap(x, y, axis=0, ddof=1, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    def test_zmap_nan_policy_raise(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            x = xp.array([1, 2, 3], dtype=dtype)
            y = xp.array([8, -4, xp.nan, 4], dtype=dtype)
            with pytest.raises(ValueError):
                scp.stats.zmap(x, y, nan_policy='raise')

@testing.with_requires('scipy')
class TestZscore:

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=rtol)
    def test_zscore_1dim(self, xp, scp, dtype):
        x = testing.shaped_random((10,), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', rtol=1e-3)
    def test_zscore_1dim_float16(self, xp, scp):
        x = testing.shaped_random((10,), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_2dim(self, xp, scp, dtype):
        x = testing.shaped_random((5, 3), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zscore_2dim_float16(self, xp, scp):
        x = testing.shaped_random((5, 3), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_multi_dim(self, xp, scp, dtype):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=dtype)
        return scp.stats.zscore(x)

    @testing.numpy_cupy_allclose(scipy_name='scp', atol=1e-3, rtol=1e-3)
    def test_zscore_multi_dim_float16(self, xp, scp):
        x = testing.shaped_random((3, 4, 5, 7), xp, dtype=xp.float16)
        return scp.stats.zscore(x)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=atol)
    def test_zscore_with_axis(self, xp, scp, dtype):
        x = testing.shaped_random((5, 6), xp, dtype=dtype)
        return scp.stats.zscore(x, axis=1)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp', atol=atol, rtol=rtol)
    def test_zscore_with_axis_ddof(self, xp, scp, dtype):
        x = testing.shaped_random((2, 3, 8), xp, dtype=dtype)
        return scp.stats.zscore(x, axis=2, ddof=2)

    @testing.for_all_dtypes(no_bool=True)
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_empty(self, xp, scp, dtype):
        x = xp.array([], dtype=dtype)
        return scp.stats.zscore(x)

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_propagate(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        with numpy.errstate(invalid='ignore'):  # numpy warns with complex
            return scp.stats.zscore(x, nan_policy='propagate')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_omit(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        return scp.stats.zscore(x, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    @testing.numpy_cupy_allclose(scipy_name='scp')
    def test_zscore_nan_policy_omit_axis_ddof(self, xp, scp, dtype):
        x = xp.array([4.0, 1.0, 1.0, xp.nan], dtype=dtype)
        return scp.stats.zscore(x, axis=0, ddof=1, nan_policy='omit')

    @testing.for_dtypes('fdFD')
    def test_zscore_nan_policy_raise(self, dtype):
        for xp, scp in [(numpy, scipy), (cupy, cupyx.scipy)]:
            x = xp.array([1, 2, 3, xp.nan], dtype=dtype)
            with pytest.raises(ValueError):
                scp.stats.zscore(x, nan_policy='raise')
<|endoftext|>


<commit_before>from flask import render_template, g, make_response, request, redirect, url_for
from jinja2.loaders import BaseLoader, TemplateNotFound
import os

default_article_template = """

        <title>{{ article.title }}</title>

{{config.blogtitle}}

{{config.blogdescription}}

{{article.title}}

Posted on {{article.ctime_tm|dateformat('%Y/%m/%d %H:%M')}} at {{article.fullname}}

Last modified on {{article.mtime_tm|dateformat('%Y/%m/%d %H:%M')}}

{{article}}

"""

default_article_list_template = """

        <title>{{config.blogtitle}} - {{collection_title}}</title>

{{config.blogtitle}}

{{config.blogdescription}}

{{collection_title}}

    {% if total_pages > 1 %}

       {% if prevpage %} Prev {% endif %}
       Page {{page}} of {{total_pages}}
       {% if nextpage %} Next {% endif %}

    {% endif %}

    {% for a in articles: %}

{{ a.title }}

Posted on {{a.ctime_tm|dateformat('%Y/%m/%d %H:%M')}} at {{a.fullname}}

Last modified on {{a.mtime_tm|dateformat('%Y/%m/%d %H:%M')}}

{{a}}

    {% endfor %}

"""

default_404_template = """

        <title>Not found</title>

Not found

"""

class YawtLoader(BaseLoader):
    def __init__(self, root):
        self.root = root

    def get_source(self, environment, template):
        current_category = os.path.dirname(template)
        template_name = template
        while current_category and not self._exists(template_name):
            current_category = os.path.dirname(category)
            template_name = _join(current_category, template_file)

        if not self._exists(template_name):
            raise TemplateNotFound(template)

        template_file = self._file(template_name)
        mtime = os.path.getmtime(template_file)
        with file(template_file) as f:
            source = f.read().decode('utf-8')
        return source, template_file, lambda: mtime == os.path.getmtime(template_file)

    def _file(self, template_name):
        return os.path.join(self.root, template_name)

    def _exists(self, template_name):
        return os.path.exists(self._file(template_name))

def _render(template_base, flavour=None, template_vars={}, content_type=None, category=""):
    template_root = g.config['path_to_templates']

    if flavour is None:
        flavour = 'html'
    template_file = template_base + "." + flavour

    current_category = category
    template_path = _join(current_category, template_file)
    while current_category and not os.path.exists(os.path.join(template_root, template_path)):
        current_category = os.path.dirname(category)
        template_path = _join(current_category, template_file)

    if (content_type is not None):
        response = make_response(render_template(template_path, **template_vars))
        response.headers['Content-Type'] = content_type
        return response
    else:
        return render_template(template_path, **template_vars)

def _join(category, slug):
    if category:
        return category + '/' + slug
    else:
        return slug

def _breadcrumbs(pathstr):
    breadcrumbs = []
    pathurl = ''
    for piece in pathstr.split('/'):
        pathurl += '/' + piece
        breadcrumbs.append({'crumb': piece, 'url': pathurl})
    return breadcrumbs

class PagingInfo(object):
    def __init__(self, page, page_size, total_count, base_url):
        self.page_size = page_size
        self.page = page
        self.base_url = base_url
        self.start = (self.page - 1) * self.page_size
        self.end = self.start + self.page_size

        self.total_count = total_count
        self.total_pages = self.total_count / self.page_size
        if self.total_count % self.page_size is not 0:
            self.total_pages = self.total_pages + 1

        self.has_next = self.page < self.total_pages
        self.has_prev = self.page > 1

    def url(self, page):
        return self.base_url + "?page=" + str(page)

    @property
    def next_url(self):
        if not self.has_next:
            return None
        return self.url(self.page + 1)

    @property
    def prev_url(self):
        if not self.has_prev:
            return None
        return self.url(self.page - 1)

    def __eq__(self, other):
        return self.page == other.page and \
               self.page_size == other.page_size and \
               self.total_count == other.total_count and \
               self.base_url == other.base_url

class YawtView(object):
    """
    Collection of utility methods for rendering YAWT views
    """
    def __init__(self, plugins, content_types):
        self._plugins = plugins
        self._content_types = content_types

    def render_missing_resource(self):
        return render_template("404.html")

    def render_article(self, flavour, article, breadcrumbs=None):
        template_vars = {'article': article,
                         'breadcrumbs': breadcrumbs}
        template_vars = self._plugins.template_vars(template_vars)
        return _render('article', flavour, template_vars, self._content_type(flavour),
                       article.category)

    def render_collection(self, flavour, articles, title, page_info, category="", breadcrumbs=None):
        template_vars = {'articles': articles[page_info.start:page_info.end],
                         'total_pages': page_info.total_pages,
                         'page': page_info.page,
                         'nextpage':  page_info.next_url,
                         'prevpage': page_info.prev_url,
                         'collection_title': title,
                         'breadcrumbs': breadcrumbs}
        template_vars = self._plugins.template_vars(template_vars)
        return _render('article_list', flavour, template_vars,
                       self._content_type(flavour), category)

    def _content_type(self, flavour):
        return self._content_types.get(flavour, None)

class ArticleView(object):
    """
    Class for rendering YAWT article views, which are views meant
    to display just one entry on your blog or site.
    """
    def __init__(self, store, yawtview):
        self._yawtview = yawtview
        self._store = store

    def dispatch_request(self, flavour, category, slug):
        article = self._store.fetch_article_by_category_slug(category, slug)
        if article is None:
            # no article by that name, but there might be a category
            fullname = _join(category, slug)
            if self._store.category_exists(fullname):
                # Normally flask handles this, but I don't think it can in this case
                # Basically, we have trouble figuring out if cooking/indian/madras
                # is an article or a category
                return redirect(url_for('category_canonical', category=fullname))
            else:
                return self._yawtview.render_missing_resource()
        else:
            return self._yawtview.render_article(flavour, article,
                                                 _breadcrumbs(article.fullname))

def create_article_view():
    return ArticleView(g.store, YawtView(g.plugins, g.config['content_types']))

class CategoryView(object):
    """
    Class for rendering YAWT category view, which are views meant to display
    categories, which usually have a reverse time list of articles.
    """
    def __init__(self, store, yawtview):
        self._yawtview = yawtview
        self._store = store

    # Could render a category article, or if not then
    # defaults to a list of posts
    def dispatch_request(self, flavour, category,
                         page=1, page_size=10, base_url='http://localhost'):
        category_article = _join(category, 'index')
        if self._store.article_exists(category_article):
            article = self._store.fetch_article_by_fullname(category_article)
            # index file exists
            return self._yawtview.render_article(flavour, article)
        else:
            # no index file.  Render an article list template with
            # all the articles in the category sent to the template
            articles = self._store.fetch_articles_by_category(category)
            if len(articles) < 1:
                return self._yawtview.render_missing_resource()
            else:
                page_info = PagingInfo(page, page_size, len(articles), base_url)
                return self._render_collection(flavour, articles, category, page_info)

    def _render_collection(self, flavour, articles, category, page_info):
        title = self._category_title(category)
        return self._yawtview.render_collection(flavour, articles, '',
                                                page_info, category,
                                                _breadcrumbs(category))

    def _category_title(self, category):
        if category is None or len(category) == 0:
            return ''
        return ' :: '.join(category.split('/'))

def create_category_view():
    return CategoryView(g.store, YawtView(g.plugins, g.config['content_types']))
<commit_msg>fix template loading issues<commit_after>from flask import render_template, g, make_response, request, redirect, url_for
from jinja2.loaders import BaseLoader, TemplateNotFound
import os

default_article_template = """

        <title>{{ article.title }}</title>

{{config.blogtitle}}

{{config.blogdescription}}

{{article.title}}

Posted on {{article.ctime_tm|dateformat('%Y/%m/%d %H:%M')}} at {{article.fullname}}

Last modified on {{article.mtime_tm|dateformat('%Y/%m/%d %H:%M')}}

{{article}}

"""

default_article_list_template = """

        <title>{{config.blogtitle}} - {{collection_title}}</title>

{{config.blogtitle}}

{{config.blogdescription}}

{{collection_title}}

    {% if total_pages > 1 %}

       {% if prevpage %} Prev {% endif %}
       Page {{page}} of {{total_pages}}
       {% if nextpage %} Next {% endif %}

    {% endif %}

    {% for a in articles: %}

{{ a.title }}

Posted on {{a.ctime_tm|dateformat('%Y/%m/%d %H:%M')}} at {{a.fullname}}

Last modified on {{a.mtime_tm|dateformat('%Y/%m/%d %H:%M')}}

{{a}}

    {% endfor %}

"""

default_404_template = """

        <title>Not found</title>

Not found

"""

class YawtLoader(BaseLoader):
    def __init__(self, root):
        self.root = root

    def get_source(self, environment, template):
        (current_category, template_base) = os.path.split(template)
        template_name = template

        while current_category and not self._exists(template_name):
            current_category = os.path.dirname(current_category)
            template_name = _join(current_category, template_base)

        if not self._exists(template_name):
            raise TemplateNotFound(template)

        template_file = self._file(template_name)
        mtime = os.path.getmtime(template_file)
        with file(template_file) as f:
            source = f.read().decode('utf-8')
        return source, template_file, lambda: mtime == os.path.getmtime(template_file)

    def _file(self, template_name):
        return os.path.join(self.root, template_name)

    def _exists(self, template_name):
        return os.path.exists(self._file(template_name))

def _render(template_base, flavour=None, template_vars={}, content_type=None, category=""):
    template_file = _join(category, template_base)
    if flavour is None:
        flavour = 'html'
    template_file = template_file + "." + flavour

    if (content_type is not None):
        response = make_response(render_template(template_file, **template_vars))
        response.headers['Content-Type'] = content_type
        return response
    else:
        return render_template(template_file, **template_vars)

def _join(category, slug):
    if category:
        return category + '/' + slug
    else:
        return slug

def _breadcrumbs(pathstr):
    breadcrumbs = []
    pathurl = ''
    for piece in pathstr.split('/'):
        pathurl += '/' + piece
        breadcrumbs.append({'crumb': piece, 'url': pathurl})
    return breadcrumbs

class PagingInfo(object):
    def __init__(self, page, page_size, total_count, base_url):
        self.page_size = page_size
        self.page = page
        self.base_url = base_url
        self.start = (self.page - 1) * self.page_size
        self.end = self.start + self.page_size

        self.total_count = total_count
        self.total_pages = self.total_count / self.page_size
        if self.total_count % self.page_size is not 0:
            self.total_pages = self.total_pages + 1

        self.has_next = self.page < self.total_pages
        self.has_prev = self.page > 1

    def url(self, page):
        return self.base_url + "?page=" + str(page)

    @property
    def next_url(self):
        if not self.has_next:
            return None
        return self.url(self.page + 1)

    @property
    def prev_url(self):
        if not self.has_prev:
            return None
        return self.url(self.page - 1)

    def __eq__(self, other):
        return self.page == other.page and \
               self.page_size == other.page_size and \
               self.total_count == other.total_count and \
               self.base_url == other.base_url

class YawtView(object):
    """
    Collection of utility methods for rendering YAWT views
    """
    def __init__(self, plugins, content_types):
        self._plugins = plugins
        self._content_types = content_types

    def render_missing_resource(self):
        return render_template("404.html")

    def render_article(self, flavour, article, breadcrumbs=None):
        template_vars = {'article': article,
                         'breadcrumbs': breadcrumbs}
        template_vars = self._plugins.template_vars(template_vars)
        return _render('article', flavour, template_vars, self._content_type(flavour),
                       article.category)

    def render_collection(self, flavour, articles, title, page_info, category="", breadcrumbs=None):
        template_vars = {'articles': articles[page_info.start:page_info.end],
                         'total_pages': page_info.total_pages,
                         'page': page_info.page,
                         'nextpage':  page_info.next_url,
                         'prevpage': page_info.prev_url,
                         'collection_title': title,
                         'breadcrumbs': breadcrumbs}
        template_vars = self._plugins.template_vars(template_vars)
        return _render('article_list', flavour, template_vars,
                       self._content_type(flavour), category)

    def _content_type(self, flavour):
        return self._content_types.get(flavour, None)

class ArticleView(object):
    """
    Class for rendering YAWT article views, which are views meant
    to display just one entry on your blog or site.
    """
    def __init__(self, store, yawtview):
        self._yawtview = yawtview
        self._store = store

    def dispatch_request(self, flavour, category, slug):
        article = self._store.fetch_article_by_category_slug(category, slug)
        if article is None:
            # no article by that name, but there might be a category
            fullname = _join(category, slug)
            if self._store.category_exists(fullname):
                # Normally flask handles this, but I don't think it can in this case
                # Basically, we have trouble figuring out if cooking/indian/madras
                # is an article or a category
                return redirect(url_for('category_canonical', category=fullname))
            else:
                return self._yawtview.render_missing_resource()
        else:
            return self._yawtview.render_article(flavour, article,
                                                 _breadcrumbs(article.fullname))

def create_article_view():
    return ArticleView(g.store, YawtView(g.plugins, g.config['content_types']))

class CategoryView(object):
    """
    Class for rendering YAWT category view, which are views meant to display
    categories, which usually have a reverse time list of articles.
    """
    def __init__(self, store, yawtview):
        self._yawtview = yawtview
        self._store = store

    # Could render a category article, or if not then
    # defaults to a list of posts
    def dispatch_request(self, flavour, category,
                         page=1, page_size=10, base_url='http://localhost'):
        category_article = _join(category, 'index')
        if self._store.article_exists(category_article):
            article = self._store.fetch_article_by_fullname(category_article)
            # index file exists
            return self._yawtview.render_article(flavour, article)
        else:
            # no index file.  Render an article list template with
            # all the articles in the category sent to the template
            articles = self._store.fetch_articles_by_category(category)
            if len(articles) < 1:
                return self._yawtview.render_missing_resource()
            else:
                page_info = PagingInfo(page, page_size, len(articles), base_url)
                return self._render_collection(flavour, articles, category, page_info)

    def _render_collection(self, flavour, articles, category, page_info):
        title = self._category_title(category)
        return self._yawtview.render_collection(flavour, articles, '',
                                                page_info, category,
                                                _breadcrumbs(category))

    def _category_title(self, category):
        if category is None or len(category) == 0:
            return ''
        return ' :: '.join(category.split('/'))

def create_category_view():
    return CategoryView(g.store, YawtView(g.plugins, g.config['content_types']))
<|endoftext|>