<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2016, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

# this is a windows documentation stub.  actual code lives in the .ps1
# file of the same name

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = r'''
---
module: win_find
version_added: "2.3"
short_description: Return a list of files based on specific criteria
description:
    - Return a list of files based on specified criteria.
    - Multiple criteria are AND'd together.
    - For non-Windows targets, use the M(find) module instead.
options:
    age:
        description:
            - Select files or folders whose age is equal to or greater than
              the specified time. Use a negative age to find files equal to or
              less than the specified time. You can choose seconds, minutes,
              hours, days or weeks by specifying the first letter of an of
              those words (e.g., "2s", "10d", 1w").
    age_stamp:
        description:
            - Choose the file property against which we compare C(age). The
              default attribute we compare with is the last modification time.
        choices: [ atime, ctime, mtime ]
        default: mtime
    checksum_algorithm:
        description:
            - Algorithm to determine the checksum of a file. Will throw an error
              if the host is unable to use specified algorithm.
        choices: [ md5, sha1, sha256, sha384, sha512 ]
        default: sha1
    file_type:
        description: Type of file to search for.
        choices: [ directory, file ]
        default: file
    follow:
        description:
            - Set this to C(yes) to follow symlinks in the path.
            - This needs to be used in conjunction with C(recurse).
        type: bool
        default: 'no'
    get_checksum:
        description:
            - Whether to return a checksum of the file in the return info (default sha1),
              use C(checksum_algorithm) to change from the default.
        type: bool
        default: 'yes'
    hidden:
        description: Set this to include hidden files or folders.
        type: bool
        default: 'no'
    paths:
        description:
            - List of paths of directories to search for files or folders in.
              This can be supplied as a single path or a list of paths.
        required: yes
    patterns:
        description:
            - One or more (powershell or regex) patterns to compare filenames
              with. The type of pattern matching is controlled by C(use_regex)
              option. The patterns retrict the list of files or folders to be
              returned based on the filenames. For a file to be matched it
              only has to match with one pattern in a list provided.
    recurse:
        description:
            - Will recursively descend into the directory looking for files
              or folders.
        type: bool
        default: 'no'
    size:
        description:
            - Select files or folders whose size is equal to or greater than
              the specified size. Use a negative value to find files equal to
              or less than the specified size. You can specify the size with
              a suffix of the byte type i.e. kilo = k, mega = m... Size is not
              evaluated for symbolic links.
    use_regex:
        description:
            - Will set patterns to run as a regex check if set to C(yes).
        type: bool
        default: 'no'
author:
- Jordan Borean (@jborean93)
'''

EXAMPLES = r'''
- name: Find files in path
  win_find:
    paths: D:\Temp

- name: Find hidden files in path
  win_find:
    paths: D:\Temp
    hidden: yes

- name: Find files in multiple paths
  win_find:
    paths:
    - C:\Temp
    - D:\Temp

- name: Find files in directory while searching recursively
  win_find:
    paths: D:\Temp
    recurse: yes

- name: Find files in directory while following symlinks
  win_find:
    paths: D:\Temp
    recurse: yes
    follow: yes

- name: Find files with .log and .out extension using powershell wildcards
  win_find:
    paths: D:\Temp
    patterns: [ '*.log', '*.out' ]

- name: Find files in path based on regex pattern
  win_find:
    paths: D:\Temp
    patterns: out_\d{8}-\d{6}.log

- name: Find files older than 1 day
  win_find:
    paths: D:\Temp
    age: 86400

- name: Find files older than 1 day based on create time
  win_find:
    paths: D:\Temp
    age: 86400
    age_stamp: ctime

- name: Find files older than 1 day with unit syntax
  win_find:
    paths: D:\Temp
    age: 1d

- name: Find files newer than 1 hour
  win_find:
    paths: D:\Temp
    age: -3600

- name: Find files newer than 1 hour with unit syntax
  win_find:
    paths: D:\Temp
    age: -1h

- name: Find files larger than 1MB
  win_find:
    paths: D:\Temp
    size: 1048576

- name: Find files larger than 1GB with unit syntax
  win_find:
    paths: D:\Temp
    size: 1g

- name: Find files smaller than 1MB
  win_find:
    paths: D:\Temp
    size: -1048576

- name: Find files smaller than 1GB with unit syntax
  win_find:
    paths: D:\Temp
    size: -1g

- name: Find folders/symlinks in multiple paths
  win_find:
    paths:
    - C:\Temp
    - D:\Temp
    file_type: directory

- name: Find files and return SHA256 checksum of files found
  win_find:
    paths: C:\Temp
    get_checksum: yes
    checksum_algorithm: sha256

- name: Find files and do not return the checksum
  win_find:
    path: C:\Temp
    get_checksum: no
'''

RETURN = r'''
examined:
    description: The number of files/folders that was checked
    returned: always
    type: int
    sample: 10
matched:
    description: The number of files/folders that match the criteria
    returned: always
    type: int
    sample: 2
files:
    description: Information on the files/folders that match the criteria returned as a list of dictionary elements for each file matched
    returned: success
    type: complex
    contains:
        attributes:
            description: attributes of the file at path in raw form
            returned: success, path exists
            type: string
            sample: "Archive, Hidden"
        checksum:
            description: The checksum of a file based on checksum_algorithm specified
            returned: success, path exists, path is a file, get_checksum == True
            type: string
            sample: 09cb79e8fc7453c84a07f644e441fd81623b7f98
        creationtime:
            description: the create time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        extension:
            description: the extension of the file at path
            returned: success, path exists, path is a file
            type: string
            sample: ".ps1"
        isarchive:
            description: if the path is ready for archiving or not
            returned: success, path exists
            type: boolean
            sample: True
        isdir:
            description: if the path is a directory or not
            returned: success, path exists
            type: boolean
            sample: True
        ishidden:
            description: if the path is hidden or not
            returned: success, path exists
            type: boolean
            sample: True
        islnk:
            description: if the path is a symbolic link or junction or not
            returned: success, path exists
            type: boolean
            sample: True
        isreadonly:
            description: if the path is read only or not
            returned: success, path exists
            type: boolean
            sample: True
        isshared:
            description: if the path is shared or not
            returned: success, path exists
            type: boolean
            sample: True
        lastaccesstime:
            description: the last access time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        lastwritetime:
            description: the last modification time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        lnk_source:
            description: the target of the symbolic link, will return null if not a link or the link is broken
            return: success, path exists, path is a symbolic link
            type: string
            sample: C:\temp
        owner:
            description: the owner of the file
            returned: success, path exists
            type: string
            sample: BUILTIN\Administrators
        path:
            description: the full absolute path to the file
            returned: success, path exists
            type: string
            sample: BUILTIN\Administrators
        sharename:
            description: the name of share if folder is shared
            returned: success, path exists, path is a directory and isshared == True
            type: string
            sample: file-share
        size:
            description: the size in bytes of a file or folder
            returned: success, path exists, path is not a link
            type: int
            sample: 1024
'''
<commit_msg>corrected typo in documentation examples (#36894)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# Copyright: (c) 2016, Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

# this is a windows documentation stub.  actual code lives in the .ps1
# file of the same name

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = r'''
---
module: win_find
version_added: "2.3"
short_description: Return a list of files based on specific criteria
description:
    - Return a list of files based on specified criteria.
    - Multiple criteria are AND'd together.
    - For non-Windows targets, use the M(find) module instead.
options:
    age:
        description:
            - Select files or folders whose age is equal to or greater than
              the specified time. Use a negative age to find files equal to or
              less than the specified time. You can choose seconds, minutes,
              hours, days or weeks by specifying the first letter of an of
              those words (e.g., "2s", "10d", 1w").
    age_stamp:
        description:
            - Choose the file property against which we compare C(age). The
              default attribute we compare with is the last modification time.
        choices: [ atime, ctime, mtime ]
        default: mtime
    checksum_algorithm:
        description:
            - Algorithm to determine the checksum of a file. Will throw an error
              if the host is unable to use specified algorithm.
        choices: [ md5, sha1, sha256, sha384, sha512 ]
        default: sha1
    file_type:
        description: Type of file to search for.
        choices: [ directory, file ]
        default: file
    follow:
        description:
            - Set this to C(yes) to follow symlinks in the path.
            - This needs to be used in conjunction with C(recurse).
        type: bool
        default: 'no'
    get_checksum:
        description:
            - Whether to return a checksum of the file in the return info (default sha1),
              use C(checksum_algorithm) to change from the default.
        type: bool
        default: 'yes'
    hidden:
        description: Set this to include hidden files or folders.
        type: bool
        default: 'no'
    paths:
        description:
            - List of paths of directories to search for files or folders in.
              This can be supplied as a single path or a list of paths.
        required: yes
    patterns:
        description:
            - One or more (powershell or regex) patterns to compare filenames
              with. The type of pattern matching is controlled by C(use_regex)
              option. The patterns retrict the list of files or folders to be
              returned based on the filenames. For a file to be matched it
              only has to match with one pattern in a list provided.
    recurse:
        description:
            - Will recursively descend into the directory looking for files
              or folders.
        type: bool
        default: 'no'
    size:
        description:
            - Select files or folders whose size is equal to or greater than
              the specified size. Use a negative value to find files equal to
              or less than the specified size. You can specify the size with
              a suffix of the byte type i.e. kilo = k, mega = m... Size is not
              evaluated for symbolic links.
    use_regex:
        description:
            - Will set patterns to run as a regex check if set to C(yes).
        type: bool
        default: 'no'
author:
- Jordan Borean (@jborean93)
'''

EXAMPLES = r'''
- name: Find files in path
  win_find:
    paths: D:\Temp

- name: Find hidden files in path
  win_find:
    paths: D:\Temp
    hidden: yes

- name: Find files in multiple paths
  win_find:
    paths:
    - C:\Temp
    - D:\Temp

- name: Find files in directory while searching recursively
  win_find:
    paths: D:\Temp
    recurse: yes

- name: Find files in directory while following symlinks
  win_find:
    paths: D:\Temp
    recurse: yes
    follow: yes

- name: Find files with .log and .out extension using powershell wildcards
  win_find:
    paths: D:\Temp
    patterns: [ '*.log', '*.out' ]

- name: Find files in path based on regex pattern
  win_find:
    paths: D:\Temp
    patterns: out_\d{8}-\d{6}.log

- name: Find files older than 1 day
  win_find:
    paths: D:\Temp
    age: 86400

- name: Find files older than 1 day based on create time
  win_find:
    paths: D:\Temp
    age: 86400
    age_stamp: ctime

- name: Find files older than 1 day with unit syntax
  win_find:
    paths: D:\Temp
    age: 1d

- name: Find files newer than 1 hour
  win_find:
    paths: D:\Temp
    age: -3600

- name: Find files newer than 1 hour with unit syntax
  win_find:
    paths: D:\Temp
    age: -1h

- name: Find files larger than 1MB
  win_find:
    paths: D:\Temp
    size: 1048576

- name: Find files larger than 1GB with unit syntax
  win_find:
    paths: D:\Temp
    size: 1g

- name: Find files smaller than 1MB
  win_find:
    paths: D:\Temp
    size: -1048576

- name: Find files smaller than 1GB with unit syntax
  win_find:
    paths: D:\Temp
    size: -1g

- name: Find folders/symlinks in multiple paths
  win_find:
    paths:
    - C:\Temp
    - D:\Temp
    file_type: directory

- name: Find files and return SHA256 checksum of files found
  win_find:
    paths: C:\Temp
    get_checksum: yes
    checksum_algorithm: sha256

- name: Find files and do not return the checksum
  win_find:
    paths: C:\Temp
    get_checksum: no
'''

RETURN = r'''
examined:
    description: The number of files/folders that was checked
    returned: always
    type: int
    sample: 10
matched:
    description: The number of files/folders that match the criteria
    returned: always
    type: int
    sample: 2
files:
    description: Information on the files/folders that match the criteria returned as a list of dictionary elements for each file matched
    returned: success
    type: complex
    contains:
        attributes:
            description: attributes of the file at path in raw form
            returned: success, path exists
            type: string
            sample: "Archive, Hidden"
        checksum:
            description: The checksum of a file based on checksum_algorithm specified
            returned: success, path exists, path is a file, get_checksum == True
            type: string
            sample: 09cb79e8fc7453c84a07f644e441fd81623b7f98
        creationtime:
            description: the create time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        extension:
            description: the extension of the file at path
            returned: success, path exists, path is a file
            type: string
            sample: ".ps1"
        isarchive:
            description: if the path is ready for archiving or not
            returned: success, path exists
            type: boolean
            sample: True
        isdir:
            description: if the path is a directory or not
            returned: success, path exists
            type: boolean
            sample: True
        ishidden:
            description: if the path is hidden or not
            returned: success, path exists
            type: boolean
            sample: True
        islnk:
            description: if the path is a symbolic link or junction or not
            returned: success, path exists
            type: boolean
            sample: True
        isreadonly:
            description: if the path is read only or not
            returned: success, path exists
            type: boolean
            sample: True
        isshared:
            description: if the path is shared or not
            returned: success, path exists
            type: boolean
            sample: True
        lastaccesstime:
            description: the last access time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        lastwritetime:
            description: the last modification time of the file represented in seconds since epoch
            returned: success, path exists
            type: float
            sample: 1477984205.15
        lnk_source:
            description: the target of the symbolic link, will return null if not a link or the link is broken
            return: success, path exists, path is a symbolic link
            type: string
            sample: C:\temp
        owner:
            description: the owner of the file
            returned: success, path exists
            type: string
            sample: BUILTIN\Administrators
        path:
            description: the full absolute path to the file
            returned: success, path exists
            type: string
            sample: BUILTIN\Administrators
        sharename:
            description: the name of share if folder is shared
            returned: success, path exists, path is a directory and isshared == True
            type: string
            sample: file-share
        size:
            description: the size in bytes of a file or folder
            returned: success, path exists, path is not a link
            type: int
            sample: 1024
'''
<|endoftext|>


<commit_before># Copyright (c) 2018 Remy Leone
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)

__metaclass__ = type

DOCUMENTATION = '''
    name: netbox
    plugin_type: inventory
    author:
        - Remy Leone (@sieben)
        - Anthony Ruhier (@Anthony25)
    short_description: NetBox inventory source
    description:
        - Get inventory hosts from NetBox
    extends_documentation_fragment:
        - constructed
    options:
        plugin:
            description: token that ensures this is a source file for the 'netbox' plugin.
            required: True
            choices: ['netbox']
        api_endpoint:
            description: Endpoint of the NetBox API
            required: True
            env:
                - name: NETBOX_API
        validate_certs:
            description:
                - Allows connection when SSL certificates are not valid. Set to C(false) when certificates are not trusted.
            default: True
            type: boolean
        token:
            required: True
            description: NetBox token.
            env:
                # in order of precedence
                - name: NETBOX_TOKEN
                - name: NETBOX_API_KEY
        group_by:
            description: Keys used to create groups.
            type: list
            choices:
                - sites
                - tenants
                - racks
                - tags
                - device_roles
                - device_types
                - manufacturers
                - platforms
            default: []
        query_filters:
            description: List of parameters passed to the query string (Multiple values may be separated by commas)
            type: list
            default: []
        timeout:
            description: Timeout for Netbox requests in seconds
            type: int
            default: 60
        compose:
            description: List of custom ansible host vars to create from the device object fetched from NetBox
            default: {}
            type: dict
'''

EXAMPLES = '''
# netbox_inventory.yml file in YAML format
# Example command line: ansible-inventory -v --list -i netbox_inventory.yml

plugin: netbox
api_endpoint: http://localhost:8000
validate_certs: True
group_by:
  - device_roles
query_filters:
  - role: network-edge-router

# Query filters are passed directly as an argument to the fetching queries.
# You can repeat tags in the query string.

query_filters:
  - role: server
  - tag: web
  - tag: production

# See the NetBox documentation at https://netbox.readthedocs.io/en/latest/api/overview/
# the query_filters work as a logical **OR**
#
# Prefix any custom fields with cf_ and pass the field value with the regular NetBox query string

query_filters:
  - cf_foo: bar

# NetBox inventory plugin also supports Constructable semantics
# You can fill your hosts vars using the compose option:

plugin: netbox
compose:
  foo: last_updated
  bar: display_name
  nested_variable: rack.display_name
'''

import json
import uuid
from sys import version as python_version
from threading import Thread
from itertools import chain

from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
from ansible.module_utils.ansible_release import __version__ as ansible_version
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_text
from ansible.module_utils.urls import open_url
from ansible.module_utils.six.moves.urllib.parse import urljoin, urlencode
from ansible.module_utils.compat.ipaddress import ip_interface

ALLOWED_DEVICE_QUERY_PARAMETERS = (
    "asset_tag",
    "cluster_id",
    "device_type_id",
    "has_primary_ip",
    "is_console_server",
    "is_full_depth",
    "is_network_device",
    "is_pdu",
    "mac_address",
    "manufacturer",
    "manufacturer_id",
    "model",
    "name",
    "platform",
    "platform_id",
    "position",
    "rack_group_id",
    "rack_id",
    "role",
    "role_id",
    "serial",
    "site",
    "site_id",
    "status",
    "tag",
    "tenant",
    "tenant_id",
    "virtual_chassis_id",
)

class InventoryModule(BaseInventoryPlugin, Constructable):
    NAME = 'netbox'

    def _fetch_information(self, url):
        response = open_url(url, headers=self.headers, timeout=self.timeout, validate_certs=self.validate_certs)

        try:
            raw_data = to_text(response.read(), errors='surrogate_or_strict')
        except UnicodeError:
            raise AnsibleError("Incorrect encoding of fetched payload from NetBox API.")

        try:
            return json.loads(raw_data)
        except ValueError:
            raise AnsibleError("Incorrect JSON payload: %s" % raw_data)

    def get_resource_list(self, api_url):
        """Retrieves resource list from netbox API.
         Returns:
            A list of all resource from netbox API.
        """
        if not api_url:
            raise AnsibleError("Please check API URL in script configuration file.")

        hosts_list = []
        # Pagination.
        while api_url:
            self.display.v("Fetching: " + api_url)
            # Get hosts list.
            api_output = self._fetch_information(api_url)
            hosts_list += api_output["results"]
            api_url = api_output["next"]

        # Get hosts list.
        return hosts_list

    @property
    def group_extractors(self):
        return {
            "sites": self.extract_site,
            "tenants": self.extract_tenant,
            "racks": self.extract_rack,
            "tags": self.extract_tags,
            "disk": self.extract_disk,
            "memory": self.extract_memory,
            "vcpus": self.extract_vcpus,
            "device_roles": self.extract_device_role,
            "platforms": self.extract_platform,
            "device_types": self.extract_device_type,
            "manufacturers": self.extract_manufacturer
        }

    def extract_disk(self, host):
        return host.get("disk")

    def extract_vcpus(self, host):
        return host.get("vcpus")

    def extract_memory(self, host):
        return host.get("memory")

    def extract_platform(self, host):
        try:
            return self.platforms_lookup[host["platform"]["id"]]
        except Exception:
            return

    def extract_device_type(self, host):
        try:
            return [self.device_types_lookup[host["device_type"]["id"]]]
        except Exception:
            return

    def extract_rack(self, host):
        try:
            return [self.racks_lookup[host["rack"]["id"]]]
        except Exception:
            return

    def extract_site(self, host):
        try:
            return [self.sites_lookup[host["site"]["id"]]]
        except Exception:
            return

    def extract_tenant(self, host):
        try:
            return [self.tenants_lookup[host["tenant"]["id"]]]
        except Exception:
            return

    def extract_device_role(self, host):
        try:
            return [self.device_roles_lookup[host["device_role"]["id"]]]
        except Exception:
            return

    def extract_manufacturer(self, host):
        try:
            return [self.manufacturers_lookup[host["device_type"]["manufacturer"]["id"]]]
        except Exception:
            return

    def extract_primary_ip(self, host):
        try:
            address = host["primary_ip"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_primary_ip4(self, host):
        try:
            address = host["primary_ip4"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_primary_ip6(self, host):
        try:
            address = host["primary_ip6"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_tags(self, host):
        return host["tags"]

    def refresh_platforms_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/platforms/?limit=0")
        platforms = self.get_resource_list(api_url=url)
        self.platforms_lookup = dict((platform["id"], platform["name"]) for platform in platforms)

    def refresh_sites_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/sites/?limit=0")
        sites = self.get_resource_list(api_url=url)
        self.sites_lookup = dict((site["id"], site["name"]) for site in sites)

    def refresh_regions_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/regions/?limit=0")
        regions = self.get_resource_list(api_url=url)
        self.regions_lookup = dict((region["id"], region["name"]) for region in regions)

    def refresh_tenants_lookup(self):
        url = urljoin(self.api_endpoint, "/api/tenancy/tenants/?limit=0")
        tenants = self.get_resource_list(api_url=url)
        self.tenants_lookup = dict((tenant["id"], tenant["name"]) for tenant in tenants)

    def refresh_racks_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/racks/?limit=0")
        racks = self.get_resource_list(api_url=url)
        self.racks_lookup = dict((rack["id"], rack["name"]) for rack in racks)

    def refresh_device_roles_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/device-roles/?limit=0")
        device_roles = self.get_resource_list(api_url=url)
        self.device_roles_lookup = dict((device_role["id"], device_role["name"]) for device_role in device_roles)

    def refresh_device_types_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/device-types/?limit=0")
        device_types = self.get_resource_list(api_url=url)
        self.device_types_lookup = dict((device_type["id"], device_type["model"]) for device_type in device_types)

    def refresh_manufacturers_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/manufacturers/?limit=0")
        manufacturers = self.get_resource_list(api_url=url)
        self.manufacturers_lookup = dict((manufacturer["id"], manufacturer["name"]) for manufacturer in manufacturers)

    def refresh_lookups(self):
        lookup_processes = (
            self.refresh_sites_lookup,
            self.refresh_regions_lookup,
            self.refresh_tenants_lookup,
            self.refresh_racks_lookup,
            self.refresh_device_roles_lookup,
            self.refresh_device_types_lookup,
            self.refresh_manufacturers_lookup,
        )

        thread_list = []
        for p in lookup_processes:
            t = Thread(target=p)
            thread_list.append(t)
            t.start()

        for thread in thread_list:
            thread.join()

    def validate_query_parameters(self, x):
        if not (isinstance(x, dict) and len(x) == 1):
            self.display.warning("Warning query parameters %s not a dict with a single key." % x)
            return

        k = tuple(x.keys())[0]
        v = tuple(x.values())[0]

        if not (k in ALLOWED_DEVICE_QUERY_PARAMETERS or k.startswith("cf_")):
            msg = "Warning: %s not in %s or starting with cf (Custom field)" % (k, ALLOWED_DEVICE_QUERY_PARAMETERS)
            self.display.warning(msg=msg)
            return
        return k, v

    def refresh_url(self):
        query_parameters = [("limit", 0)]
        if self.query_filters:
            query_parameters.extend(filter(lambda x: x,
                                           map(self.validate_query_parameters, self.query_filters)))
        self.device_url = self.api_endpoint + "/api/dcim/devices/" + "?" + urlencode(query_parameters)
        self.virtual_machines_url = "".join([self.api_endpoint,
                                             "/api/virtualization/virtual-machines/",
                                             "?",
                                             urlencode(query_parameters)])

    def fetch_hosts(self):
        return chain(
            self.get_resource_list(self.device_url),
            self.get_resource_list(self.virtual_machines_url),
        )

    def extract_name(self, host):
        # An host in an Ansible inventory requires an hostname.
        # name is an unique but not required attribute for a device in NetBox
        # We default to an UUID for hostname in case the name is not set in NetBox
        return host["name"] or str(uuid.uuid4())

    def add_host_to_groups(self, host, hostname):
        for group in self.group_by:
            sub_groups = self.group_extractors[group](host)

            if not sub_groups:
                continue

            for sub_group in sub_groups:
                group_name = "_".join([group, sub_group])
                self.inventory.add_group(group=group_name)
                self.inventory.add_host(group=group_name, host=hostname)

    def _fill_host_variables(self, host, hostname):
        for attribute, extractor in self.group_extractors.items():
            if not extractor(host):
                continue
            self.inventory.set_variable(hostname, attribute, extractor(host))

        if self.extract_primary_ip(host):
            self.inventory.set_variable(hostname, "ansible_host", self.extract_primary_ip(host=host))

        if self.extract_primary_ip4(host):
            self.inventory.set_variable(hostname, "primary_ip4", self.extract_primary_ip4(host=host))

        if self.extract_primary_ip6(host):
            self.inventory.set_variable(hostname, "primary_ip6", self.extract_primary_ip6(host=host))

    def main(self):
        self.refresh_lookups()
        self.refresh_url()
        hosts_list = self.fetch_hosts()

        for host in hosts_list:
            hostname = self.extract_name(host=host)
            self.inventory.add_host(host=hostname)
            self._fill_host_variables(host=host, hostname=hostname)

            strict = self.get_option("strict")

            # Composed variables
            self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict)
            # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group
            self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict)

            # Create groups based on variable values and add the corresponding hosts to it
            self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname, strict=strict)
            self.add_host_to_groups(host=host, hostname=hostname)

    def parse(self, inventory, loader, path, cache=True):
        super(InventoryModule, self).parse(inventory, loader, path)
        self._read_config_data(path=path)

        # Netbox access
        token = self.get_option("token")
        self.api_endpoint = self.get_option("api_endpoint")
        self.timeout = self.get_option("timeout")
        self.validate_certs = self.get_option("validate_certs")
        self.headers = {
            'Authorization': "Token %s" % token,
            'User-Agent': "ansible %s Python %s" % (ansible_version, python_version.split(' ')[0]),
            'Content-type': 'application/json'
        }

        # Filter and group_by options
        self.group_by = self.get_option("group_by")
        self.query_filters = self.get_option("query_filters")
        self.main()
<commit_msg>Fix netbox url with urljoin (#47655)<commit_after># Copyright (c) 2018 Remy Leone
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)

__metaclass__ = type

DOCUMENTATION = '''
    name: netbox
    plugin_type: inventory
    author:
        - Remy Leone (@sieben)
        - Anthony Ruhier (@Anthony25)
    short_description: NetBox inventory source
    description:
        - Get inventory hosts from NetBox
    extends_documentation_fragment:
        - constructed
    options:
        plugin:
            description: token that ensures this is a source file for the 'netbox' plugin.
            required: True
            choices: ['netbox']
        api_endpoint:
            description: Endpoint of the NetBox API
            required: True
            env:
                - name: NETBOX_API
        validate_certs:
            description:
                - Allows connection when SSL certificates are not valid. Set to C(false) when certificates are not trusted.
            default: True
            type: boolean
        token:
            required: True
            description: NetBox token.
            env:
                # in order of precedence
                - name: NETBOX_TOKEN
                - name: NETBOX_API_KEY
        group_by:
            description: Keys used to create groups.
            type: list
            choices:
                - sites
                - tenants
                - racks
                - tags
                - device_roles
                - device_types
                - manufacturers
                - platforms
            default: []
        query_filters:
            description: List of parameters passed to the query string (Multiple values may be separated by commas)
            type: list
            default: []
        timeout:
            description: Timeout for Netbox requests in seconds
            type: int
            default: 60
        compose:
            description: List of custom ansible host vars to create from the device object fetched from NetBox
            default: {}
            type: dict
'''

EXAMPLES = '''
# netbox_inventory.yml file in YAML format
# Example command line: ansible-inventory -v --list -i netbox_inventory.yml

plugin: netbox
api_endpoint: http://localhost:8000
validate_certs: True
group_by:
  - device_roles
query_filters:
  - role: network-edge-router

# Query filters are passed directly as an argument to the fetching queries.
# You can repeat tags in the query string.

query_filters:
  - role: server
  - tag: web
  - tag: production

# See the NetBox documentation at https://netbox.readthedocs.io/en/latest/api/overview/
# the query_filters work as a logical **OR**
#
# Prefix any custom fields with cf_ and pass the field value with the regular NetBox query string

query_filters:
  - cf_foo: bar

# NetBox inventory plugin also supports Constructable semantics
# You can fill your hosts vars using the compose option:

plugin: netbox
compose:
  foo: last_updated
  bar: display_name
  nested_variable: rack.display_name
'''

import json
import uuid
from sys import version as python_version
from threading import Thread
from itertools import chain

from ansible.plugins.inventory import BaseInventoryPlugin, Constructable
from ansible.module_utils.ansible_release import __version__ as ansible_version
from ansible.errors import AnsibleError
from ansible.module_utils._text import to_text
from ansible.module_utils.urls import open_url
from ansible.module_utils.six.moves.urllib.parse import urljoin, urlencode
from ansible.module_utils.compat.ipaddress import ip_interface

ALLOWED_DEVICE_QUERY_PARAMETERS = (
    "asset_tag",
    "cluster_id",
    "device_type_id",
    "has_primary_ip",
    "is_console_server",
    "is_full_depth",
    "is_network_device",
    "is_pdu",
    "mac_address",
    "manufacturer",
    "manufacturer_id",
    "model",
    "name",
    "platform",
    "platform_id",
    "position",
    "rack_group_id",
    "rack_id",
    "role",
    "role_id",
    "serial",
    "site",
    "site_id",
    "status",
    "tag",
    "tenant",
    "tenant_id",
    "virtual_chassis_id",
)

class InventoryModule(BaseInventoryPlugin, Constructable):
    NAME = 'netbox'

    def _fetch_information(self, url):
        response = open_url(url, headers=self.headers, timeout=self.timeout, validate_certs=self.validate_certs)

        try:
            raw_data = to_text(response.read(), errors='surrogate_or_strict')
        except UnicodeError:
            raise AnsibleError("Incorrect encoding of fetched payload from NetBox API.")

        try:
            return json.loads(raw_data)
        except ValueError:
            raise AnsibleError("Incorrect JSON payload: %s" % raw_data)

    def get_resource_list(self, api_url):
        """Retrieves resource list from netbox API.
         Returns:
            A list of all resource from netbox API.
        """
        if not api_url:
            raise AnsibleError("Please check API URL in script configuration file.")

        hosts_list = []
        # Pagination.
        while api_url:
            self.display.v("Fetching: " + api_url)
            # Get hosts list.
            api_output = self._fetch_information(api_url)
            hosts_list += api_output["results"]
            api_url = api_output["next"]

        # Get hosts list.
        return hosts_list

    @property
    def group_extractors(self):
        return {
            "sites": self.extract_site,
            "tenants": self.extract_tenant,
            "racks": self.extract_rack,
            "tags": self.extract_tags,
            "disk": self.extract_disk,
            "memory": self.extract_memory,
            "vcpus": self.extract_vcpus,
            "device_roles": self.extract_device_role,
            "platforms": self.extract_platform,
            "device_types": self.extract_device_type,
            "manufacturers": self.extract_manufacturer
        }

    def extract_disk(self, host):
        return host.get("disk")

    def extract_vcpus(self, host):
        return host.get("vcpus")

    def extract_memory(self, host):
        return host.get("memory")

    def extract_platform(self, host):
        try:
            return self.platforms_lookup[host["platform"]["id"]]
        except Exception:
            return

    def extract_device_type(self, host):
        try:
            return [self.device_types_lookup[host["device_type"]["id"]]]
        except Exception:
            return

    def extract_rack(self, host):
        try:
            return [self.racks_lookup[host["rack"]["id"]]]
        except Exception:
            return

    def extract_site(self, host):
        try:
            return [self.sites_lookup[host["site"]["id"]]]
        except Exception:
            return

    def extract_tenant(self, host):
        try:
            return [self.tenants_lookup[host["tenant"]["id"]]]
        except Exception:
            return

    def extract_device_role(self, host):
        try:
            return [self.device_roles_lookup[host["device_role"]["id"]]]
        except Exception:
            return

    def extract_manufacturer(self, host):
        try:
            return [self.manufacturers_lookup[host["device_type"]["manufacturer"]["id"]]]
        except Exception:
            return

    def extract_primary_ip(self, host):
        try:
            address = host["primary_ip"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_primary_ip4(self, host):
        try:
            address = host["primary_ip4"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_primary_ip6(self, host):
        try:
            address = host["primary_ip6"]["address"]
            return str(ip_interface(address).ip)
        except Exception:
            return

    def extract_tags(self, host):
        return host["tags"]

    def refresh_platforms_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/platforms/?limit=0")
        platforms = self.get_resource_list(api_url=url)
        self.platforms_lookup = dict((platform["id"], platform["name"]) for platform in platforms)

    def refresh_sites_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/sites/?limit=0")
        sites = self.get_resource_list(api_url=url)
        self.sites_lookup = dict((site["id"], site["name"]) for site in sites)

    def refresh_regions_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/regions/?limit=0")
        regions = self.get_resource_list(api_url=url)
        self.regions_lookup = dict((region["id"], region["name"]) for region in regions)

    def refresh_tenants_lookup(self):
        url = urljoin(self.api_endpoint, "/api/tenancy/tenants/?limit=0")
        tenants = self.get_resource_list(api_url=url)
        self.tenants_lookup = dict((tenant["id"], tenant["name"]) for tenant in tenants)

    def refresh_racks_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/racks/?limit=0")
        racks = self.get_resource_list(api_url=url)
        self.racks_lookup = dict((rack["id"], rack["name"]) for rack in racks)

    def refresh_device_roles_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/device-roles/?limit=0")
        device_roles = self.get_resource_list(api_url=url)
        self.device_roles_lookup = dict((device_role["id"], device_role["name"]) for device_role in device_roles)

    def refresh_device_types_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/device-types/?limit=0")
        device_types = self.get_resource_list(api_url=url)
        self.device_types_lookup = dict((device_type["id"], device_type["model"]) for device_type in device_types)

    def refresh_manufacturers_lookup(self):
        url = urljoin(self.api_endpoint, "/api/dcim/manufacturers/?limit=0")
        manufacturers = self.get_resource_list(api_url=url)
        self.manufacturers_lookup = dict((manufacturer["id"], manufacturer["name"]) for manufacturer in manufacturers)

    def refresh_lookups(self):
        lookup_processes = (
            self.refresh_sites_lookup,
            self.refresh_regions_lookup,
            self.refresh_tenants_lookup,
            self.refresh_racks_lookup,
            self.refresh_device_roles_lookup,
            self.refresh_device_types_lookup,
            self.refresh_manufacturers_lookup,
        )

        thread_list = []
        for p in lookup_processes:
            t = Thread(target=p)
            thread_list.append(t)
            t.start()

        for thread in thread_list:
            thread.join()

    def validate_query_parameters(self, x):
        if not (isinstance(x, dict) and len(x) == 1):
            self.display.warning("Warning query parameters %s not a dict with a single key." % x)
            return

        k = tuple(x.keys())[0]
        v = tuple(x.values())[0]

        if not (k in ALLOWED_DEVICE_QUERY_PARAMETERS or k.startswith("cf_")):
            msg = "Warning: %s not in %s or starting with cf (Custom field)" % (k, ALLOWED_DEVICE_QUERY_PARAMETERS)
            self.display.warning(msg=msg)
            return
        return k, v

    def refresh_url(self):
        query_parameters = [("limit", 0)]
        if self.query_filters:
            query_parameters.extend(filter(lambda x: x,
                                           map(self.validate_query_parameters, self.query_filters)))
        self.device_url = urljoin(self.api_endpoint,
                                  "/api/dcim/devices/?" + urlencode(query_parameters))
        self.virtual_machines_url = urljoin(self.api_endpoint,
                                            "/api/virtualization/virtual-machines/?" + urlencode(query_parameters))

    def fetch_hosts(self):
        return chain(
            self.get_resource_list(self.device_url),
            self.get_resource_list(self.virtual_machines_url),
        )

    def extract_name(self, host):
        # An host in an Ansible inventory requires an hostname.
        # name is an unique but not required attribute for a device in NetBox
        # We default to an UUID for hostname in case the name is not set in NetBox
        return host["name"] or str(uuid.uuid4())

    def add_host_to_groups(self, host, hostname):
        for group in self.group_by:
            sub_groups = self.group_extractors[group](host)

            if not sub_groups:
                continue

            for sub_group in sub_groups:
                group_name = "_".join([group, sub_group])
                self.inventory.add_group(group=group_name)
                self.inventory.add_host(group=group_name, host=hostname)

    def _fill_host_variables(self, host, hostname):
        for attribute, extractor in self.group_extractors.items():
            if not extractor(host):
                continue
            self.inventory.set_variable(hostname, attribute, extractor(host))

        if self.extract_primary_ip(host):
            self.inventory.set_variable(hostname, "ansible_host", self.extract_primary_ip(host=host))

        if self.extract_primary_ip4(host):
            self.inventory.set_variable(hostname, "primary_ip4", self.extract_primary_ip4(host=host))

        if self.extract_primary_ip6(host):
            self.inventory.set_variable(hostname, "primary_ip6", self.extract_primary_ip6(host=host))

    def main(self):
        self.refresh_lookups()
        self.refresh_url()
        hosts_list = self.fetch_hosts()

        for host in hosts_list:
            hostname = self.extract_name(host=host)
            self.inventory.add_host(host=hostname)
            self._fill_host_variables(host=host, hostname=hostname)

            strict = self.get_option("strict")

            # Composed variables
            self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict)
            # Complex groups based on jinja2 conditionals, hosts that meet the conditional are added to group
            self._set_composite_vars(self.get_option('compose'), host, hostname, strict=strict)

            # Create groups based on variable values and add the corresponding hosts to it
            self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname, strict=strict)
            self.add_host_to_groups(host=host, hostname=hostname)

    def parse(self, inventory, loader, path, cache=True):
        super(InventoryModule, self).parse(inventory, loader, path)
        self._read_config_data(path=path)

        # Netbox access
        token = self.get_option("token")
        self.api_endpoint = self.get_option("api_endpoint")
        self.timeout = self.get_option("timeout")
        self.validate_certs = self.get_option("validate_certs")
        self.headers = {
            'Authorization': "Token %s" % token,
            'User-Agent': "ansible %s Python %s" % (ansible_version, python_version.split(' ')[0]),
            'Content-type': 'application/json'
        }

        # Filter and group_by options
        self.group_by = self.get_option("group_by")
        self.query_filters = self.get_option("query_filters")
        self.main()
<|endoftext|>


<commit_before>################################################################################
# Assessment tools
################################################################################
import dense.assessment
import dense.decomposition

import numpy as np
from scipy.sparse import issparse

def is_transition_matrix(T, tol=1e-15):
    """
    True if T is a transition matrix

    Parameters
    ----------
    T : numpy.ndarray, shape(d, d) or scipy.sparse matrix
        Matrix to check
    tol : float
        tolerance to check with

    Returns
    -------
    Truth value: bool
        True, if T is positive and normed
        False, otherwise

    """
    if issparse(T):
        return sparse.assessment._is_stochastic_matrix_impl(T, tol)
    elif isinstance(T, np.ndarray):
        return dense.assessment._is_stochastic_matrix_impl(T, tol)
    else:
        raise ValueError("unsupported matrix type")

# TODO: martin: Implement in Python directly
def is_rate_matrix(K, tol=1e-15):
    """
        True if K is a rate matrix
    """

# TODO: Implement in Python directly
def is_ergodic(T, tol=1e-15):
    """
        True if T is connected (irreducible) and aperiodic
    """

# TODO: martin: Implement in Python directly
def is_reversible(T, mu=None, tol=1e-15):
    """
        True if T is a transition matrix
        mu : tests with respect to this stationary distribution
    """

################################################################################
# Eigenvalues and eigenvectors
################################################################################

# TODO: ben: Implement in Python directly
def mu(T):
    r"""Compute stationary distribution of stochastic matrix T.

    The stationary distribution is the left eigenvector corresponding to the
    non-degenerate eigenvalue :math: `\lambda=1`.

    Input
    -----
    T : numpy array, shape(d,d) or scipy.sparse matrix
        Transition matrix (stochastic matrix).

    Returns
    -------
    mu : numpy array, shape(d,)
        Vector of stationary probabilities.

    """
    if issparse(T):
        raise TypeError("Not implemented for sparse matrices.")
    elif isinstance(T, np.ndarray):
        return dense.decomposition.mu(T)
    else:
        raise TypeError("T is not a numpy.ndarray or a scipy.sparse matrix.")

# TODO: Implement in Python directly
def mu_sensitivity(T):
    """
        compute the sensitivity matrix of the stationary distribution of T
    """

# TODO: ben: Implement in Python directly
def statdist(T):
    r"""Compute stationary distribution of stochastic matrix T.

    The stationary distribution is the left eigenvector corresponding to the
    non-degenerate eigenvalue :math: `\lambda=1`.

    Input
    -----
    T : numpy array, shape(d,d) or scipy.sparse matrix
        Transition matrix (stochastic matrix).

    Returns
    -------
    mu : numpy array, shape(d,)
        Vector of stationary probabilities.

    """
    statdist=mu(T)
    return statdist

# TODO: Implement in Python directly
def statdist_sensitivity(T):
    """
        compute the sensitivity matrix of the stationary distribution of T
    """

def eigenvalues(T, k=None):
    """
        computes the eigenvalues

        T : transition matrix
        k : int (optional) or tuple of ints
            Compute the first k eigenvalues of T.
    """
    eig = np.sort(np.linalg.eigvals(T))[::-1]
    if isinstance(k, (list, set, tuple)):
        try:
            return [eig[n] for n in k]
        except IndexError:
            raise ValueError("given indices do not exist: ", n)
    elif k != None:
        return eig[: k]
    else:
        return eig

# TODO: Implement in Python directly
def eigenvalues_sensitivity(T, k=None):
    """
        computes the sensitivity of the specified eigenvalue

        k : int (optional)
            Compute the sensitivity of the first k eigenvalues of T.
    """

# TODO: ben: Implement in Python directly
def timescales(T, tau=1, k=None):
    """
        T: transition matrix
        tau: lag time
        k : int (optional)
            Compute the first k implied time scales of T.
    """

# TODO: ben: Implement in Python directly
def eigenvectors(T, k=None, right=True):
    r"""Compute eigenvectors of given transition matrix.

    Eigenvectors are computed using the numpy.linalg interface
    for the corresponding LAPACK routines.

    Input
    -----
    T : numpy.ndarray, shape(d,d)
        Transition matrix (stochastic matrix).
    k : int (optional) or array-like
        For integer k compute the first k eigenvalues of T
        else return those eigenvector sepcified by integer indices in k.

    Returns
    -------
    eigvec : numpy.ndarray, shape=(d, n)
        The eigenvectors of T ordered with decreasing absolute value of
        the corresponding eigenvalue. If k is None then n=d, if k is
        int then n=k otherwise n is the length of the given indices array.

    """
    if issparse(T):
        raise TypeError("Not implemented for sparse matrices.")
    elif isinstance(T, np.ndarray):
        return dense.decomposition.eigenvectors(T, k=k, right=right)
    else:
        raise TypeError("T is not a numpy.ndarray or a scipy.sparse matrix.")

# TODO: Implement in Python directly
def eigenvectors_sensitivity(T, k=None, right=True):
    """Compute eigenvector snesitivity of T

    k : int (optional)
        Compute eigenvectors to
    right : bool
        If True compute right eigenvectors, otherwise compute left eigenvectors.
    """

# TODO: ben: Implement in Python directly
def rdl_decomposition(T, k=None, norm='standard'):
    """Compute eigenvector snesitivity of T

        T : transition matrix
        k : int (optional)
            Number of eigenvector/eigenvalue pairs
        norm: standard | reversible
            standard: (L'R) = Id, l[0] is a probability distribution, r[i] have a 2-norm of 1
            reversible: R and L are related by stationary distribution of T

        Returns
        -------
        (R,D,L):    tuple of nd_arrays for right and left eigenvector matrix. Eigenvalues are passed as 1-d array
    """

# TODO: Implement in Python directly
def mfpt(T, i):
    """
    Computes vector of mean first passage times for given target state.

    Parameters
    ----------
    P : ndarray, shape=(n,n)
        Transition matrix.
    i : Integer
        Target state for mfpt calculation.

    Returns
    -------
    x : ndarray, shape=(n,)
        Vector of mean first passage times.

   """

# TODO: Implement in Python directly
def mfpt_sensitivity(T, i):
    """
    """

################################################################################
# Expectations
################################################################################

# TODO: martin: Implement in Python directly
def expectation(T, a):
    """
        computes the expectation value of a
    """

# TODO: Implement in Python directly
def expectation_sensitivity(T, a):
    """
        computes the sensitivity of the expectation value of a
    """

# TODO: ben: Implement in Python directly
def expected_counts(p0, T, N):
   """
   Expected transition counts for Markov chain with n steps.

                                \sum_{k=0}^n-1 diag(p^{T} T^{k})*T  n \geq 1
   E[C^{(n)}(x_0,\dotsc,x_n)]=
                                0                                  n=0

   Parameters
   ----------
   p0 : numpy array, shape=(n,)
      Starting (probability) vector of the chain, numpy.sum(p)=1.0.
   T : numpy array, shape=(n,n)
      Transition matrix for the chain. T\geq 0, numpy.sum(T,axis=1)=(1,...,1)
   N : int
      Number of steps for chain.

   Returns
   --------
   EC : numpy array, shape=(n,n)
       Expected value for transition counts after a propagation of n steps.

   """

# TODO: ben: Implement in Python directly
def expected_counts_stationary(P, N, mu=None):
   """
   Expected transition counts for Markov chain in equilibrium.

   Since mu is stationary for T we have

      E(C^{(n)})(x_0,dotsc,x_{n-1})=n diag(mu)*T.

   Parameters
   -----------
   P : numpy array, shape=(n,n)
      Transition matrix for the chain. T\geq 0, numpy.sum(T,axis=1)=(1,...,1)
   n : int
      Number of steps for chain.
   mu : numpy array, shape=(n,)
      Stationary probability vector of the chain, numpy.sum(p)=1.0.
      If mu is not specified it will be computed via diagonalization of T.

   Returns:
   --------
   EC : numpy array, shape=(n,n)
       Expected value for transition counts after a propagation of n steps.

   """

################################################################################
# Fingerprints
################################################################################

# TODO: martin: Implement in Python directly
def autocorrelation(P, obs):
    """Compute dynamical fingerprint crosscorrelation.

    The dynamical fingerprint autocorrelation is the timescale
    amplitude spectrum of the autocorrelation of the given observables
    under the action of the dynamics P

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs : ndarray, shape=(n,)
        Vector representing observable on discrete states

    Returns
    -------

    """

# TODO: Implement in Python directly
def crosscorrelation(P, obs1, obs2):
    """Compute dynamical fingerprint crosscorrelation.

    The dynamical fingerprint crosscorrelation is the timescale
    amplitude spectrum of the crosscorrelation of the given observables
    under the action of the dynamics P

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs1 : ndarray, shape=(n,)
        Vector representing observable on discrete states
    obs2 : ndarray, shape=(n,)
        Vector representing observable on discrete states

    Returns
    -------

    """

# TODO: Implement in Python directly
def perturbation(P, obs, p0):
    """

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs : ndarray, shape=(n,)
        Vector representing observable on discrete states
    p0 : ndarray, shape=(n,)
        Vector of initial probabilities.

    Returns :
    ---------

    """

################################################################################
# PCCA
################################################################################

# TODO: Implement in Python directly
def pcca(T, n):
    """
        returns a PCCA object
        T: transition matrix
        n: number of metastable processes
    """

################################################################################
# Transition path theory
################################################################################

# TODO: Implement in Python directly
def committor(P, A, B, forward=True):
    """Compute the committor between sets of microstates.

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    A : array_like
        List of integer state labels for set A
    B : array_like
        List of integer state labels for set B
    forward : bool
        If True compute the forward committor, else
        compute the backward committor.

    Returns
    -------
    x : ndarray, shape=(n, )
        Commitor vector.

    """

# TODO: Implement in Python directly
def committor_sensitivity(P, A, B, forward=True):
    """Compute the committor between sets of microstates.

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    A : array_like
        List of integer state labels for set A
    B : array_like
        List of integer state labels for set B
    forward : bool
        If True compute the forward committor, else
        compute the backward committor.

    Returns
    -------
    x : ndarray, shape=(n, )
        Commitor vector.

    """

# TODO: Translate from stallone
def tpt(T, A, B):
    """
        returns a TPT object
    """
<commit_msg>Updated TODOs in api.py<commit_after>################################################################################
# Assessment tools
################################################################################
import dense.assessment
import dense.decomposition

import numpy as np
from scipy.sparse import issparse

def is_transition_matrix(T, tol=1e-15):
    """
    True if T is a transition matrix

    Parameters
    ----------
    T : numpy.ndarray, shape(d, d) or scipy.sparse matrix
        Matrix to check
    tol : float
        tolerance to check with

    Returns
    -------
    Truth value: bool
        True, if T is positive and normed
        False, otherwise

    """
    if issparse(T):
        return sparse.assessment._is_stochastic_matrix_impl(T, tol)
    elif isinstance(T, np.ndarray):
        return dense.assessment._is_stochastic_matrix_impl(T, tol)
    else:
        raise ValueError("unsupported matrix type")

# TODO: martin: Implement in Python directly
def is_rate_matrix(K, tol=1e-15):
    """
        True if K is a rate matrix
    """

# TODO: Implement in Python directly
def is_ergodic(T, tol=1e-15):
    """
        True if T is connected (irreducible) and aperiodic
    """

# TODO: martin: Implement in Python directly
def is_reversible(T, mu=None, tol=1e-15):
    """
        True if T is a transition matrix
        mu : tests with respect to this stationary distribution
    """

################################################################################
# Eigenvalues and eigenvectors
################################################################################

# TODO: ben: Implement in Python directly
def mu(T):
    r"""Compute stationary distribution of stochastic matrix T.

    The stationary distribution is the left eigenvector corresponding to the
    non-degenerate eigenvalue :math: `\lambda=1`.

    Input
    -----
    T : numpy array, shape(d,d) or scipy.sparse matrix
        Transition matrix (stochastic matrix).

    Returns
    -------
    mu : numpy array, shape(d,)
        Vector of stationary probabilities.

    """
    if issparse(T):
        raise TypeError("Not implemented for sparse matrices.")
    elif isinstance(T, np.ndarray):
        return dense.decomposition.mu(T)
    else:
        raise TypeError("T is not a numpy.ndarray or a scipy.sparse matrix.")

# TODO: Implement in Python directly
def mu_sensitivity(T):
    """
        compute the sensitivity matrix of the stationary distribution of T
    """

def statdist(T):
    r"""Compute stationary distribution of stochastic matrix T.

    The stationary distribution is the left eigenvector corresponding to the
    non-degenerate eigenvalue :math: `\lambda=1`.

    Input
    -----
    T : numpy array, shape(d,d) or scipy.sparse matrix
        Transition matrix (stochastic matrix).

    Returns
    -------
    mu : numpy array, shape(d,)
        Vector of stationary probabilities.

    """
    statdist=mu(T)
    return statdist

# TODO: Implement in Python directly
def statdist_sensitivity(T):
    """
        compute the sensitivity matrix of the stationary distribution of T
    """
# TODO: Martin move implementation to dense.decomposition create implementation in sparse.decomposition
def eigenvalues(T, k=None):
    """
        computes the eigenvalues

        T : transition matrix
        k : int (optional) or tuple of ints
            Compute the first k eigenvalues of T.
    """
    eig = np.sort(np.linalg.eigvals(T))[::-1]
    if isinstance(k, (list, set, tuple)):
        try:
            return [eig[n] for n in k]
        except IndexError:
            raise ValueError("given indices do not exist: ", n)
    elif k != None:
        return eig[: k]
    else:
        return eig

# TODO: Implement in Python directly
def eigenvalues_sensitivity(T, k=None):
    """
        computes the sensitivity of the specified eigenvalue

        k : int (optional)
            Compute the sensitivity of the first k eigenvalues of T.
    """

# TODO: ben: Implement in Python directly
def timescales(T, tau=1, k=None):
    """
        T: transition matrix
        tau: lag time
        k : int (optional)
            Compute the first k implied time scales of T.
    """

# TODO: ben: Implement in Python directly
def eigenvectors(T, k=None, right=True):
    r"""Compute eigenvectors of given transition matrix.

    Eigenvectors are computed using the numpy.linalg interface
    for the corresponding LAPACK routines.

    Input
    -----
    T : numpy.ndarray, shape(d,d)
        Transition matrix (stochastic matrix).
    k : int (optional) or array-like
        For integer k compute the first k eigenvalues of T
        else return those eigenvector sepcified by integer indices in k.

    Returns
    -------
    eigvec : numpy.ndarray, shape=(d, n)
        The eigenvectors of T ordered with decreasing absolute value of
        the corresponding eigenvalue. If k is None then n=d, if k is
        int then n=k otherwise n is the length of the given indices array.

    """
    if issparse(T):
        raise TypeError("Not implemented for sparse matrices.")
    elif isinstance(T, np.ndarray):
        return dense.decomposition.eigenvectors(T, k=k, right=right)
    else:
        raise TypeError("T is not a numpy.ndarray or a scipy.sparse matrix.")

# TODO: Implement in Python directly
def eigenvectors_sensitivity(T, k=None, right=True):
    """Compute eigenvector snesitivity of T

    k : int (optional)
        Compute eigenvectors to
    right : bool
        If True compute right eigenvectors, otherwise compute left eigenvectors.
    """

# TODO: ben: Implement in Python directly
def rdl_decomposition(T, k=None, norm='standard'):
    """Compute eigenvector snesitivity of T

        T : transition matrix
        k : int (optional)
            Number of eigenvector/eigenvalue pairs
        norm: standard | reversible
            standard: (L'R) = Id, l[0] is a probability distribution, r[i] have a 2-norm of 1
            reversible: R and L are related by stationary distribution of T

        Returns
        -------
        (R,D,L):    tuple of nd_arrays for right and left eigenvector matrix. Eigenvalues are passed as 1-d array
    """

# TODO: Implement in Python directly
def mfpt(T, i):
    """
    Computes vector of mean first passage times for given target state.

    Parameters
    ----------
    P : ndarray, shape=(n,n)
        Transition matrix.
    i : Integer
        Target state for mfpt calculation.

    Returns
    -------
    x : ndarray, shape=(n,)
        Vector of mean first passage times.

   """

# TODO: Implement in Python directly
def mfpt_sensitivity(T, i):
    """
    """

################################################################################
# Expectations
################################################################################

# TODO: martin: Implement in Python directly
def expectation(T, a):
    """
        computes the expectation value of a
    """

# TODO: Implement in Python directly
def expectation_sensitivity(T, a):
    """
        computes the sensitivity of the expectation value of a
    """

# TODO: ben: Implement in Python directly
def expected_counts(p0, T, N):
   """
   Expected transition counts for Markov chain with n steps.

                                \sum_{k=0}^n-1 diag(p^{T} T^{k})*T  n \geq 1
   E[C^{(n)}(x_0,\dotsc,x_n)]=
                                0                                  n=0

   Parameters
   ----------
   p0 : numpy array, shape=(n,)
      Starting (probability) vector of the chain, numpy.sum(p)=1.0.
   T : numpy array, shape=(n,n)
      Transition matrix for the chain. T\geq 0, numpy.sum(T,axis=1)=(1,...,1)
   N : int
      Number of steps for chain.

   Returns
   --------
   EC : numpy array, shape=(n,n)
       Expected value for transition counts after a propagation of n steps.

   """

# TODO: ben: Implement in Python directly
def expected_counts_stationary(P, N, mu=None):
   """
   Expected transition counts for Markov chain in equilibrium.

   Since mu is stationary for T we have

      E(C^{(n)})(x_0,dotsc,x_{n-1})=n diag(mu)*T.

   Parameters
   -----------
   P : numpy array, shape=(n,n)
      Transition matrix for the chain. T\geq 0, numpy.sum(T,axis=1)=(1,...,1)
   n : int
      Number of steps for chain.
   mu : numpy array, shape=(n,)
      Stationary probability vector of the chain, numpy.sum(p)=1.0.
      If mu is not specified it will be computed via diagonalization of T.

   Returns:
   --------
   EC : numpy array, shape=(n,n)
       Expected value for transition counts after a propagation of n steps.

   """

################################################################################
# Fingerprints
################################################################################

# TODO: martin: Implement in Python directly
def autocorrelation(P, obs):
    """Compute dynamical fingerprint crosscorrelation.

    The dynamical fingerprint autocorrelation is the timescale
    amplitude spectrum of the autocorrelation of the given observables
    under the action of the dynamics P

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs : ndarray, shape=(n,)
        Vector representing observable on discrete states

    Returns
    -------

    """

# TODO: Implement in Python directly
def crosscorrelation(P, obs1, obs2):
    """Compute dynamical fingerprint crosscorrelation.

    The dynamical fingerprint crosscorrelation is the timescale
    amplitude spectrum of the crosscorrelation of the given observables
    under the action of the dynamics P

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs1 : ndarray, shape=(n,)
        Vector representing observable on discrete states
    obs2 : ndarray, shape=(n,)
        Vector representing observable on discrete states

    Returns
    -------

    """

# TODO: Implement in Python directly
def perturbation(P, obs, p0):
    """

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
       Transition matrix
    obs : ndarray, shape=(n,)
        Vector representing observable on discrete states
    p0 : ndarray, shape=(n,)
        Vector of initial probabilities.

    Returns :
    ---------

    """

################################################################################
# PCCA
################################################################################

# TODO: Implement in Python directly
def pcca(T, n):
    """
        returns a PCCA object
        T: transition matrix
        n: number of metastable processes
    """

################################################################################
# Transition path theory
################################################################################

# TODO: Implement in Python directly
def committor(P, A, B, forward=True):
    """Compute the committor between sets of microstates.

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    A : array_like
        List of integer state labels for set A
    B : array_like
        List of integer state labels for set B
    forward : bool
        If True compute the forward committor, else
        compute the backward committor.

    Returns
    -------
    x : ndarray, shape=(n, )
        Commitor vector.

    """

# TODO: Implement in Python directly
def committor_sensitivity(P, A, B, forward=True):
    """Compute the committor between sets of microstates.

    Parameters
    ----------
    P : ndarray, shape=(n, n) or scipy.sparse matrix
        Transition matrix
    A : array_like
        List of integer state labels for set A
    B : array_like
        List of integer state labels for set B
    forward : bool
        If True compute the forward committor, else
        compute the backward committor.

    Returns
    -------
    x : ndarray, shape=(n, )
        Commitor vector.

    """

# TODO: Translate from stallone
def tpt(T, A, B):
    """
        returns a TPT object
    """
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import requests

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

import redis

from celery.utils.log import get_task_logger
from celery import Celery

from lxml import etree
from lxml.etree import QName

from scrapy.utils.project import get_project_settings

from mydm.model import save_spider_settings, save_feed, is_exists_spider
from mydm.spiderfactory import mk_spider_cls
from mydm.util import parse_redis_url

settings = get_project_settings()
app = Celery('tasks', broker=settings['BROKER_URL'])
app.conf.CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
app.conf.CELERYD_MAX_TASKS_PER_CHILD = 1

def get_feed_name(url):
    parser = urlparse(url)
    names = parser.hostname.split('.')
    if len(names) == 1:
        return re.sub(r'[^a-zA-Z]', '', names[0]).capitalize()
    else:
        return ''.join([re.sub(r'[^a-z]', '', name.lower()).capitalize()
                        for name in names[:-1]
                        if name.lower() != 'www'])

def get_stats(custom_settings):
    conf = parse_redis_url(custom_settings['STATS_URL'])
    r = redis.Redis(host=conf.host, port=conf.port, db=conf.database)
    key = custom_settings['STATS_KEY']
    return int(r.get(key))

def check_spider(sp_setting):
    from scrapy.crawler import CrawlerProcess

    spcls = mk_spider_cls(sp_setting)
    custom_settings = {'ITEM_PIPELINES': {'mydm.pipelines.CountPipeline':
                                          255},
                       'WEBSERVICE_ENABLED': False,
                       'TELNETCONSOLE_ENABLED': False,
                       'LOG_LEVEL': 'INFO',
                       'LOG_STDOUT': True,
                       'LOG_ENABLED': True,
                       'STATS_URL': settings['TEMP_SPIDER_STATS_URL'],
                       'STATS_KEY': str(id(spcls))}

    p = CrawlerProcess(custom_settings)
    p.crawl(spcls)
    p.start()
    n = get_stats(custom_settings)
    return True if n > 0 else False

def _gen_lxmlspider(url, args):
    logger = get_task_logger(__name__)
    r = requests.get(url, headers=settings['DEFAULT_REQUEST_HEADERS'])
    if r.status_code != 200:
        logger.error('download {} error: status={}'.format(url, r.status_code))
        return False
    parser = etree.XMLParser(ns_clean=True)
    root = etree.XML(r.content, parser)
    while len(root) == 1:
        root = root[0]
    sp_setting = {'start_urls': [url]}
    for e in root:
        try:
            en = QName(e.tag).localname.lower()
        except ValueError:
            continue
        if en == 'title':
            sp_setting['title'] = re.sub(r'^(\r|\n|\s)+|(\r|\n|\s)+$',
                                         '',
                                         e.text)
    sp_setting['name'] = get_feed_name(url)
    if 'title' not in sp_setting:
        sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = args['category']
    if sp_setting['category'] not in settings['ARTICLE_CATEGORIES']:
        logger.error(u'{} category error'.format(sp_setting['category']))
        return False
    if 'item_content_xpath' in args:
        sp_setting['item_content_xpath'] = args['item_content_xpath']
    sp_setting['type'] = 'xml'
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    logger.error('gen_lxmlspider error for {}'.format(url))
    return False

@app.task(name='lxmlspider-creator')
def gen_lxmlspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    logger.info('gen_lxmlspider for {}'.format(url))
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    args = {k: v for k, v in spargs.items() if v != ''}
    if 'content' in spargs and len(spargs['content']) > 0:
        args['item_content_xpath'] = spargs['content']
    if not is_exists_spider(url):
        if _gen_lxmlspider(url, args):
            return True
    return False

@app.task(name='blogspider-creator')
def gen_blogspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    spattrs = ('entry', 'item_title', 'item_link', 'item_content')
    if any(spattr not in spargs for spattr in spattrs):
        return False
    sp_setting = {'{}_xpath'.format(k): v
                  for k, v in spargs.items() if k in spattrs}
    if 'removed_xpath_nodes' in spargs:
        nodes = [n.strip() for n in spargs['removed_xpath_nodes'].split(',')]
        sp_setting['removed_xpath_nodes'] = nodes
    sp_setting['name'] = get_feed_name(url)
    sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = spargs['category']
    sp_setting['type'] = 'blog'
    sp_setting['start_urls'] = [spargs['url']]
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    return False

"""
task for crawl
"""

@app.task(name='crawl-job')
def crawl(args):
    if len(args) == 0:
        return False

    def init_logger(settings):
        import logging
        logger = get_task_logger(__name__)
        LEVELS = {'DEBUG': logging.DEBUG,
                  'INFO': logging.INFO,
                  'WARNING': logging.WARNING,
                  'ERROR': logging.ERROR,
                  'CRITICAL': logging.CRITICAL}
        logger.setLevel(LEVELS[settings['LOG_LEVEL']])
        # from scrapy.utils.log import configure_logging
        # configure_logging(None, install_root_handler=False)

    init_logger(settings)

    from twisted.internet import reactor

    def run_spiders(settings):
        from scrapy.crawler import CrawlerRunner

        runner = CrawlerRunner(settings)
        loader = runner.spider_loader
        if args[0] == 'all':
            spiders = [loader.load(spid) for spid in loader.list()]
        else:
            spiders = [loader.load(spid)
                       for spid in args if spid in loader.list()]

        map(lambda sp: runner.crawl(sp), spiders)
        d = runner.join()
        d.addBoth(lambda _: reactor.stop())

    run_spiders(settings)
    reactor.run()
    return True
<commit_msg>logging<commit_after># -*- coding: utf-8 -*-

import re
import requests

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

import redis

from celery.utils.log import get_task_logger
from celery import Celery

from lxml import etree
from lxml.etree import QName

from scrapy.utils.project import get_project_settings

from mydm.model import save_spider_settings, save_feed, is_exists_spider
from mydm.spiderfactory import mk_spider_cls
from mydm.util import parse_redis_url

settings = get_project_settings()
app = Celery('tasks', broker=settings['BROKER_URL'])
app.conf.CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
app.conf.CELERYD_MAX_TASKS_PER_CHILD = 1

def get_feed_name(url):
    parser = urlparse(url)
    names = parser.hostname.split('.')
    if len(names) == 1:
        return re.sub(r'[^a-zA-Z]', '', names[0]).capitalize()
    else:
        return ''.join([re.sub(r'[^a-z]', '', name.lower()).capitalize()
                        for name in names[:-1]
                        if name.lower() != 'www'])

def get_stats(custom_settings):
    conf = parse_redis_url(custom_settings['STATS_URL'])
    r = redis.Redis(host=conf.host, port=conf.port, db=conf.database)
    key = custom_settings['STATS_KEY']
    return int(r.get(key))

def check_spider(sp_setting):
    from scrapy.crawler import CrawlerProcess

    spcls = mk_spider_cls(sp_setting)
    custom_settings = {'ITEM_PIPELINES': {'mydm.pipelines.CountPipeline':
                                          255},
                       'WEBSERVICE_ENABLED': False,
                       'TELNETCONSOLE_ENABLED': False,
                       'LOG_LEVEL': 'INFO',
                       'LOG_STDOUT': True,
                       'LOG_ENABLED': True,
                       'STATS_URL': settings['TEMP_SPIDER_STATS_URL'],
                       'STATS_KEY': str(id(spcls))}

    p = CrawlerProcess(custom_settings)
    p.crawl(spcls)
    p.start()
    n = get_stats(custom_settings)
    return True if n > 0 else False

def _gen_lxmlspider(url, args):
    logger = get_task_logger(__name__)
    r = requests.get(url, headers=settings['DEFAULT_REQUEST_HEADERS'])
    if r.status_code != 200:
        logger.error('download {} error: status={}'.format(url, r.status_code))
        return False
    parser = etree.XMLParser(ns_clean=True)
    root = etree.XML(r.content, parser)
    while len(root) == 1:
        root = root[0]
    sp_setting = {'start_urls': [url]}
    for e in root:
        try:
            en = QName(e.tag).localname.lower()
        except ValueError:
            continue
        if en == 'title':
            sp_setting['title'] = re.sub(r'^(\r|\n|\s)+|(\r|\n|\s)+$',
                                         '',
                                         e.text)
    sp_setting['name'] = get_feed_name(url)
    if 'title' not in sp_setting:
        sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = args['category']
    if sp_setting['category'] not in settings['ARTICLE_CATEGORIES']:
        logger.error(u'{} category error'.format(sp_setting['category']))
        return False
    if 'item_content_xpath' in args:
        sp_setting['item_content_xpath'] = args['item_content_xpath']
    sp_setting['type'] = 'xml'
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    logger.error('gen_lxmlspider error for {}'.format(url))
    return False

@app.task(name='lxmlspider-creator')
def gen_lxmlspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    logger.info('gen_lxmlspider for {}'.format(url))
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    args = {k: v for k, v in spargs.items() if v != ''}
    if 'content' in spargs and len(spargs['content']) > 0:
        args['item_content_xpath'] = spargs['content']
    if not is_exists_spider(url):
        if _gen_lxmlspider(url, args):
            return True
    return False

@app.task(name='blogspider-creator')
def gen_blogspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    spattrs = ('entry', 'item_title', 'item_link', 'item_content')
    if any(spattr not in spargs for spattr in spattrs):
        return False
    sp_setting = {'{}_xpath'.format(k): v
                  for k, v in spargs.items() if k in spattrs}
    if 'removed_xpath_nodes' in spargs:
        nodes = [n.strip() for n in spargs['removed_xpath_nodes'].split(',')]
        sp_setting['removed_xpath_nodes'] = nodes
    sp_setting['name'] = get_feed_name(url)
    sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = spargs['category']
    sp_setting['type'] = 'blog'
    sp_setting['start_urls'] = [spargs['url']]
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    return False

"""
task for crawl
"""

@app.task(name='crawl-job')
def crawl(args):
    if len(args) == 0:
        return False

    def init_logger(settings):
        import logging
        logger = get_task_logger(__name__)
        LEVELS = {'DEBUG': logging.DEBUG,
                  'INFO': logging.INFO,
                  'WARNING': logging.WARNING,
                  'ERROR': logging.ERROR,
                  'CRITICAL': logging.CRITICAL}
        logger.setLevel(LEVELS[settings['LOG_LEVEL']])
        from scrapy.utils.log import configure_logging
        configure_logging(None, install_root_handler=False)

    init_logger(settings)

    from twisted.internet import reactor

    def run_spiders(settings):
        from scrapy.crawler import CrawlerRunner

        runner = CrawlerRunner(settings)
        loader = runner.spider_loader
        if args[0] == 'all':
            spiders = [loader.load(spid) for spid in loader.list()]
        else:
            spiders = [loader.load(spid)
                       for spid in args if spid in loader.list()]

        map(lambda sp: runner.crawl(sp), spiders)
        d = runner.join()
        d.addBoth(lambda _: reactor.stop())

    run_spiders(settings)
    reactor.run()
    return True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import requests

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

import redis

from celery.utils.log import get_task_logger
from celery import Celery

from lxml import etree
from lxml.etree import QName

from scrapy.utils.project import get_project_settings

from mydm.model import save_spider_settings, save_feed, is_exists_spider
from mydm.spiderfactory import mk_spider_cls
from mydm.util import parse_redis_url

settings = get_project_settings()
app = Celery('tasks', broker=settings['BROKER_URL'])
app.conf.CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
app.conf.CELERYD_MAX_TASKS_PER_CHILD = 1

def get_feed_name(url):
    parser = urlparse(url)
    names = parser.hostname.split('.')
    if len(names) == 1:
        return re.sub(r'[^a-zA-Z]', '', names[0]).capitalize()
    else:
        return ''.join([re.sub(r'[^a-z]', '', name.lower()).capitalize()
                        for name in names[:-1]
                        if name.lower() != 'www'])

def get_stats(custom_settings):
    conf = parse_redis_url(custom_settings['STATS_URL'])
    r = redis.Redis(host=conf.host, port=conf.port, db=conf.database)
    key = custom_settings['STATS_KEY']
    return int(r.get(key))

def check_spider(sp_setting):
    from scrapy.crawler import CrawlerProcess

    spcls = mk_spider_cls(sp_setting)
    custom_settings = {'ITEM_PIPELINES': {'mydm.pipelines.CountPipeline':
                                          255},
                       'WEBSERVICE_ENABLED': False,
                       'TELNETCONSOLE_ENABLED': False,
                       'LOG_LEVEL': 'INFO',
                       'LOG_STDOUT': True,
                       'LOG_ENABLED': True,
                       'STATS_URL': settings['TEMP_SPIDER_STATS_URL'],
                       'STATS_KEY': str(id(spcls))}

    p = CrawlerProcess(custom_settings)
    p.crawl(spcls)
    p.start()
    n = get_stats(custom_settings)
    return True if n > 0 else False

def _gen_lxmlspider(url, args):
    logger = get_task_logger(__name__)
    r = requests.get(url, headers=settings['DEFAULT_REQUEST_HEADERS'])
    if r.status_code != 200:
        logger.error('download {} error: status={}'.format(url, r.status_code))
        return False
    parser = etree.XMLParser(ns_clean=True)
    root = etree.XML(r.content, parser)
    while len(root) == 1:
        root = root[0]
    sp_setting = {'start_urls': [url]}
    for e in root:
        try:
            en = QName(e.tag).localname.lower()
        except ValueError:
            continue
        if en == 'title':
            sp_setting['title'] = re.sub(r'^(\r|\n|\s)+|(\r|\n|\s)+$',
                                         '',
                                         e.text)
    sp_setting['name'] = get_feed_name(url)
    if 'title' not in sp_setting:
        sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = args['category']
    if sp_setting['category'] not in settings['ARTICLE_CATEGORIES']:
        logger.error(u'{} category error'.format(sp_setting['category']))
        return False
    if 'item_content_xpath' in args:
        sp_setting['item_content_xpath'] = args['item_content_xpath']
    sp_setting['type'] = 'xml'
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    logger.error('gen_lxmlspider error for {}'.format(url))
    return False

@app.task(name='lxmlspider-creator')
def gen_lxmlspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    logger.info('gen_lxmlspider for {}'.format(url))
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    args = {k: v for k, v in spargs.items() if v != ''}
    if 'content' in spargs and len(spargs['content']) > 0:
        args['item_content_xpath'] = spargs['content']
    if not is_exists_spider(url):
        if _gen_lxmlspider(url, args):
            return True
    return False

@app.task(name='blogspider-creator')
def gen_blogspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    spattrs = ('entry', 'item_title', 'item_link', 'item_content')
    if any(spattr not in spargs for spattr in spattrs):
        return False
    sp_setting = {'{}_xpath'.format(k): v
                  for k, v in spargs.items() if k in spattrs}
    if 'removed_xpath_nodes' in spargs:
        nodes = [n.strip() for n in spargs['removed_xpath_nodes'].split(',')]
        sp_setting['removed_xpath_nodes'] = nodes
    sp_setting['name'] = get_feed_name(url)
    sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = spargs['category']
    sp_setting['type'] = 'blog'
    sp_setting['start_urls'] = [spargs['url']]
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    return False

"""
task for crawl
"""

@app.task(name='crawl-job')
def crawl(args):
    if len(args) == 0:
        return False

    def init_logger(settings):
        import logging
        logger = get_task_logger(__name__)
        LEVELS = {'DEBUG': logging.DEBUG,
                  'INFO': logging.INFO,
                  'WARNING': logging.WARNING,
                  'ERROR': logging.ERROR,
                  'CRITICAL': logging.CRITICAL}
        logger.setLevel(LEVELS[settings['LOG_LEVEL']])
        from scrapy.utils.log import configure_logging
        configure_logging(None, install_root_handler=False)

    init_logger(settings)

    from twisted.internet import reactor

    def run_spiders(settings):
        from scrapy.crawler import CrawlerRunner

        runner = CrawlerRunner(settings)
        loader = runner.spider_loader
        if args[0] == 'all':
            spiders = [loader.load(spid) for spid in loader.list()]
        else:
            spiders = [loader.load(spid)
                       for spid in args if spid in loader.list()]

        map(lambda sp: runner.crawl(sp), spiders)
        d = runner.join()
        d.addBoth(lambda _: reactor.stop())

    run_spiders(settings)
    reactor.run()
    return True
<commit_msg>logging<commit_after># -*- coding: utf-8 -*-

import re
import requests

try:
    from urllib.parse import urlparse
except ImportError:
    from urlparse import urlparse

import redis

from celery.utils.log import get_task_logger
from celery import Celery

from lxml import etree
from lxml.etree import QName

from scrapy.utils.project import get_project_settings

from mydm.model import save_spider_settings, save_feed, is_exists_spider
from mydm.spiderfactory import mk_spider_cls
from mydm.util import parse_redis_url

settings = get_project_settings()
app = Celery('tasks', broker=settings['BROKER_URL'])
app.conf.CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
app.conf.CELERYD_MAX_TASKS_PER_CHILD = 1

def get_feed_name(url):
    parser = urlparse(url)
    names = parser.hostname.split('.')
    if len(names) == 1:
        return re.sub(r'[^a-zA-Z]', '', names[0]).capitalize()
    else:
        return ''.join([re.sub(r'[^a-z]', '', name.lower()).capitalize()
                        for name in names[:-1]
                        if name.lower() != 'www'])

def get_stats(custom_settings):
    conf = parse_redis_url(custom_settings['STATS_URL'])
    r = redis.Redis(host=conf.host, port=conf.port, db=conf.database)
    key = custom_settings['STATS_KEY']
    return int(r.get(key))

def check_spider(sp_setting):
    from scrapy.crawler import CrawlerProcess

    spcls = mk_spider_cls(sp_setting)
    custom_settings = {'ITEM_PIPELINES': {'mydm.pipelines.CountPipeline':
                                          255},
                       'WEBSERVICE_ENABLED': False,
                       'TELNETCONSOLE_ENABLED': False,
                       'LOG_LEVEL': 'INFO',
                       'LOG_STDOUT': True,
                       'LOG_ENABLED': True,
                       'STATS_URL': settings['TEMP_SPIDER_STATS_URL'],
                       'STATS_KEY': str(id(spcls))}

    p = CrawlerProcess(custom_settings)
    p.crawl(spcls)
    p.start()
    n = get_stats(custom_settings)
    return True if n > 0 else False

def _gen_lxmlspider(url, args):
    logger = get_task_logger(__name__)
    r = requests.get(url, headers=settings['DEFAULT_REQUEST_HEADERS'])
    if r.status_code != 200:
        logger.error('download {} error: status={}'.format(url, r.status_code))
        return False
    parser = etree.XMLParser(ns_clean=True)
    root = etree.XML(r.content, parser)
    while len(root) == 1:
        root = root[0]
    sp_setting = {'start_urls': [url]}
    for e in root:
        try:
            en = QName(e.tag).localname.lower()
        except ValueError:
            continue
        if en == 'title':
            sp_setting['title'] = re.sub(r'^(\r|\n|\s)+|(\r|\n|\s)+$',
                                         '',
                                         e.text)
    sp_setting['name'] = get_feed_name(url)
    if 'title' not in sp_setting:
        sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = args['category']
    if sp_setting['category'] not in settings['ARTICLE_CATEGORIES']:
        logger.error(u'{} category error'.format(sp_setting['category']))
        return False
    if 'item_content_xpath' in args:
        sp_setting['item_content_xpath'] = args['item_content_xpath']
    sp_setting['type'] = 'xml'
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    logger.error('gen_lxmlspider error for {}'.format(url))
    return False

@app.task(name='lxmlspider-creator')
def gen_lxmlspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    logger.info('gen_lxmlspider for {}'.format(url))
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    args = {k: v for k, v in spargs.items() if v != ''}
    if 'content' in spargs and len(spargs['content']) > 0:
        args['item_content_xpath'] = spargs['content']
    if not is_exists_spider(url):
        if _gen_lxmlspider(url, args):
            return True
    return False

@app.task(name='blogspider-creator')
def gen_blogspider(spargs):
    logger = get_task_logger(__name__)
    url = spargs['url']
    parser = urlparse(url)
    if parser.scheme == '' or parser.netloc == '':
        logger.error('{} invalid url'.format(url))
        return False
    save_feed(url)
    spattrs = ('entry', 'item_title', 'item_link', 'item_content')
    if any(spattr not in spargs for spattr in spattrs):
        return False
    sp_setting = {'{}_xpath'.format(k): v
                  for k, v in spargs.items() if k in spattrs}
    if 'removed_xpath_nodes' in spargs:
        nodes = [n.strip() for n in spargs['removed_xpath_nodes'].split(',')]
        sp_setting['removed_xpath_nodes'] = nodes
    sp_setting['name'] = get_feed_name(url)
    sp_setting['title'] = sp_setting['name']
    sp_setting['category'] = spargs['category']
    sp_setting['type'] = 'blog'
    sp_setting['start_urls'] = [spargs['url']]
    if check_spider(sp_setting):
        save_spider_settings(sp_setting)
        return True
    return False

"""
task for crawl
"""

@app.task(name='crawl-job')
def crawl(args):
    if len(args) == 0:
        return False

    def init_logger(settings):
        import logging
        logger = get_task_logger(__name__)
        LEVELS = {'DEBUG': logging.DEBUG,
                  'INFO': logging.INFO,
                  'WARNING': logging.WARNING,
                  'ERROR': logging.ERROR,
                  'CRITICAL': logging.CRITICAL}
        logger.setLevel(LEVELS[settings['LOG_LEVEL']])
        # from scrapy.utils.log import configure_logging
        # configure_logging(None, install_root_handler=False)

    init_logger(settings)

    from twisted.internet import reactor

    def run_spiders(settings):
        from scrapy.crawler import CrawlerRunner

        runner = CrawlerRunner(settings)
        loader = runner.spider_loader
        if args[0] == 'all':
            spiders = [loader.load(spid) for spid in loader.list()]
        else:
            spiders = [loader.load(spid)
                       for spid in args if spid in loader.list()]

        map(lambda sp: runner.crawl(sp), spiders)
        d = runner.join()
        d.addBoth(lambda _: reactor.stop())

    run_spiders(settings)
    reactor.run()
    return True
<|endoftext|>