<commit_before>from calendar import c
import click
import cooler
from numpy import cov

from .. import coverage

from . import cli
from .. import api
from ..lib.common import make_cooler_view
from ..lib.io import read_viewframe_from_file
import bioframe
import multiprocessing as mp

@cli.command()
@click.argument(
    "cool_path", metavar="COOL_PATH", type=str, nargs=1,
)
@click.option(
    "--output",
    "-o",
    help="Specify output file name to store the coverage in a tsv format.",
    type=str,
    required=False,
)
@click.option(
    "--ignore-diags",
    help="The number of diagonals to ignore. By default, equals"
    " the number of diagonals ignored during IC balancing.",
    type=int,
    default=None,
    show_default=True,
)
@click.option(
    "--store",
    help="Append columns with coverage (cis_raw_cov, tot_raw_cov) "
    " to the cooler bin table.",
    is_flag=True,
)
@click.option(
    "--chunksize",
    help="Split the contact matrix pixel records into equally sized chunks to"
    " save memory and/or parallelize. Default is 10^7",
    type=int,
    default=1e7,
    show_default=True,
)
@click.option(
    "--bigwig",
    help="Also save output as bigWig files for cis and total coverage"
    " with the names <output>.<cis/tot>.bw",
    is_flag=True,
    default=False,
)
@click.option(
    "-p",
    "--nproc",
    help="Number of processes to split the work between."
    " [default: 1, i.e. no process pool]",
    default=1,
    type=int,
)
def coverage(
    cool_path, output, ignore_diags, store, chunksize, bigwig, nproc,
):
    """
    Calculate the sums of cis and genome-wide contacts (aka coverage aka marginals) for
    a sparse Hi-C contact map in Cooler HDF5 format.
    Note that the sum(tot_cov) from this function is two times the number of reads
    contributing to the cooler, as each side contributes to the coverage.

    COOL_PATH : The paths to a .cool file with a balanced Hi-C map.

    """

    clr = cooler.Cooler(cool_path)

    # Create view:
    cooler_view_df = make_cooler_view(clr)
    if nproc > 1:
        pool = mp.Pool(nproc)
        _map = pool.imap
    else:
        _map = map

    cis_cov, tot_cov = api.coverage.coverage(
        clr, ignore_diags=ignore_diags, chunksize=chunksize, store=store
    )

    if nproc > 1:
        pool.close()

    coverage_table = clr.bins()[:][["chrom", "start", "end"]]
    coverage_table["cis_raw_cov"] = cis_cov.astype(int)
    coverage_table["tot_raw_cov"] = tot_cov.astype(int)

    # output to file if specified:
    if output:
        coverage_table.to_csv(output, sep="\t", index=False, na_rep="nan")
    # or print into stdout otherwise:
    else:
        print(coverage_table.to_csv(sep="\t", index=False, na_rep="nan"))

    # Write the coverage tracks as a bigwigs:
    if bigwig:
        bioframe.to_bigwig(
            coverage_table,
            clr.chromsizes,
            f"{output}.cis.bw",
            value_field="cis_raw_cov",
        )
        bioframe.to_bigwig(
            coverage_table,
            clr.chromsizes,
            f"{output}.tot.bw",
            value_field="tot_raw_cov",
        )
<commit_msg>actually pass _map to function<commit_after>from calendar import c
import click
import cooler
from numpy import cov

from .. import coverage

from . import cli
from .. import api
from ..lib.common import make_cooler_view
from ..lib.io import read_viewframe_from_file
import bioframe
import multiprocessing as mp

@cli.command()
@click.argument(
    "cool_path", metavar="COOL_PATH", type=str, nargs=1,
)
@click.option(
    "--output",
    "-o",
    help="Specify output file name to store the coverage in a tsv format.",
    type=str,
    required=False,
)
@click.option(
    "--ignore-diags",
    help="The number of diagonals to ignore. By default, equals"
    " the number of diagonals ignored during IC balancing.",
    type=int,
    default=None,
    show_default=True,
)
@click.option(
    "--store",
    help="Append columns with coverage (cis_raw_cov, tot_raw_cov) "
    " to the cooler bin table.",
    is_flag=True,
)
@click.option(
    "--chunksize",
    help="Split the contact matrix pixel records into equally sized chunks to"
    " save memory and/or parallelize. Default is 10^7",
    type=int,
    default=1e7,
    show_default=True,
)
@click.option(
    "--bigwig",
    help="Also save output as bigWig files for cis and total coverage"
    " with the names <output>.<cis/tot>.bw",
    is_flag=True,
    default=False,
)
@click.option(
    "-p",
    "--nproc",
    help="Number of processes to split the work between."
    " [default: 1, i.e. no process pool]",
    default=1,
    type=int,
)
def coverage(
    cool_path, output, ignore_diags, store, chunksize, bigwig, nproc,
):
    """
    Calculate the sums of cis and genome-wide contacts (aka coverage aka marginals) for
    a sparse Hi-C contact map in Cooler HDF5 format.
    Note that the sum(tot_cov) from this function is two times the number of reads
    contributing to the cooler, as each side contributes to the coverage.

    COOL_PATH : The paths to a .cool file with a balanced Hi-C map.

    """

    clr = cooler.Cooler(cool_path)

    # Create view:
    cooler_view_df = make_cooler_view(clr)
    if nproc > 1:
        pool = mp.Pool(nproc)
        _map = pool.imap
    else:
        _map = map

    cis_cov, tot_cov = api.coverage.coverage(
        clr, ignore_diags=ignore_diags, chunksize=chunksize, map=_map, store=store
    )

    if nproc > 1:
        pool.close()

    coverage_table = clr.bins()[:][["chrom", "start", "end"]]
    coverage_table["cis_raw_cov"] = cis_cov.astype(int)
    coverage_table["tot_raw_cov"] = tot_cov.astype(int)

    # output to file if specified:
    if output:
        coverage_table.to_csv(output, sep="\t", index=False, na_rep="nan")
    # or print into stdout otherwise:
    else:
        print(coverage_table.to_csv(sep="\t", index=False, na_rep="nan"))

    # Write the coverage tracks as a bigwigs:
    if bigwig:
        bioframe.to_bigwig(
            coverage_table,
            clr.chromsizes,
            f"{output}.cis.bw",
            value_field="cis_raw_cov",
        )
        bioframe.to_bigwig(
            coverage_table,
            clr.chromsizes,
            f"{output}.tot.bw",
            value_field="tot_raw_cov",
        )
<|endoftext|>


<commit_before>import os
import stat
import subprocess
from abc import ABCMeta
from os.path import join
from subprocess import PIPE

from coon.pac_cache.local_cache import LocalCache
from coon.packages.config.config import ConfigFile
from coon.packages.package import Package
from coon.tool.tool import AbstractTool
from coon.utils.file_utils import check_cmd
from coon.utils.logger import critical, error, info, debug

def run_cmd(cmd: str or list, project: str, path: str,
            env_vars: dict or None = None, shell=False, output=PIPE) -> bool:
    debug(cmd)
    if env_vars is None:
        env_vars = dict(os.environ)
    ensure_runnable(cmd)
    p = subprocess.Popen(cmd, stdout=output, stderr=output, cwd=path, env=env_vars, shell=shell)
    if p.wait() != 0:
        critical(project + ' failed.')
        if output is not None:
            error(p.stderr.read().decode('utf8'))
            error(p.stdout.read().decode('utf8'))
        return False
    else:
        return True

def ensure_runnable(cmd: str):
    if cmd[0].startswith("./"):
        if not os.access(cmd, os.X_OK):
            st = os.stat(cmd)
            os.chmod(cmd[0], st.st_mode | stat.S_IEXEC)

class AbstractCompiler(metaclass=ABCMeta):
    def __init__(self, package: Package, executable='erlc'):
        self._package = package
        self._executable = executable
        self._tool = None

    @property
    def package(self):
        return self._package

    @property
    def project_name(self) -> str:
        return self.package.name

    @property
    def executable(self) -> str:
        return self._executable

    @property
    def root_path(self) -> str:  # Project path.
        return self.package.path

    @property
    def src_path(self) -> str:
        return join(self.package.path, 'src')

    @property
    def include_path(self) -> str:
        return join(self.package.path, 'include')

    @property
    def output_path(self) -> str:
        return join(self.package.path, 'ebin')

    @property
    def test_path(self) -> str:
        return join(self.package.path, 'test')

    @property
    def tool(self) -> AbstractTool or None:
        return self._tool

    @property
    def build_vars(self) -> list:
        return self.package.config.build_vars

    def compile(self, override_config: ConfigFile or None = None) -> bool:
        info(self.executable + ' build ' + self.project_name)
        return run_cmd(self.executable, self.project_name, self.root_path, output=None)

    def unit(self) -> bool:
        raise RuntimeError("Don't know how to run unit tests with " + self.executable)

    def common(self, log_dir: str) -> bool:
        raise RuntimeError("Don't know how to run common tests with " + self.executable)

    # find tool and link to project
    def ensure_tool(self, cache: LocalCache):
        if self.tool is None:  # no need to check tool for this compiler
            return
        maybe_tool = check_cmd(self.root_path, self.tool.name)
        if maybe_tool is True:  # found locally
            self._executable = self.tool.local_executable
            return
        elif maybe_tool:  # installed in system
            return
        elif maybe_tool is False:  # not found
            maybe_tool = self.__find_in_local(cache, self.tool)
        if maybe_tool is None:  # not found in local cache
            maybe_tool = self.__build_tool(cache, self.tool)
        self._executable = maybe_tool  # was linked to local dir

    def __find_in_local(self, cache: LocalCache, tool: AbstractTool):
        if cache.tool_exists(tool.name):  # tool is in local cache
            cache.link_tool(self.package, tool.name)
            return tool.local_executable
        return None

    def __build_tool(self, cache: LocalCache, tool: AbstractTool):
        path = cache.temp_dir
        tool_path = tool.ensure(path)
        cache.add_tool(tool.name, tool_path)
        cache.link_tool(self.package, tool.name)
        return tool.local_executable
<commit_msg>another quickfix<commit_after>import os
import stat
import subprocess
from abc import ABCMeta
from os.path import join
from subprocess import PIPE

from coon.pac_cache.local_cache import LocalCache
from coon.packages.config.config import ConfigFile
from coon.packages.package import Package
from coon.tool.tool import AbstractTool
from coon.utils.file_utils import check_cmd
from coon.utils.logger import critical, error, info, debug

def run_cmd(cmd: str or list, project: str, path: str,
            env_vars: dict or None = None, shell=False, output=PIPE) -> bool:
    debug(cmd)
    if env_vars is None:
        env_vars = dict(os.environ)
    ensure_runnable(cmd)
    p = subprocess.Popen(cmd, stdout=output, stderr=output, cwd=path, env=env_vars, shell=shell)
    if p.wait() != 0:
        critical(project + ' failed.')
        if output is not None:
            error(p.stderr.read().decode('utf8'))
            error(p.stdout.read().decode('utf8'))
        return False
    else:
        return True

def ensure_runnable(cmd: str):
    command = cmd[0]
    if command.startswith("./"):
        if not os.access(command, os.X_OK):
            st = os.stat(command)
            os.chmod(command, st.st_mode | stat.S_IEXEC)

class AbstractCompiler(metaclass=ABCMeta):
    def __init__(self, package: Package, executable='erlc'):
        self._package = package
        self._executable = executable
        self._tool = None

    @property
    def package(self):
        return self._package

    @property
    def project_name(self) -> str:
        return self.package.name

    @property
    def executable(self) -> str:
        return self._executable

    @property
    def root_path(self) -> str:  # Project path.
        return self.package.path

    @property
    def src_path(self) -> str:
        return join(self.package.path, 'src')

    @property
    def include_path(self) -> str:
        return join(self.package.path, 'include')

    @property
    def output_path(self) -> str:
        return join(self.package.path, 'ebin')

    @property
    def test_path(self) -> str:
        return join(self.package.path, 'test')

    @property
    def tool(self) -> AbstractTool or None:
        return self._tool

    @property
    def build_vars(self) -> list:
        return self.package.config.build_vars

    def compile(self, override_config: ConfigFile or None = None) -> bool:
        info(self.executable + ' build ' + self.project_name)
        return run_cmd(self.executable, self.project_name, self.root_path, output=None)

    def unit(self) -> bool:
        raise RuntimeError("Don't know how to run unit tests with " + self.executable)

    def common(self, log_dir: str) -> bool:
        raise RuntimeError("Don't know how to run common tests with " + self.executable)

    # find tool and link to project
    def ensure_tool(self, cache: LocalCache):
        if self.tool is None:  # no need to check tool for this compiler
            return
        maybe_tool = check_cmd(self.root_path, self.tool.name)
        if maybe_tool is True:  # found locally
            self._executable = self.tool.local_executable
            return
        elif maybe_tool:  # installed in system
            return
        elif maybe_tool is False:  # not found
            maybe_tool = self.__find_in_local(cache, self.tool)
        if maybe_tool is None:  # not found in local cache
            maybe_tool = self.__build_tool(cache, self.tool)
        self._executable = maybe_tool  # was linked to local dir

    def __find_in_local(self, cache: LocalCache, tool: AbstractTool):
        if cache.tool_exists(tool.name):  # tool is in local cache
            cache.link_tool(self.package, tool.name)
            return tool.local_executable
        return None

    def __build_tool(self, cache: LocalCache, tool: AbstractTool):
        path = cache.temp_dir
        tool_path = tool.ensure(path)
        cache.add_tool(tool.name, tool_path)
        cache.link_tool(self.package, tool.name)
        return tool.local_executable
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_junkfiles
# Purpose:      From Spidering, identifies backup and temporary files.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     23/08/2014
# Copyright:   (c) Steve Micallef 2014
# Licence:     GPL
# -------------------------------------------------------------------------------

from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_junkfiles(SpiderFootPlugin):
    """Junk Files:Footprint:Looks for old/temporary and other similar files."""

    # Default options
    opts = {
        'fileexts': ['tmp', 'bak', 'old', 'backup', 'new'],
        'urlextstry': ['html', 'htm', 'php', 'jsp', 'txt', 'js'],
        'files': ['x', 'xxx', 'crap', 'old', 'a', 'aaa', 'z', 'zzz',
                  'out', 'sql', "passwd", ".htaccess", ".htpasswd",
                  "Thumbs.db", "asd", "asdf"],
        'dirs': ['zip', 'tar.gz', 'tgz', 'tar']
    }

    # Option descriptions
    optdescs = {
        'fileexts': "File extensions to try.",
        'urlextstry': "Try those extensions against URLs with these extensions.",
        'files': "Try to fetch each of these files from the directory of the URL.",
        'dirs': "Try to fetch the containing folder with these extensions."
    }

    results = list()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = list()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["LINKED_URL_INTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["JUNK_FILE"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventData in self.results:
            return None
        else:
            self.results.append(eventData)

        # http://www/blah/abc.html -> try http://www/blah/abc.html.[fileexts]
        for ext in self.opts['urlextstry']:
            if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \
                    eventData.endswith(ext):
                bits = eventData.split("?")
                for x in self.opts['fileexts']:
                    self.sf.debug("Trying " + x + " against " + eventData)
                    fetch = bits[0] + "." + x
                    if fetch not in self.results:
                        self.results.append(fetch)
                    else:
                        continue
                    res = self.sf.fetchUrl(fetch,
                                           timeout=self.opts['_fetchtimeout'],
                                           useragent=self.opts['_useragent'])
                    if res['content'] is not None and res['code'] != "404":
                        evt = SpiderFootEvent("JUNK_FILE", fetch,
                                              self.__name__, event)
                        self.notifyListeners(evt)

        # http://www or http://www/
        # -> try index pages
        if eventData.endswith("/") or eventData.count("/") == 2:
            for ext in ["index.html", "index.htm", "default.htm"]:
                self.sf.debug("Trying " + ext + " against " + eventData)
                for x in self.opts['fileexts']:
                    fetch = eventData + "/" + ext + "." + x
                    if fetch not in self.results:
                        self.results.append(fetch)
                    else:
                        continue
                    res = self.sf.fetchUrl(fetch,
                                           timeout=self.opts['_fetchtimeout'],
                                           useragent=self.opts['_useragent'])
                    if res['content'] is not None and res['code'] != "404":
                        evt = SpiderFootEvent("JUNK_FILE", fetch,
                                              self.__name__, event)
                        self.notifyListeners(evt)

        base = self.sf.urlBaseDir(eventData)
        # don't do anything with the root directory of a site
        if base == eventData + "/" or base == eventData:
            return None

        # http://www/blah/abc.html -> try http://www/blah.[dirs]
        for dirfile in self.opts['dirs']:
            self.sf.debug("Trying " + dirfile + " against " + eventData)
            fetch = base[0:len(base) - 1] + "." + dirfile
            if fetch not in self.results:
                self.results.append(fetch)
            else:
                continue
            res = self.sf.fetchUrl(fetch,
                                   timeout=self.opts['_fetchtimeout'],
                                   useragent=self.opts['_useragent'])
            if res['content'] is not None and res['code'] != "404":
                evt = SpiderFootEvent("JUNK_FILE", fetch,
                                      self.__name__, event)
                self.notifyListeners(evt)

        # http://www/blah/abc.html -> try http://www/blah/[files]
        for f in self.opts['files']:
            self.sf.debug("Trying " + f + " against " + eventData)
            fetch = base + f
            if fetch not in self.results:
                self.results.append(fetch)
            else:
                continue
            res = self.sf.fetchUrl(fetch,
                                   timeout=self.opts['_fetchtimeout'],
                                   useragent=self.opts['_useragent'])
            if res['content'] is not None and res['code'] != "404":
                evt = SpiderFootEvent("JUNK_FILE", fetch,
                                      self.__name__, event)
                self.notifyListeners(evt)

        return None

# End of sfp_junkfiles class
<commit_msg>Made junk file checker abort aware.<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_junkfiles
# Purpose:      From Spidering, identifies backup and temporary files.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     23/08/2014
# Copyright:   (c) Steve Micallef 2014
# Licence:     GPL
# -------------------------------------------------------------------------------

from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_junkfiles(SpiderFootPlugin):
    """Junk Files:Footprint:Looks for old/temporary and other similar files."""

    # Default options
    opts = {
        'fileexts': ['tmp', 'bak', 'old', 'backup', 'new'],
        'urlextstry': ['html', 'htm', 'php', 'jsp', 'txt', 'js'],
        'files': ['x', 'xxx', 'crap', 'old', 'a', 'aaa', 'z', 'zzz',
                  'out', 'sql', "passwd", ".htaccess", ".htpasswd",
                  "Thumbs.db", "asd", "asdf"],
        'dirs': ['zip', 'tar.gz', 'tgz', 'tar']
    }

    # Option descriptions
    optdescs = {
        'fileexts': "File extensions to try.",
        'urlextstry': "Try those extensions against URLs with these extensions.",
        'files': "Try to fetch each of these files from the directory of the URL.",
        'dirs': "Try to fetch the containing folder with these extensions."
    }

    results = list()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = list()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["LINKED_URL_INTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["JUNK_FILE"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventData in self.results:
            return None
        else:
            self.results.append(eventData)

        # http://www/blah/abc.html -> try http://www/blah/abc.html.[fileexts]
        for ext in self.opts['urlextstry']:
            if "." + ext + "?" in eventData or "." + ext + "#" in eventData or \
                    eventData.endswith(ext):
                bits = eventData.split("?")
                for x in self.opts['fileexts']:
                    if self.checkForStop():
                        return None

                    self.sf.debug("Trying " + x + " against " + eventData)
                    fetch = bits[0] + "." + x
                    if fetch not in self.results:
                        self.results.append(fetch)
                    else:
                        continue
                    res = self.sf.fetchUrl(fetch,
                                           timeout=self.opts['_fetchtimeout'],
                                           useragent=self.opts['_useragent'])
                    if res['content'] is not None and res['code'] != "404":
                        evt = SpiderFootEvent("JUNK_FILE", fetch,
                                              self.__name__, event)
                        self.notifyListeners(evt)

        # http://www or http://www/
        # -> try index pages
        if eventData.endswith("/") or eventData.count("/") == 2:
            for ext in ["index.html", "index.htm", "default.htm"]:
                self.sf.debug("Trying " + ext + " against " + eventData)
                for x in self.opts['fileexts']:
                    if self.checkForStop():
                        return None

                    fetch = eventData + "/" + ext + "." + x
                    if fetch not in self.results:
                        self.results.append(fetch)
                    else:
                        continue
                    res = self.sf.fetchUrl(fetch,
                                           timeout=self.opts['_fetchtimeout'],
                                           useragent=self.opts['_useragent'])
                    if res['content'] is not None and res['code'] != "404":
                        evt = SpiderFootEvent("JUNK_FILE", fetch,
                                              self.__name__, event)
                        self.notifyListeners(evt)

        base = self.sf.urlBaseDir(eventData)
        # don't do anything with the root directory of a site
        if base == eventData + "/" or base == eventData:
            return None

        # http://www/blah/abc.html -> try http://www/blah.[dirs]
        for dirfile in self.opts['dirs']:
            if self.checkForStop():
                return None

            self.sf.debug("Trying " + dirfile + " against " + eventData)
            fetch = base[0:len(base) - 1] + "." + dirfile
            if fetch not in self.results:
                self.results.append(fetch)
            else:
                continue
            res = self.sf.fetchUrl(fetch,
                                   timeout=self.opts['_fetchtimeout'],
                                   useragent=self.opts['_useragent'])
            if res['content'] is not None and res['code'] != "404":
                evt = SpiderFootEvent("JUNK_FILE", fetch,
                                      self.__name__, event)
                self.notifyListeners(evt)

        # http://www/blah/abc.html -> try http://www/blah/[files]
        for f in self.opts['files']:
            if self.checkForStop():
                return None

            self.sf.debug("Trying " + f + " against " + eventData)
            fetch = base + f
            if fetch not in self.results:
                self.results.append(fetch)
            else:
                continue
            res = self.sf.fetchUrl(fetch,
                                   timeout=self.opts['_fetchtimeout'],
                                   useragent=self.opts['_useragent'])
            if res['content'] is not None and res['code'] != "404":
                evt = SpiderFootEvent("JUNK_FILE", fetch,
                                      self.__name__, event)
                self.notifyListeners(evt)

        return None

# End of sfp_junkfiles class
<|endoftext|>


<commit_before>#-------------------------------------------------------------------------------
# Name:        sfp_numverify
# Purpose:     SpiderFoot plug-in to search numverify.com API for a phone number
#              and retrieve location and carrier information.
#
# Author:      <bcoles@gmail.com>
#
# Created:     2019-05-25
# Copyright:   (c) bcoles 2019
# Licence:     GPL
#-------------------------------------------------------------------------------

import json
import re
import urllib
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_numverify(SpiderFootPlugin):
    """numverify:Footprint,Investigate,Passive:Real World:apikey:Lookup phone number location and carrier information from numverify.com."""

    # Default options
    opts = {
        'api_key': ''
    }

    # Option descriptions
    optdescs = {
        'api_key': 'numverify API key.'
    }

    results = dict()
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.__dataSource__ = "numverify"
        self.results = dict()
        self.errorState = False

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['PHONE_NUMBER']

    # What events this module produces
    def producedEvents(self):
        return ['RAW_RIR_DATA', 'GEOINFO', 'PROVIDER_TELCO']

    # Query numverify API for the specified phone number
    # https://numverify.com/documentation
    def query(self, qry):
        number = qry.strip('+').strip('(').strip(')')

        params = {
            'number': number.encode('raw_unicode_escape'),
            'country_code': '',
            'format': '0', # set to "1" for prettified debug output
            'access_key': self.opts['api_key']
        }

        # Free API does not support HTTPS for no adequately explained reason
        res = self.sf.fetchUrl("http://apilayer.net/api/validate?" + urllib.urlencode(params),
                               timeout=self.opts['_fetchtimeout'],
                               useragent=self.opts['_useragent'])

        time.sleep(1)

        if res['content'] is None:
            self.sf.debug('No response from apilayer.net')
            return None

        if res['code'] == '101':
            self.sf.error('API error: invalid API key', False)
            self.errorState = True
            return None

        if res['code'] == '102':
            self.sf.error('API error: user account deactivated', False)
            self.errorState = True
            return None

        if res['code'] == '104':
            self.sf.error('API error: usage limit exceeded', False)
            self.errorState = True
            return None

        try:
            data = json.loads(res['content'])
        except BaseException as e:
            self.sf.debug('Error processing JSON response: ' + str(e))
            return None

        if data.get('error') is not None:
            self.sf.error('API error: ' + str(data.get('error')), False)
            return None

        return data

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if self.errorState:
            return None

        if self.opts['api_key'] == "":
            self.sf.error("You enabled sfp_numverify but did not set an API key!", False)
            self.errorState = True
            return None

        if eventData in self.results:
            return None

        self.results[eventData] = True

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        data = self.query(eventData)

        if data is None:
            self.sf.debug("No phone information found for " + eventData)
            return None

        evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
        self.notifyListeners(evt)

        if data.get('country_code') is not None:
            if data.get('location') == '':
                location = data.get('country_code')
            else:
                location = data.get('location') + ', ' + data.get('country_code')

            evt = SpiderFootEvent("GEOINFO", location, self.__name__, event)
            self.notifyListeners(evt)
        else:
            self.sf.debug("No location information found for " + eventData)

        if data.get('carrier') is not None:
            evt = SpiderFootEvent("PROVIDER_TELCO", data.get('carrier'), self.__name__, event)
            self.notifyListeners(evt)
        else:
            self.sf.debug("No carrier information found for " + eventData)

# End of sfp_numverify class
<commit_msg>Use string join<commit_after>#-------------------------------------------------------------------------------
# Name:        sfp_numverify
# Purpose:     SpiderFoot plug-in to search numverify.com API for a phone number
#              and retrieve location and carrier information.
#
# Author:      <bcoles@gmail.com>
#
# Created:     2019-05-25
# Copyright:   (c) bcoles 2019
# Licence:     GPL
#-------------------------------------------------------------------------------

import json
import re
import urllib
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_numverify(SpiderFootPlugin):
    """numverify:Footprint,Investigate,Passive:Real World:apikey:Lookup phone number location and carrier information from numverify.com."""

    # Default options
    opts = {
        'api_key': ''
    }

    # Option descriptions
    optdescs = {
        'api_key': 'numverify API key.'
    }

    results = dict()
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.__dataSource__ = "numverify"
        self.results = dict()
        self.errorState = False

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['PHONE_NUMBER']

    # What events this module produces
    def producedEvents(self):
        return ['RAW_RIR_DATA', 'GEOINFO', 'PROVIDER_TELCO']

    # Query numverify API for the specified phone number
    # https://numverify.com/documentation
    def query(self, qry):
        number = qry.strip('+').strip('(').strip(')')

        params = {
            'number': number.encode('raw_unicode_escape'),
            'country_code': '',
            'format': '0', # set to "1" for prettified debug output
            'access_key': self.opts['api_key']
        }

        # Free API does not support HTTPS for no adequately explained reason
        res = self.sf.fetchUrl("http://apilayer.net/api/validate?" + urllib.urlencode(params),
                               timeout=self.opts['_fetchtimeout'],
                               useragent=self.opts['_useragent'])

        time.sleep(1)

        if res['content'] is None:
            self.sf.debug('No response from apilayer.net')
            return None

        if res['code'] == '101':
            self.sf.error('API error: invalid API key', False)
            self.errorState = True
            return None

        if res['code'] == '102':
            self.sf.error('API error: user account deactivated', False)
            self.errorState = True
            return None

        if res['code'] == '104':
            self.sf.error('API error: usage limit exceeded', False)
            self.errorState = True
            return None

        try:
            data = json.loads(res['content'])
        except BaseException as e:
            self.sf.debug('Error processing JSON response: ' + str(e))
            return None

        if data.get('error') is not None:
            self.sf.error('API error: ' + str(data.get('error')), False)
            return None

        return data

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if self.errorState:
            return None

        if self.opts['api_key'] == "":
            self.sf.error("You enabled sfp_numverify but did not set an API key!", False)
            self.errorState = True
            return None

        if eventData in self.results:
            return None

        self.results[eventData] = True

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        data = self.query(eventData)

        if data is None:
            self.sf.debug("No phone information found for " + eventData)
            return None

        evt = SpiderFootEvent("RAW_RIR_DATA", str(data), self.__name__, event)
        self.notifyListeners(evt)

        if data.get('country_code') is not None:
            location = ', '.join(filter(None, [data.get('location'), data.get('country_code')]))
            evt = SpiderFootEvent("GEOINFO", location, self.__name__, event)
            self.notifyListeners(evt)
        else:
            self.sf.debug("No location information found for " + eventData)

        if data.get('carrier') is not None:
            evt = SpiderFootEvent("PROVIDER_TELCO", data.get('carrier'), self.__name__, event)
            self.notifyListeners(evt)
        else:
            self.sf.debug("No carrier information found for " + eventData)

# End of sfp_numverify class
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_tool_nmap
# Purpose:      SpiderFoot plug-in for using nmap to perform OS fingerprinting.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     03/05/2020
# Copyright:   (c) Steve Micallef 2020
# Licence:     GPL
# -------------------------------------------------------------------------------

import os.path
from subprocess import PIPE, Popen

from netaddr import IPNetwork

from spiderfoot import SpiderFootEvent, SpiderFootPlugin

class sfp_tool_nmap(SpiderFootPlugin):

    meta = {
        'name': "Tool - Nmap",
        'summary': "Identify what Operating System might be used.",
        'flags': ["tool", "slow", "invasive"],
        'useCases': ["Footprint", "Investigate"],
        'categories': ["Crawling and Scanning"],
        'toolDetails': {
            'name': "Nmap",
            'description': "Nmap (\"Network Mapper\") is a free and open source utility for network discovery and security auditing.\n"
            "Nmap uses raw IP packets in novel ways to determine what hosts are available on the network, "
            "what services (application name and version) those hosts are offering, "
            "what operating systems (and OS versions) they are running, "
            "what type of packet filters/firewalls are in use, and dozens of other characteristics.\n",
            'website': "https://nmap.org/",
            'repository': "https://svn.nmap.org/nmap"
        },
    }

    # Default options
    opts = {
        'nmappath': "",
        'netblockscan': True,
        'netblockscanmax': 24
    }

    # Option descriptions
    optdescs = {
        'nmappath': "Path to the where the nmap binary lives. Must be set.",
        'netblockscan': "Port scan all IPs within identified owned netblocks?",
        'netblockscanmax': "Maximum netblock/subnet size to scan IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"
    }

    results = None
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()
        self.errorState = False
        self.__dataSource__ = "Target Network"

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['IP_ADDRESS', 'NETBLOCK_OWNER']

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["OPERATING_SYSTEM", "IP_ADDRESS"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if srcModuleName == "sfp_tool_nmap":
            self.debug("Skipping event from myself.")
            return

        self.debug(f"Received event, {eventName}, from {srcModuleName}")

        if self.errorState:
            return

        try:
            if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
                net = IPNetwork(eventData)
                if net.prefixlen < self.opts['netblockscanmax']:
                    self.debug("Skipping port scanning of " + eventData + ", too big.")
                    return

        except Exception as e:
            self.error("Strange netblock identified, unable to parse: " + eventData + " (" + str(e) + ")")
            return

        # Don't look up stuff twice, check IP == IP here
        if eventData in self.results:
            self.debug("Skipping " + eventData + " as already scanned.")
            return
        else:
            # Might be a subnet within a subnet or IP within a subnet
            for addr in self.results:
                if IPNetwork(eventData) in IPNetwork(addr):
                    self.debug("Skipping " + eventData + " as already within a scanned range.")
                    return

        self.results[eventData] = True

        if not self.opts['nmappath']:
            self.error("You enabled sfp_tool_nmap but did not set a path to the tool!")
            self.errorState = True
            return

        # Normalize path
        if self.opts['nmappath'].endswith('nmap'):
            exe = self.opts['nmappath']
        elif self.opts['nmappath'].endswith('/'):
            exe = self.opts['nmappath'] + "nmap"
        else:
            self.error("Could not recognize your nmap path configuration.")
            self.errorState = True

        # If tool is not found, abort
        if not os.path.isfile(exe):
            self.error("File does not exist: " + exe)
            self.errorState = True
            return

        # Sanitize domain name.
        if not self.sf.validIP(eventData) and not self.sf.validIpNetwork(eventData):
            self.error("Invalid input, refusing to run.")
            return

        try:
            p = Popen([exe, "-O", "--osscan-limit", eventData], stdout=PIPE, stderr=PIPE)
            stdout, stderr = p.communicate(input=None)
            if p.returncode == 0:
                content = stdout.decode('utf-8', errors='replace')
            else:
                self.error("Unable to read Nmap content.")
                self.debug(f"Error running Nmap: {stderr}, {stdout}")
                return

            if "No exact OS matches for host" in content or "OSScan results may be unreliable" in content:
                self.debug(f"Couldn't reliably detect the OS for {eventData}")
                return
        except Exception as e:
            self.error(f"Unable to run Nmap: {e}")
            return

        if not content:
            self.debug("No content from Nmap to parse.")
            return

        if eventName == "IP_ADDRESS":
            try:
                opsys = None
                for line in content.split('\n'):
                    if "OS details:" in line:
                        junk, opsys = line.split(": ")
                if opsys:
                    evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, event)
                    self.notifyListeners(evt)
            except Exception as e:
                self.error("Couldn't parse the output of Nmap: " + str(e))
                return

        if eventName == "NETBLOCK_OWNER":
            try:
                currentIp = None
                for line in content.split('\n'):
                    opsys = None
                    if "scan report for" in line:
                        currentIp = line.split("(")[1].replace(")", "")
                    if "OS details:" in line:
                        junk, opsys = line.split(": ")

                    if opsys and currentIp:
                        ipevent = SpiderFootEvent("IP_ADDRESS", currentIp, self.__name__, event)
                        self.notifyListeners(ipevent)

                        evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, ipevent)
                        self.notifyListeners(evt)
                        currentIp = None
            except Exception as e:
                self.error(f"Couldn't parse the output of Nmap: {e}")
                return

# End of sfp_tool_nmap class
<commit_msg>sfp_tool_nmap: Return on being unable to locate the binary.<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_tool_nmap
# Purpose:      SpiderFoot plug-in for using nmap to perform OS fingerprinting.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     03/05/2020
# Copyright:   (c) Steve Micallef 2020
# Licence:     GPL
# -------------------------------------------------------------------------------

import os.path
from subprocess import PIPE, Popen

from netaddr import IPNetwork

from spiderfoot import SpiderFootEvent, SpiderFootPlugin

class sfp_tool_nmap(SpiderFootPlugin):

    meta = {
        'name': "Tool - Nmap",
        'summary': "Identify what Operating System might be used.",
        'flags': ["tool", "slow", "invasive"],
        'useCases': ["Footprint", "Investigate"],
        'categories': ["Crawling and Scanning"],
        'toolDetails': {
            'name': "Nmap",
            'description': "Nmap (\"Network Mapper\") is a free and open source utility for network discovery and security auditing.\n"
            "Nmap uses raw IP packets in novel ways to determine what hosts are available on the network, "
            "what services (application name and version) those hosts are offering, "
            "what operating systems (and OS versions) they are running, "
            "what type of packet filters/firewalls are in use, and dozens of other characteristics.\n",
            'website': "https://nmap.org/",
            'repository': "https://svn.nmap.org/nmap"
        },
    }

    # Default options
    opts = {
        'nmappath': "",
        'netblockscan': True,
        'netblockscanmax': 24
    }

    # Option descriptions
    optdescs = {
        'nmappath': "Path to the where the nmap binary lives. Must be set.",
        'netblockscan': "Port scan all IPs within identified owned netblocks?",
        'netblockscanmax': "Maximum netblock/subnet size to scan IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"
    }

    results = None
    errorState = False

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = self.tempStorage()
        self.errorState = False
        self.__dataSource__ = "Target Network"

        for opt in list(userOpts.keys()):
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ['IP_ADDRESS', 'NETBLOCK_OWNER']

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["OPERATING_SYSTEM", "IP_ADDRESS"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if srcModuleName == "sfp_tool_nmap":
            self.debug("Skipping event from myself.")
            return

        self.debug(f"Received event, {eventName}, from {srcModuleName}")

        if self.errorState:
            return

        try:
            if eventName == "NETBLOCK_OWNER" and self.opts['netblockscan']:
                net = IPNetwork(eventData)
                if net.prefixlen < self.opts['netblockscanmax']:
                    self.debug("Skipping port scanning of " + eventData + ", too big.")
                    return

        except Exception as e:
            self.error("Strange netblock identified, unable to parse: " + eventData + " (" + str(e) + ")")
            return

        # Don't look up stuff twice, check IP == IP here
        if eventData in self.results:
            self.debug("Skipping " + eventData + " as already scanned.")
            return
        else:
            # Might be a subnet within a subnet or IP within a subnet
            for addr in self.results:
                if IPNetwork(eventData) in IPNetwork(addr):
                    self.debug("Skipping " + eventData + " as already within a scanned range.")
                    return

        self.results[eventData] = True

        if not self.opts['nmappath']:
            self.error("You enabled sfp_tool_nmap but did not set a path to the tool!")
            self.errorState = True
            return

        # Normalize path
        if self.opts['nmappath'].endswith('nmap'):
            exe = self.opts['nmappath']
        elif self.opts['nmappath'].endswith('/'):
            exe = self.opts['nmappath'] + "nmap"
        else:
            self.error("Could not recognize your nmap path configuration.")
            self.errorState = True
            return

        # If tool is not found, abort
        if not os.path.isfile(exe):
            self.error("File does not exist: " + exe)
            self.errorState = True
            return

        # Sanitize domain name.
        if not self.sf.validIP(eventData) and not self.sf.validIpNetwork(eventData):
            self.error("Invalid input, refusing to run.")
            return

        try:
            p = Popen([exe, "-O", "--osscan-limit", eventData], stdout=PIPE, stderr=PIPE)
            stdout, stderr = p.communicate(input=None)
            if p.returncode == 0:
                content = stdout.decode('utf-8', errors='replace')
            else:
                self.error("Unable to read Nmap content.")
                self.debug(f"Error running Nmap: {stderr}, {stdout}")
                return

            if "No exact OS matches for host" in content or "OSScan results may be unreliable" in content:
                self.debug(f"Couldn't reliably detect the OS for {eventData}")
                return
        except Exception as e:
            self.error(f"Unable to run Nmap: {e}")
            return

        if not content:
            self.debug("No content from Nmap to parse.")
            return

        if eventName == "IP_ADDRESS":
            try:
                opsys = None
                for line in content.split('\n'):
                    if "OS details:" in line:
                        junk, opsys = line.split(": ")
                if opsys:
                    evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, event)
                    self.notifyListeners(evt)
            except Exception as e:
                self.error("Couldn't parse the output of Nmap: " + str(e))
                return

        if eventName == "NETBLOCK_OWNER":
            try:
                currentIp = None
                for line in content.split('\n'):
                    opsys = None
                    if "scan report for" in line:
                        currentIp = line.split("(")[1].replace(")", "")
                    if "OS details:" in line:
                        junk, opsys = line.split(": ")

                    if opsys and currentIp:
                        ipevent = SpiderFootEvent("IP_ADDRESS", currentIp, self.__name__, event)
                        self.notifyListeners(ipevent)

                        evt = SpiderFootEvent("OPERATING_SYSTEM", opsys, self.__name__, ipevent)
                        self.notifyListeners(evt)
                        currentIp = None
            except Exception as e:
                self.error(f"Couldn't parse the output of Nmap: {e}")
                return

# End of sfp_tool_nmap class
<|endoftext|>


<commit_before>import unittest

class TestLMtests(unittest.TestCase):
    def test___init__(self):
        # l_mtests = LMtests(ols, w, tests)
        assert False # TODO: implement your test here

class TestMoranRes(unittest.TestCase):
    def test___init__(self):
        # moran_res = MoranRes(ols, w, z)
        assert False # TODO: implement your test here

class TestAKtest(unittest.TestCase):
    def test___init__(self):
        # a_ktest = AKtest(iv, w, case)
        assert False # TODO: implement your test here

class TestSpDcache(unittest.TestCase):
    def TestAB(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.AB())
        assert False # TODO: implement your test here

    def test___init__(self):
        # sp_dcache = spDcache(reg, w)
        assert False # TODO: implement your test here

    def test_j(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.j())
        assert False # TODO: implement your test here

    def test_t(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.t())
        assert False # TODO: implement your test here

    def test_trA(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.trA())
        assert False # TODO: implement your test here

    def test_utwuDs(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.utwuDs())
        assert False # TODO: implement your test here

    def test_utwyDs(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.utwyDs())
        assert False # TODO: implement your test here

    def test_wu(self):
        # sp_dcache = spDcache(reg, w)
        # self.assertEqual(expected, sp_dcache.wu())
        assert False # TODO: implement your test here

class TestLmErr(unittest.TestCase):
    def test_lm_err(self):
        # self.assertEqual(expected, lmErr(reg, w, spDcache))
        assert False # TODO: implement your test here

class TestLmLag(unittest.TestCase):
    def test_lm_lag(self):
        # self.assertEqual(expected, lmLag(ols, w, spDcache))
        assert False # TODO: implement your test here

class TestRlmErr(unittest.TestCase):
    def test_rlm_err(self):
        # self.assertEqual(expected, rlmErr(ols, w, spDcache))
        assert False # TODO: implement your test here

class TestRlmLag(unittest.TestCase):
    def test_rlm_lag(self):
        # self.assertEqual(expected, rlmLag(ols, w, spDcache))
        assert False # TODO: implement your test here

class TestLmSarma(unittest.TestCase):
    def test_lm_sarma(self):
        # self.assertEqual(expected, lmSarma(ols, w, spDcache))
        assert False # TODO: implement your test here

class TestGetMI(unittest.TestCase):
    def test_get_m_i(self):
        # self.assertEqual(expected, get_mI(reg, w, spDcache))
        assert False # TODO: implement your test here

class TestGetVI(unittest.TestCase):
    def test_get_v_i(self):
        # self.assertEqual(expected, get_vI(ols, w, ei, spDcache))
        assert False # TODO: implement your test here

class TestGetEI(unittest.TestCase):
    def test_get_e_i(self):
        # self.assertEqual(expected, get_eI(ols, w, spDcache))
        assert False # TODO: implement your test here

class TestGetZI(unittest.TestCase):
    def test_get_z_i(self):
        # self.assertEqual(expected, get_zI(I, ei, vi))
        assert False # TODO: implement your test here

class TestAkTest(unittest.TestCase):
    def test_ak_test(self):
        # self.assertEqual(expected, akTest(iv, w, spDcache))
        assert False # TODO: implement your test here

if __name__ == '__main__':
    unittest.main()
<commit_msg>Adding unit tests for diagnostics_sp.<commit_after>import unittest
import numpy as np
import pysal
from pysal.spreg import diagnostics
from pysal.spreg.ols import BaseOLS as OLS
from pysal.spreg.twosls import BaseTSLS as TSLS
from pysal.spreg.twosls_sp import GM_Lag
from pysal.spreg.diagnostics_sp import LMtests, MoranRes, spDcache, AKtest

class TestLMtests(unittest.TestCase):
    def setUp(self):
        db = pysal.open(pysal.examples.get_path("columbus.dbf"),"r")
        y = np.array(db.by_col("HOVAL"))
        y = np.reshape(y, (49,1))
        X = []
        X.append(db.by_col("INC"))
        X.append(db.by_col("CRIME"))
        X = np.array(X).T
        self.y = y
        self.X = X
        ols = OLS(self.y, self.X)
        self.ols = ols
        w = pysal.open(pysal.examples.get_path('columbus.gal'), 'r').read()
        w.transform='r'
        self.w = w

    def test_lm_err(self):
        lms = LMtests(self.ols, self.w)
        lme = np.array([3.097094,  0.078432])
        np.testing.assert_array_almost_equal(lms.lme, lme, decimal=6)

    def test_lm_lag(self):
        lms = LMtests(self.ols, self.w)
        lml = np.array([ 0.981552,  0.321816])
        np.testing.assert_array_almost_equal(lms.lml, lml, decimal=6)

    def test_rlm_err(self):
        lms = LMtests(self.ols, self.w)
        rlme = np.array([ 3.209187,  0.073226])
        np.testing.assert_array_almost_equal(lms.rlme, rlme, decimal=6)

    def test_rlm_lag(self):
        lms = LMtests(self.ols, self.w)
        rlml = np.array([ 1.093645,  0.295665])
        np.testing.assert_array_almost_equal(lms.rlml, rlml, decimal=6)

    def test_lm_sarma(self):
        lms = LMtests(self.ols, self.w)
        sarma = np.array([ 4.190739,  0.123025])
        np.testing.assert_array_almost_equal(lms.sarma, sarma, decimal=6)

class TestMoranRes(unittest.TestCase):
    def setUp(self):
        db = pysal.open(pysal.examples.get_path("columbus.dbf"),"r")
        y = np.array(db.by_col("HOVAL"))
        y = np.reshape(y, (49,1))
        X = []
        X.append(db.by_col("INC"))
        X.append(db.by_col("CRIME"))
        X = np.array(X).T
        self.y = y
        self.X = X
        ols = OLS(self.y, self.X)
        self.ols = ols
        w = pysal.open(pysal.examples.get_path('columbus.gal'), 'r').read()
        w.transform='r'
        self.w = w

    def test_get_m_i(self):
        m = MoranRes(self.ols, self.w, z=True)
        np.testing.assert_array_almost_equal(m.I, 0.17130999999999999, decimal=6)

    def test_get_v_i(self):
        m = MoranRes(self.ols, self.w, z=True)
        np.testing.assert_array_almost_equal(m.vI, 0.0081300000000000001, decimal=6)

    def test_get_e_i(self):
        m = MoranRes(self.ols, self.w, z=True)
        np.testing.assert_array_almost_equal(m.eI, -0.034522999999999998, decimal=6)

    def test_get_z_i(self):
        m = MoranRes(self.ols, self.w, z=True)
        np.testing.assert_array_almost_equal(m.zI, 2.2827389999999999, decimal=6)

class TestAKTest(unittest.TestCase):
    def setUp(self):
        db = pysal.open(pysal.examples.get_path("columbus.dbf"),'r')
        y = np.array(db.by_col("CRIME"))
        y = np.reshape(y, (49,1))
        self.y = y
        X = []
        X.append(db.by_col("INC"))
        X = np.array(X).T
        self.X = X
        yd = []
        yd.append(db.by_col("HOVAL"))
        yd = np.array(yd).T
        self.yd = yd
        q = []
        q.append(db.by_col("DISCBD"))
        q = np.array(q).T
        self.q = q
        reg = TSLS(y, X, yd, q=q)
        self.reg = reg
        w = pysal.rook_from_shapefile(pysal.examples.get_path("columbus.shp"))
        w.transform = 'r'
        self.w = w

    def test_gen_mi(self):
        ak = AKtest(self.reg, self.w)
        np.testing.assert_array_almost_equal(ak.mi, 0.2232672865437263, decimal=6)

    def test_gen_ak(self):
        ak = AKtest(self.reg, self.w)
        np.testing.assert_array_almost_equal(ak.ak, 4.6428948758930852, decimal=6)

    def test_gen_p(self):
        ak = AKtest(self.reg, self.w)
        np.testing.assert_array_almost_equal(ak.p, 0.031182360054340875, decimal=6)

    def test_sp_mi(self):
        ak = AKtest(self.reg, self.w, case='gen')
        np.testing.assert_array_almost_equal(ak.mi, 0.2232672865437263, decimal=6)

    def test_sp_ak(self):
        ak = AKtest(self.reg, self.w,case='gen')
        np.testing.assert_array_almost_equal(ak.ak, 1.1575928784397795, decimal=6)

    def test_sp_p(self):
        ak = AKtest(self.reg, self.w, case='gen')
        np.testing.assert_array_almost_equal(ak.p, 0.28196531619791054, decimal=6)

class TestSpDcache(unittest.TestCase):
    def setUp(self):
        db = pysal.open(pysal.examples.get_path("columbus.dbf"),"r")
        y = np.array(db.by_col("HOVAL"))
        y = np.reshape(y, (49,1))
        X = []
        X.append(db.by_col("INC"))
        X.append(db.by_col("CRIME"))
        X = np.array(X).T
        self.y = y
        self.X = X
        ols = OLS(self.y, self.X)
        self.ols = ols
        w = pysal.open(pysal.examples.get_path('columbus.gal'), 'r').read()
        w.transform='r'
        self.w = w

    def test_j(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.j[0][0], 0.62330311259039439, decimal=6)

    def test_t(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.t, 22.751186696900984, decimal=6)

    def test_trA(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.trA, 1.5880426389276328, decimal=6)

    def test_utwuDs(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.utwuDs[0][0], 8.3941977502916068, decimal=6)

    def test_utwyDs(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.utwyDs[0][0], 5.475255215067957, decimal=6)

    def test_wu(self):
        cache = spDcache(self.ols, self.w)
        np.testing.assert_array_almost_equal(cache.wu[0][0], -10.681344941514411, decimal=6)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright (c) 2018 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
"""
The core module in Craftr implements targets and their properties independent
from the aspect of Craftr DSL.
"""

from nr.stream import stream
from nr.datastructures.mappings import ObjectFromMapping

import collections
import warnings

import {Action, BuildGraph, FileSet} from './build'
import proplib from './proplib'

class Context:
  """
  The context contains global information for the build system.
  """

  def __init__(self):
    self.module_properties = proplib.PropertySet(allow_any=True)
    Module.setup_standard_properties(self.module_properties)
    self.target_properties = proplib.PropertySet()
    Target.setup_standard_properties(self.target_properties)
    self.dependency_properties = proplib.PropertySet()
    Dependency.setup_standard_properties(self.dependency_properties)
    self.handlers = []
    self.modules = []
    self.graph = BuildGraph()

  def register_handler(self, handler):
    handler.init(self)
    self.handlers.append(handler)

  def translate_targets(self):
    """
    Invokes the translation process for all targets in all modules in the
    Context.
    """

    for handler in self.handlers:
      handler.translate_begin()

    seen = set()
    def handle_target(target):
      if target in seen:
        return False
      seen.add(target)
      for handler in self.handlers:
        handler.translate_target(target)
      # Check the layers of the target -- handling them may create new layers.
      nextround = True
      while nextround:
        nextround = False
        for layer in target.iter_layers():
          if handle_target(layer):
            nextround = True
      return True
    def translate(target):
      for dep in target.dependencies:
        for other_target in dep.sources:
          translate(other_target)
      handle_target(target)

    for module in self.modules:
      for target in module.targets.values():
        translate(target)

    for handler in self.handlers:
      handler.translate_end()

    def add_target_actions(target):
      self.graph.add_actions(target.actions.values())
      for layer in target.iter_layers():
        add_target_actions(layer)

    for module in self.modules:
      for target in module.targets.values():
        add_target_actions(target)

class Module:
  """
  Represents a namespace for targets. Every build script in the Craftr DSL
  has its own namespace, which must be explicitly declared with the `namespace`
  keyword. Additionally, a version number may be declared which can be used
  by target handlers for default filenames, etc. Every namespace is also
  associated with a directory that all paths, if they are not absolute, should
  be treated relative to.
  """

  @staticmethod
  def setup_standard_properties(props):
    pass

  def __init__(self, context, name, version, directory):
    self.context = context
    self.name = name
    self.version = version
    self.directory = directory
    self.targets = collections.OrderedDict()
    self.pools = collections.OrderedDict()
    self.props = proplib.Properties(context.module_properties, self)

  def __repr__(self):
    return 'Module(name={!r}, version={!r}, directory={!r})'.format(
      self.name, self.version, self.directory)

  def add_target_with_class(self, target_cls, name, public=False):
    if name in self.targets:
      raise ValueError('target name {!r} already occupied'.format(name))
    target = target_cls(self, name, public)
    self.targets[target.name] = target
    for handler in self.context.handlers:
      handler.target_created(target)
    return target

  def add_target(self, *args, **kwargs):
    return self.add_target_with_class(Target, *args, **kwargs)

  def add_pool(self, name, depth):
    if name in self.pools:
      raise ValueError('pool {!r} already defined'.format(name))
    self.pools[name] = depth

  def public_targets(self):
    return (x for x in self.targets.values() if x .public)

class Target:
  """
  A target represents a collection of properties and dependencies to other
  targets. Targets will be translated into build actions by target handlers
  based on their properties and dependencies.
  """

  @staticmethod
  def setup_standard_properties(propset):
    propset.add('this.directory', proplib.String)
    propset.add('this.outputDirectory', proplib.String)
    propset.add('this.pool', proplib.String)
    propset.add('this.explicit', proplib.Bool, default=False)
    propset.add('this.syncio', proplib.Bool, default=False)

  def __init__(self, module, name, public, parent=None, redirect_actions=False):
    if parent:
      assert module == parent.module, "differing modules"
    if redirect_actions and not parent:
      raise ValueError('"redirect_actions" can not be True without "parent"')
    self.parent = parent
    self.redirect_actions = redirect_actions
    self.module = module
    self.name = name
    self.public = public
    self.props = proplib.Properties(module.context.target_properties, self)
    self.exported_props = proplib.Properties(module.context.target_properties, self)
    self.dependencies = []
    self.actions = collections.OrderedDict()
    self.layers = {}
    self._layers_list = []  # Can be appended to during iteration.

  def __repr__(self):
    return 'Target(id={!r}, public={!r})'.format(self.id, self.public)

  @property
  def context(self):
    return self.module.context

  @property
  def id(self):
    return '{}@{}'.format(self.module.name, self.full_name)

  @property
  def full_name(self):
    if self.parent:
      return self.parent.full_name + '/' + self.name
    return self.name

  @property
  def directory(self):
    result = self.get_prop('this.directory', default=None)
    if result is None:
      if self.parent:
        result = self.parent.directory
      else:
        result = self.module.directory
    return result

  @property
  def output_actions(self):
    """
    Returns a list for the target's output actions. If there is at least
    one action that is explicitly marked as output, only actions that are
    marked as such will be returned. Otherwise, the last action that was
    created and not explicitly marked as NO output will be returned.
    """

    result = []
    last_default_output = None
    for action in self.actions.values():
      if action.is_output:
        result.append(action)
      elif action.is_output is None:
        last_default_output = action
    if not result and last_default_output:
      result.append(last_default_output)
    return result

  def get_prop(self, prop_name, inherit=False, default=NotImplemented):
    """
    Returns a property value, preferring the value in the #exported_props.

    If *inherit* is #True, the property must be a #proplib.List property
    and the values in the exported and non-exported property containers as
    well as transitive dependencies are respected.
    """

    if inherit:
      def iter_values():
        yield self.exported_props[prop_name]
        yield self.props[prop_name]
        for target in self.transitive_dependencies().attr('sources').concat():
          yield target.exported_props[prop_name]
      prop = self.context.target_properties[prop_name]
      return prop.type.inherit(prop_name, iter_values())
    else:
      if self.exported_props.is_set(prop_name):
        return self.exported_props[prop_name]
      elif self.props.is_set(prop_name):
        return self.props[prop_name]
      elif default is NotImplemented:
        return self.context.target_properties[prop_name].get_default()
      else:
        return default

  def get_props(self, prefix='', as_object=False):
    """
    Returns a dictionary that contains all property values, optionally from
    the specified prefix.
    """

    result = {}
    propset = self.context.target_properties
    for prop in filter(lambda x: x.name.startswith(prefix), propset.values()):
      inherit = prop.options.get('inherit', False)
      value = self.get_prop(prop.name, inherit)
      result[prop.name[len(prefix):]] = value

    if as_object:
      result = ObjectFromMapping(result)

    return result

  def transitive_dependencies(self):
    def worker(target, private=False):
      for dep in target.dependencies:
        if dep.public or private:
          yield dep
        for t in dep.sources:
          yield from worker(t)
    return stream.unique(worker(self, private=True))

  def add_dependency_with_class(self, dependency_cls, sources, public=False):
    dep = dependency_cls(self, sources, public)
    self.dependencies.append(dep)
    return dep

  def add_dependency(self, *args, **kwargs):
    return self.add_dependency_with_class(Dependency, *args, **kwargs)

  def add_action(self, name=None, *, input=None, output=None, deps=None, **kwargs):
    """
    Creates a new action in the target that consists of one or more system
    commands. Unless otherwise explicitly set with the *input* or *deps*
    parameters, the first action that is being created for a target will have
    the *input* parameter default to #True, in which case it will be connected
    to all outputs of the dependencies of this target.

    Dependencies can also be specified explicitly by passing a list of #Action
    objects to the *deps* parameter.

    Otherwise, actions created after the first will receive the previously
    created action as dependency.

    Passing #True for *output* will mark the action as an output action,
    which will be connected with the actions generated by the dependents of
    this target (unless they explicitly specifiy the dependencies).
    To explicitly mark an action as NO output action, set *output* to #False.

    All other arguments are forwarded to the #Action constructor.
    """

    if self.redirect_actions:
      return self.parent(name, input=input, output=output, deps=deps, **kwargs)

    if name is None:
      # TODO: Automatically add the name of the program in the first command.
      name = str(len(self.actions))

    if name in self.actions:
      raise ValueError('action name already used: {!r}'.format(name))

    if input is None:
      input = not self.actions
    if deps is None:
      deps_was_unset = True
      deps = []
    else:
      deps_was_unset = False
      deps = list(deps)
    if input:
      for dep in self.transitive_dependencies():
        for target in dep.sources:
          deps += target.output_actions
    elif deps_was_unset and self.actions:
      output_actions = self.output_actions
      if output_actions:
        deps.append(output_actions[-1])

    # Filter 'None' values from the actions list.
    deps = [x for x in deps if x is not None]

    # TODO: Assign the action to the pool specified in the target.
    kwargs.setdefault('explicit', self.get_prop('this.explicit'))
    kwargs.setdefault('syncio', self.get_prop('this.syncio'))
    action = Action(self.id, name, deps=deps, **kwargs)
    self.actions[name] = action
    action.is_output = output
    return action

  def actions_and_files_tagged(self, tags, transitive=False):
    """
    Returns a tuple of (actions, files) where *actions* is a list of actions
    in the target that have at least one buildset where the specified *tags*
    match, and *files* is a list of the files that match these tags.

    If *transitive* is #True, the actions and files of the transitive
    dependencies are returned instead.
    """

    if self.redirect_actions:
      return self.parent.actions_and_files_tagged(tags, transitive)

    if isinstance(tags, str):
      tags = [x.strip() for x in tags.split(',')]

    actions = []
    files = []
    if transitive:
      for target in self.transitive_dependencies().attr('sources').concat():
        res = target.actions_and_files_tagged(tags)
        actions += res[0]
        files += res[1]
    else:
      for action in self.actions.values():
        matched_files = action.all_files_tagged(tags)
        if matched_files:
          actions.append(action)
          files += matched_files

    return actions, files

  def files_tagged(self, tags, transitive=False):
    """
    Returns only files with the specified tags.
    """

    if self.redirect_actions:
      return self.parent.files_tagged(tags, transitive)
    return self.actions_and_files_tagged(tags, transitive)[1]

  def iter_layers(self):
    return iter(self._layers_list)

  def add_layer_with_class(self, target_cls, name, public=False, redirect_actions=False):
    if name in self.layers:
      raise ValueError('Target layer {!r} already exists'.format(name))
    target = target_cls(self.module, name, public, parent=self,
      redirect_actions=redirect_actions)
    self.layers[name] = target
    self._layers_list.append(target)
    return target

  def add_layer(self, *args, **kwargs):
    return self.add_layer_with_class(type(self), *args, **kwargs)

class Dependency:
  """
  Resembles a dependency to one or more other targets.
  """

  @staticmethod
  def setup_standard_properties(propset):
    pass

  def __init__(self, target, sources, public):
    if isinstance(sources, collections.Iterator):
      sources = list(sources)
    sources = proplib.List[proplib.InstanceOf[Target]]().coerce('sources', sources)
    self.target = target
    self.sources = sources
    self.public = public
    self.props = proplib.Properties(target.context.dependency_properties, self)

  def __repr__(self):
    return 'Dependency({!r}, {!r}, public={!r})'.format(
      self.target, self.sources, self.public)

  @property
  def context(self):
    return self.target.context

class TargetHandler:
  """
  Interface for target handlers -- the objects that are capable of taking
  target properties and their dependencies and converting them to build
  actions.
  """

  def init(self, context):
    pass

  def target_created(self, target):
    pass

  def translate_begin(self):
    pass

  def translate_target(self, target):
    pass

  def translate_end(self):
    pass
<commit_msg>add TargetHandler.preprocess_target(), Context.iter_targets() and Context.iter_targets_additive()<commit_after># Copyright (c) 2018 Niklas Rosenstein
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
"""
The core module in Craftr implements targets and their properties independent
from the aspect of Craftr DSL.
"""

from nr.stream import stream
from nr.datastructures.mappings import ObjectFromMapping

import collections
import warnings

import {Action, BuildGraph, FileSet} from './build'
import proplib from './proplib'

class Context:
  """
  The context contains global information for the build system.
  """

  def __init__(self):
    self.module_properties = proplib.PropertySet(allow_any=True)
    Module.setup_standard_properties(self.module_properties)
    self.target_properties = proplib.PropertySet()
    Target.setup_standard_properties(self.target_properties)
    self.dependency_properties = proplib.PropertySet()
    Dependency.setup_standard_properties(self.dependency_properties)
    self.handlers = []
    self.modules = []
    self.graph = BuildGraph()

  def register_handler(self, handler):
    handler.init(self)
    self.handlers.append(handler)

  def iter_targets(self):
    """
    Iterates over all targets in the context in post-order.
    """

    seen = set()

    def recurse_target(target):
      for dep in target.transitive_dependencies().attr('sources').concat():
        if dep not in seen:
          seen.add(dep)
          yield dep
          yield from recurse_layers(dep)
          yield from recurse_target(dep)

    def recurse_layers(target):
      for layer in target.iter_layers():
        if layer not in seen:
          seen.add(layer)
          yield from recurse_target(layer)
          yield layer

    for module in self.modules:
      for target in module.targets.values():
        yield from recurse_target(target)
        yield target
        yield from recurse_layers(target)

  def iter_targets_additive(self):
    """
    Iterates over all targets in the context in post-order, until no more new
    targets are created.
    """

    seen = set()
    has_new_targets = True
    while has_new_targets:
      has_new_targets = False
      for target in self.iter_targets():
        if target not in seen:
          seen.add(target)
          yield target
          has_new_targets = True

  def translate_targets(self):
    """
    Invokes the translation process for all targets in all modules in the
    Context.
    """

    for handler in self.handlers:
      handler.translate_begin()

    for target in self.iter_targets_additive():
      for handler in self.handlers:
        handler.preprocess_target(target)
    for target in self.iter_targets_additive():
      for handler in self.handlers:
        handler.translate_target(target)

    for handler in self.handlers:
      handler.translate_end()

    def add_target_actions(target):
      self.graph.add_actions(target.actions.values())
      for layer in target.iter_layers():
        add_target_actions(layer)

    for module in self.modules:
      for target in module.targets.values():
        add_target_actions(target)

class Module:
  """
  Represents a namespace for targets. Every build script in the Craftr DSL
  has its own namespace, which must be explicitly declared with the `namespace`
  keyword. Additionally, a version number may be declared which can be used
  by target handlers for default filenames, etc. Every namespace is also
  associated with a directory that all paths, if they are not absolute, should
  be treated relative to.
  """

  @staticmethod
  def setup_standard_properties(props):
    pass

  def __init__(self, context, name, version, directory):
    self.context = context
    self.name = name
    self.version = version
    self.directory = directory
    self.targets = collections.OrderedDict()
    self.pools = collections.OrderedDict()
    self.props = proplib.Properties(context.module_properties, self)

  def __repr__(self):
    return 'Module(name={!r}, version={!r}, directory={!r})'.format(
      self.name, self.version, self.directory)

  def add_target_with_class(self, target_cls, name, public=False):
    if name in self.targets:
      raise ValueError('target name {!r} already occupied'.format(name))
    target = target_cls(self, name, public)
    self.targets[target.name] = target
    for handler in self.context.handlers:
      handler.target_created(target)
    return target

  def add_target(self, *args, **kwargs):
    return self.add_target_with_class(Target, *args, **kwargs)

  def add_pool(self, name, depth):
    if name in self.pools:
      raise ValueError('pool {!r} already defined'.format(name))
    self.pools[name] = depth

  def public_targets(self):
    return (x for x in self.targets.values() if x .public)

class Target:
  """
  A target represents a collection of properties and dependencies to other
  targets. Targets will be translated into build actions by target handlers
  based on their properties and dependencies.
  """

  @staticmethod
  def setup_standard_properties(propset):
    propset.add('this.directory', proplib.String)
    propset.add('this.outputDirectory', proplib.String)
    propset.add('this.pool', proplib.String)
    propset.add('this.explicit', proplib.Bool, default=False)
    propset.add('this.syncio', proplib.Bool, default=False)

  def __init__(self, module, name, public, parent=None, redirect_actions=False):
    if parent:
      assert module == parent.module, "differing modules"
    if redirect_actions and not parent:
      raise ValueError('"redirect_actions" can not be True without "parent"')
    self.parent = parent
    self.redirect_actions = redirect_actions
    self.module = module
    self.name = name
    self.public = public
    self.props = proplib.Properties(module.context.target_properties, self)
    self.exported_props = proplib.Properties(module.context.target_properties, self)
    self.dependencies = []
    self.actions = collections.OrderedDict()
    self.layers = {}
    self._layers_list = []  # Can be appended to during iteration.

  def __repr__(self):
    return 'Target(id={!r}, public={!r})'.format(self.id, self.public)

  @property
  def context(self):
    return self.module.context

  @property
  def id(self):
    return '{}@{}'.format(self.module.name, self.full_name)

  @property
  def full_name(self):
    if self.parent:
      return self.parent.full_name + '/' + self.name
    return self.name

  @property
  def directory(self):
    result = self.get_prop('this.directory', default=None)
    if result is None:
      if self.parent:
        result = self.parent.directory
      else:
        result = self.module.directory
    return result

  @property
  def output_actions(self):
    """
    Returns a list for the target's output actions. If there is at least
    one action that is explicitly marked as output, only actions that are
    marked as such will be returned. Otherwise, the last action that was
    created and not explicitly marked as NO output will be returned.
    """

    result = []
    last_default_output = None
    for action in self.actions.values():
      if action.is_output:
        result.append(action)
      elif action.is_output is None:
        last_default_output = action
    if not result and last_default_output:
      result.append(last_default_output)
    return result

  def get_prop(self, prop_name, inherit=False, default=NotImplemented):
    """
    Returns a property value, preferring the value in the #exported_props.

    If *inherit* is #True, the property must be a #proplib.List property
    and the values in the exported and non-exported property containers as
    well as transitive dependencies are respected.
    """

    if inherit:
      def iter_values():
        yield self.exported_props[prop_name]
        yield self.props[prop_name]
        for target in self.transitive_dependencies().attr('sources').concat():
          yield target.exported_props[prop_name]
      prop = self.context.target_properties[prop_name]
      return prop.type.inherit(prop_name, iter_values())
    else:
      if self.exported_props.is_set(prop_name):
        return self.exported_props[prop_name]
      elif self.props.is_set(prop_name):
        return self.props[prop_name]
      elif default is NotImplemented:
        return self.context.target_properties[prop_name].get_default()
      else:
        return default

  def get_props(self, prefix='', as_object=False):
    """
    Returns a dictionary that contains all property values, optionally from
    the specified prefix.
    """

    result = {}
    propset = self.context.target_properties
    for prop in filter(lambda x: x.name.startswith(prefix), propset.values()):
      inherit = prop.options.get('inherit', False)
      value = self.get_prop(prop.name, inherit)
      result[prop.name[len(prefix):]] = value

    if as_object:
      result = ObjectFromMapping(result)

    return result

  def transitive_dependencies(self):
    def worker(target, private=False):
      for dep in target.dependencies:
        if dep.public or private:
          yield dep
        for t in dep.sources:
          yield from worker(t)
    return stream.unique(worker(self, private=True))

  def add_dependency_with_class(self, dependency_cls, sources, public=False):
    dep = dependency_cls(self, sources, public)
    self.dependencies.append(dep)
    return dep

  def add_dependency(self, *args, **kwargs):
    return self.add_dependency_with_class(Dependency, *args, **kwargs)

  def add_action(self, name=None, *, input=None, output=None, deps=None, **kwargs):
    """
    Creates a new action in the target that consists of one or more system
    commands. Unless otherwise explicitly set with the *input* or *deps*
    parameters, the first action that is being created for a target will have
    the *input* parameter default to #True, in which case it will be connected
    to all outputs of the dependencies of this target.

    Dependencies can also be specified explicitly by passing a list of #Action
    objects to the *deps* parameter.

    Otherwise, actions created after the first will receive the previously
    created action as dependency.

    Passing #True for *output* will mark the action as an output action,
    which will be connected with the actions generated by the dependents of
    this target (unless they explicitly specifiy the dependencies).
    To explicitly mark an action as NO output action, set *output* to #False.

    All other arguments are forwarded to the #Action constructor.
    """

    if self.redirect_actions:
      return self.parent(name, input=input, output=output, deps=deps, **kwargs)

    if name is None:
      # TODO: Automatically add the name of the program in the first command.
      name = str(len(self.actions))

    if name in self.actions:
      raise ValueError('action name already used: {!r}'.format(name))

    if input is None:
      input = not self.actions
    if deps is None:
      deps_was_unset = True
      deps = []
    else:
      deps_was_unset = False
      deps = list(deps)
    if input:
      for dep in self.transitive_dependencies():
        for target in dep.sources:
          deps += target.output_actions
    elif deps_was_unset and self.actions:
      output_actions = self.output_actions
      if output_actions:
        deps.append(output_actions[-1])

    # Filter 'None' values from the actions list.
    deps = [x for x in deps if x is not None]

    # TODO: Assign the action to the pool specified in the target.
    kwargs.setdefault('explicit', self.get_prop('this.explicit'))
    kwargs.setdefault('syncio', self.get_prop('this.syncio'))
    action = Action(self.id, name, deps=deps, **kwargs)
    self.actions[name] = action
    action.is_output = output
    return action

  def actions_and_files_tagged(self, tags, transitive=False):
    """
    Returns a tuple of (actions, files) where *actions* is a list of actions
    in the target that have at least one buildset where the specified *tags*
    match, and *files* is a list of the files that match these tags.

    If *transitive* is #True, the actions and files of the transitive
    dependencies are returned instead.
    """

    if self.redirect_actions:
      return self.parent.actions_and_files_tagged(tags, transitive)

    if isinstance(tags, str):
      tags = [x.strip() for x in tags.split(',')]

    actions = []
    files = []
    if transitive:
      for target in self.transitive_dependencies().attr('sources').concat():
        res = target.actions_and_files_tagged(tags)
        actions += res[0]
        files += res[1]
    else:
      for action in self.actions.values():
        matched_files = action.all_files_tagged(tags)
        if matched_files:
          actions.append(action)
          files += matched_files

    return actions, files

  def files_tagged(self, tags, transitive=False):
    """
    Returns only files with the specified tags.
    """

    if self.redirect_actions:
      return self.parent.files_tagged(tags, transitive)
    return self.actions_and_files_tagged(tags, transitive)[1]

  def iter_layers(self):
    return iter(self._layers_list)

  def add_layer_with_class(self, target_cls, name, public=False, redirect_actions=False):
    if name in self.layers:
      raise ValueError('Target layer {!r} already exists'.format(name))
    target = target_cls(self.module, name, public, parent=self,
      redirect_actions=redirect_actions)
    self.layers[name] = target
    self._layers_list.append(target)
    return target

  def add_layer(self, *args, **kwargs):
    return self.add_layer_with_class(type(self), *args, **kwargs)

class Dependency:
  """
  Resembles a dependency to one or more other targets.
  """

  @staticmethod
  def setup_standard_properties(propset):
    pass

  def __init__(self, target, sources, public):
    if isinstance(sources, collections.Iterator):
      sources = list(sources)
    sources = proplib.List[proplib.InstanceOf[Target]]().coerce('sources', sources)
    self.target = target
    self.sources = sources
    self.public = public
    self.props = proplib.Properties(target.context.dependency_properties, self)

  def __repr__(self):
    return 'Dependency({!r}, {!r}, public={!r})'.format(
      self.target, self.sources, self.public)

  @property
  def context(self):
    return self.target.context

class TargetHandler:
  """
  Interface for target handlers -- the objects that are capable of taking
  target properties and their dependencies and converting them to build
  actions.
  """

  def init(self, context):
    pass

  def target_created(self, target):
    pass

  def translate_begin(self):
    pass

  def preprocess_target(self, target):
    pass

  def translate_target(self, target):
    pass

  def translate_end(self):
    pass
<|endoftext|>