<commit_before>import os
import sys
import asyncio
import pymongo
import traceback

import virtool.utils

LIST_PROJECTION = [
    "_id",
    "task",
    "status",
    "proc",
    "mem",
    "user"
]

def dispatch_processor(document):
    """
    Removes the ``status`` and ``args`` fields from the job document.
    Adds a ``username`` field, an ``added`` date taken from the first status entry in the job document, and
    ``state`` and ``progress`` fields taken from the most recent status entry in the job document.
    :param document: a document to process.
    :type document: dict

    :return: a processed documents.
    :rtype: dict
    """
    document = virtool.utils.base_processor(document)

    status = document.pop("status")

    last_update = status[-1]

    document.update({
        "state": last_update["state"],
        "stage": last_update["stage"],
        "created_at": status[0]["timestamp"],
        "progress": status[-1]["progress"]
    })

    return document

class Job:

    def __init__(self, loop, executor, db, settings, dispatch, job_id, task_name, task_args, proc, mem):
        self.loop = loop
        self.executor = executor
        self.db = db
        self.settings = settings
        self.dispatch = dispatch
        self.id = job_id
        self.task_name = task_name
        self.task_args = task_args
        self.proc = proc
        self.mem = mem

        self.started = False
        self.finished = False

        self._progress = 0
        self._state = "waiting"
        self._stage = None
        self._error = None
        self._cancelled = False
        self._task = None
        self._process_task = None
        self._stage_list = None
        self._log_path = os.path.join(self.settings.get("data_path"), "logs", "jobs", self.id)
        self._log_buffer = list()

    def start(self):
        self._task = self.loop.create_task(self.run())
        self.started = True

    async def run(self):
        for method in self._stage_list:
            name = method.__name__

            await self.add_status(stage=name, state="running")

            try:
                await self.add_log("Stage: {}".format(name))
                await method()
            except asyncio.CancelledError:
                self._cancelled = True
            except:
                self._error = handle_exception()

            if self._error or self._cancelled:
                break

        self._progress = 1

        if self._error:
            await self.add_status(state="error")
            await self.cleanup()
        elif self._cancelled:
            await self.add_status(state="cancelled")
            await self.cleanup()
        else:
            await self.add_status(state="complete")

        await self.run_in_executor(flush_log, self._log_path, self._log_buffer)

        self.finished = True

    async def run_in_executor(self, func, *args):
        await self.add_log("Process: {}".format(func.__name__))
        self._process_task = self.loop.run_in_executor(self.executor, func, *args)
        result = await self._process_task
        self._process_task = None

        return result

    async def run_subprocess(self, command, error_test=None, log_stdout=False, log_stderr=True, env=None):
        print("STARTED ADDING INIT LOG")

        await self.add_log("Command: {}".format(" ".join(command)))

        print("FINISHED ADDING INIT LOG")

        print("STARTED SETTING EVENT LOOP")

        asyncio.set_event_loop(self.loop)
        asyncio.get_child_watcher().attach_loop(self.loop)

        print("FINISHED SETTING EVENT LOOP")

        print("STARTED WAITING FOR START")

        proc = await asyncio.create_subprocess_exec(
            *command,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
            loop=self.loop,
            env=env
        )

        print("FINISHED WAITING FOR START")

        out = list()
        err = list()

        print("STARTED LISTENING TO STDOUT")

        while True:
            line = await proc.stdout.readline()

            if not line:
                break

            line = line.decode().rstrip()

            out.append(line)

            if log_stdout:
                await self.add_log(line, indent=1)

        print("FINISHED LISTENING TO STDOUT")

        print("STARTED LISTENING TO STDERR")

        while True:
            line = await proc.stderr.readline()

            if not line:
                break

            line = line.decode().rstrip()

            err.append(line)

            if log_stderr:
                await self.add_log(line, indent=1)

        print("FINISHED LISTENING TO STDERR")

        print("STARTED WAITING FOR PROC")

        await proc.wait()

        print("FINISHED WAITING FOR PROC")

        if proc.returncode != 0 or (error_test and error_test(out, err)):
            raise SubprocessError("Command failed: {}. Check job log.".format(" ".join(command)))

    async def add_status(self, state=None, stage=None):
        self._state = state or self._state
        self._stage = stage or self._stage

        if self._progress != 1:
            stage_index = [m.__name__ for m in self._stage_list].index(self._stage)
            self._progress = round((stage_index + 1) / (len(self._stage_list) + 1), 2)

        document = await self.db.jobs.find_one_and_update({"_id": self.id}, {
            "$push": {
                "status": {
                    "state": self._state,
                    "stage": self._stage,
                    "error": self._error,
                    "progress": self._progress,
                    "timestamp": virtool.utils.timestamp()
                }
            }
        }, return_document=pymongo.ReturnDocument.AFTER, projection=LIST_PROJECTION)

        await self.dispatch("jobs", "update", dispatch_processor(document))

    async def add_log(self, line, indent=0):
        timestamp = virtool.utils.timestamp().isoformat()

        self._log_buffer.append("{}{}    {}".format(timestamp, " " * indent * 4, line.rstrip()))

        if len(self._log_buffer) == 15:
            await self.flush_log()
            del self._log_buffer[:]

    async def flush_log(self):
        await self.run_in_executor(flush_log, self._log_path, self._log_buffer)

    async def cancel(self):
        if self.started and not self.finished:
            self._task.cancel()

            while not self.finished:
                await asyncio.sleep(0.1, loop=self.loop)

        await self.cleanup()

        self.finished = True

    async def cleanup(self):
        pass

def stage_method(func):
    func.is_stage_method = True
    return func

def flush_log(path, buffer):
    with open(path, "a") as handle:
        handle.write("\n".join(buffer))

def handle_exception(max_tb=50):
    exception, value, trace_info = sys.exc_info()

    return {
        "type": exception.__name__,
        "traceback": traceback.format_tb(trace_info, max_tb),
        "details": [str(l) for l in value.args]
    }

class SubprocessError(Exception):
    pass
<commit_msg>Don't explicity set event loop in run_subprocess<commit_after>import os
import sys
import asyncio
import pymongo
import traceback

import virtool.utils

LIST_PROJECTION = [
    "_id",
    "task",
    "status",
    "proc",
    "mem",
    "user"
]

def dispatch_processor(document):
    """
    Removes the ``status`` and ``args`` fields from the job document.
    Adds a ``username`` field, an ``added`` date taken from the first status entry in the job document, and
    ``state`` and ``progress`` fields taken from the most recent status entry in the job document.
    :param document: a document to process.
    :type document: dict

    :return: a processed documents.
    :rtype: dict
    """
    document = virtool.utils.base_processor(document)

    status = document.pop("status")

    last_update = status[-1]

    document.update({
        "state": last_update["state"],
        "stage": last_update["stage"],
        "created_at": status[0]["timestamp"],
        "progress": status[-1]["progress"]
    })

    return document

class Job:

    def __init__(self, loop, executor, db, settings, dispatch, job_id, task_name, task_args, proc, mem):
        self.loop = loop
        self.executor = executor
        self.db = db
        self.settings = settings
        self.dispatch = dispatch
        self.id = job_id
        self.task_name = task_name
        self.task_args = task_args
        self.proc = proc
        self.mem = mem

        self.started = False
        self.finished = False

        self._progress = 0
        self._state = "waiting"
        self._stage = None
        self._error = None
        self._cancelled = False
        self._task = None
        self._process_task = None
        self._stage_list = None
        self._log_path = os.path.join(self.settings.get("data_path"), "logs", "jobs", self.id)
        self._log_buffer = list()

    def start(self):
        self._task = self.loop.create_task(self.run())
        self.started = True

    async def run(self):
        for method in self._stage_list:
            name = method.__name__

            await self.add_status(stage=name, state="running")

            try:
                await self.add_log("Stage: {}".format(name))
                await method()
            except asyncio.CancelledError:
                self._cancelled = True
            except:
                self._error = handle_exception()

            if self._error or self._cancelled:
                break

        self._progress = 1

        if self._error:
            await self.add_status(state="error")
            await self.cleanup()
        elif self._cancelled:
            await self.add_status(state="cancelled")
            await self.cleanup()
        else:
            await self.add_status(state="complete")

        await self.run_in_executor(flush_log, self._log_path, self._log_buffer)

        self.finished = True

    async def run_in_executor(self, func, *args):
        await self.add_log("Process: {}".format(func.__name__))
        self._process_task = self.loop.run_in_executor(self.executor, func, *args)
        result = await self._process_task
        self._process_task = None

        return result

    async def run_subprocess(self, command, error_test=None, log_stdout=False, log_stderr=True, env=None):
        print("STARTED ADDING INIT LOG")

        await self.add_log("Command: {}".format(" ".join(command)))

        print("FINISHED ADDING INIT LOG")

        # print("STARTED SETTING EVENT LOOP")

        # asyncio.set_event_loop(self.loop)
        # asyncio.get_child_watcher().attach_loop(self.loop)

        # print("FINISHED SETTING EVENT LOOP")

        # print("STARTED WAITING FOR START")

        proc = await asyncio.create_subprocess_exec(
            *command,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
            loop=self.loop,
            env=env
        )

        print("FINISHED WAITING FOR START")

        out = list()
        err = list()

        print("STARTED LISTENING TO STDOUT")

        while True:
            line = await proc.stdout.readline()

            if not line:
                break

            line = line.decode().rstrip()

            out.append(line)

            if log_stdout:
                await self.add_log(line, indent=1)

        print("FINISHED LISTENING TO STDOUT")

        print("STARTED LISTENING TO STDERR")

        while True:
            line = await proc.stderr.readline()

            if not line:
                break

            line = line.decode().rstrip()

            err.append(line)

            if log_stderr:
                await self.add_log(line, indent=1)

        print("FINISHED LISTENING TO STDERR")

        print("STARTED WAITING FOR PROC")

        await proc.wait()

        print("FINISHED WAITING FOR PROC")

        if proc.returncode != 0 or (error_test and error_test(out, err)):
            raise SubprocessError("Command failed: {}. Check job log.".format(" ".join(command)))

    async def add_status(self, state=None, stage=None):
        self._state = state or self._state
        self._stage = stage or self._stage

        if self._progress != 1:
            stage_index = [m.__name__ for m in self._stage_list].index(self._stage)
            self._progress = round((stage_index + 1) / (len(self._stage_list) + 1), 2)

        document = await self.db.jobs.find_one_and_update({"_id": self.id}, {
            "$push": {
                "status": {
                    "state": self._state,
                    "stage": self._stage,
                    "error": self._error,
                    "progress": self._progress,
                    "timestamp": virtool.utils.timestamp()
                }
            }
        }, return_document=pymongo.ReturnDocument.AFTER, projection=LIST_PROJECTION)

        await self.dispatch("jobs", "update", dispatch_processor(document))

    async def add_log(self, line, indent=0):
        timestamp = virtool.utils.timestamp().isoformat()

        self._log_buffer.append("{}{}    {}".format(timestamp, " " * indent * 4, line.rstrip()))

        if len(self._log_buffer) == 15:
            await self.flush_log()
            del self._log_buffer[:]

    async def flush_log(self):
        await self.run_in_executor(flush_log, self._log_path, self._log_buffer)

    async def cancel(self):
        if self.started and not self.finished:
            self._task.cancel()

            while not self.finished:
                await asyncio.sleep(0.1, loop=self.loop)

        await self.cleanup()

        self.finished = True

    async def cleanup(self):
        pass

def stage_method(func):
    func.is_stage_method = True
    return func

def flush_log(path, buffer):
    with open(path, "a") as handle:
        handle.write("\n".join(buffer))

def handle_exception(max_tb=50):
    exception, value, trace_info = sys.exc_info()

    return {
        "type": exception.__name__,
        "traceback": traceback.format_tb(trace_info, max_tb),
        "details": [str(l) for l in value.args]
    }

class SubprocessError(Exception):
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

import threading
from itertools import izip_longest
import os
import urlparse
import re
import json
import time
import urllib2
import urllib
import hashlib
import requests

class VirusTotal2(object):
    _SCAN_ID_RE = re.compile(r"^[a-fA-F0-9]{64}-[0-9]{10}$")

    def __init__(self, api_key, limit_per_min=None):
        limit_per_min = limit_per_min if limit_per_min is not None else 4

        super(VirusTotal2, self).__init__()

        self.api_key = api_key
        self.limit_per_min = limit_per_min
        self.limits = []
        self.limit_lock = threading.Lock()

    #we only scan files and URLs
    def scan(self, thing, thing_type=None, raw=False, rescan=False):
        """
        Submit a file to or URL to VirusTotal for scanning.
        Returns a VirusTotal2Report object

        Keyword arguments:
         thing - a file name on the local system or a URL or list of URLs
         thing_type - Optional, a hint to the function as to what you are sending it
         raw - Optional, if True return the raw JSON output from VT

        Raises a TypeError if it gets something other than a file or URL/list of URLs
        Raises an TypeError if VirusTotal returns something we can't parse.
        """
        #identify the thing
        thing_id = self._whatisthing(thing)
        if thing_type is None:
            thing_type = thing_id

        data = {"apikey": self.api_key}

        #set up and execute the query based on what the thing is
        #we can only take URLs and Files for scan()
        if thing_type == "url":
            endpoint = "https://www.virustotal.com/vtapi/v2/url/scan"
            if isinstance(thing, list):
                data["url"] = "\n".join(thing)
            else:
                data["url"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()

        elif thing_type == "file" and rescan is False:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/scan"
            with open(thing, 'rb') as f:
                file_contents = f.read()

            self._limit_call_handler()
            result = requests.post(endpoint, data=data, files={"file": (os.path.basename(thing), file_contents)}).text
        elif thing_type == "file" and rescan is True:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/rescan"
            fh = open(thing, 'rb')
            content = fh.read()
            data["resource"] = hashlib.sha256(content).hexdigest()

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()
        elif thing_type == "hash" and rescan is True:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/rescan"
            if isinstance(thing, list):
                data["resource"] = ", ".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()
        elif thing_type == "hash" and rescan is not True:
            raise TypeError("Hahses can only be re-scanned, please set rescan=True")
        else:
            raise TypeError("Unable to scan type '"+thing_type+".")

        #should we just return raw JSON?
        if raw:
            return result

        return self._generate_report(result, thing_id, thing)

    def retrieve(self, thing, thing_type=None, raw=False):
        """
        Retrieve a report from VirusTotal based on a hash, IP, domain, file or URL.  NOTE: URLs must include the scheme
         (e.g. http://)
        Returns a VirusTotal2Report object

        Keyword arguments:
         thing - a file name on the local system, a URL or list of URLs,
            an IP or list of IPs, a domain or list of domains, a hash or list of hashes
         thing_type - Optional, a hint to the function as to what you are sending it
         raw - Optional, if True return the raw JSON output from VT

        Raises a TypeError if it gets something other than a filename, URL, IP domain or hash
        Raises an TypeError if VirusTotal returns something we can't parse.
        """
        #trust the user-supplied type over the automatic identification
        thing_id = self._whatisthing(thing)
        if thing_type is None:
            thing_type = thing_id

        data = {"apikey": self.api_key}

        if thing_type == "url":
            endpoint = "http://www.virustotal.com/vtapi/v2/url/report"
            if isinstance(thing, list):
                data["resource"] = "\n".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == "ip":
            endpoint = "http://www.virustotal.com/vtapi/v2/ip-address/report"
            #IPs don't support bulk queries
            if isinstance(thing, list):
                raise TypeError
            data["ip"] = thing

            req = urllib2.Request("%s?%s" % (endpoint, urllib.urlencode([(k, v) for k, v in data.items()])))

        elif thing_type == "file":
            endpoint = "http://www.virustotal.com/vtapi/v2/file/report"
            hashes = []
            if not isinstance(thing, list):
                thing = [thing]

            for f in thing:
                fh = open(f, 'rb')
                content = fh.read()
                hashval = hashlib.sha256(content).hexdigest()
                hashes.append(hashval)

            data["resource"] = ", ".join(hashes)

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == 'domain':
            endpoint = "http://www.virustotal.com/vtapi/v2/domain/report"
            #domains don't support bulk queries
            if isinstance(thing, list):
                raise TypeError
            data["domain"] = thing

            req = urllib2.Request("%s?%s" % (endpoint, urllib.urlencode([(k, v) for k, v in data.items()])))

        elif thing_type == 'hash':
            endpoint = "http://www.virustotal.com/vtapi/v2/file/report"
            if isinstance(thing, list):
                data["resource"] = ", ".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == "scanid":
            #TODO ???
            raise TypeError

        else:
            raise TypeError("Unable to scan type '"+thing_type+".")

        #blocks until we won't violate the 4 queries per min rule
        self._limit_call_handler()

        #result here is a json string
        result = urllib2.urlopen(req).read()

        #should we just return raw JSON?
        if raw:
            return result

        return self._generate_report(result, thing_id, thing)

    def _generate_report(self, result, thing_id, thing):
        """
        Generate a VirusTotal2Report object based on the passed JSON
        Returns a VirusTotal2Report object

        Keyword arguments:
         result - a JSON string to parse into a report.
         thing - the item we're reporting on
         thing_id - what kind of item thing is

        Raises an TypeError if report is something we can't parse.
        """
        report = []

        if isinstance(result, basestring):
            try:
                obj = json.loads(result)
                if isinstance(obj, dict):
                    #one result
                    report.append(VirusTotal2Report(obj, self, thing_id, thing))
                else:
                    #obj is a list
                    for (i, rep) in enumerate(obj):
                        report.append(VirusTotal2Report(rep, self, thing_id, thing[i]))
            except:
                raise TypeError("VT String unparsable: "+str(result))
        else:
            raise TypeError("VT String unparsable: "+result)

        return report if len(report) > 1 else report[0]

    def _limit_call_handler(self):
        """
        Ensure we don't exceed the 4 requests a minute limit by leveraging a thread lock

        Keyword arguments:
            None
        """
        with self.limit_lock:
            if self.limit_per_min <= 0:
                return

            now = time.time()
            self.limits = [l for l in self.limits if l > now]
            self.limits.append(now + 60)

            if len(self.limits) >= self.limit_per_min:
                time.sleep(self.limits[0] - now)

    @staticmethod
    def _grouped(iterable, n):
        """
        take a list of items and return a list of groups of size n.  Fill any missing values at the end with None

        Keyword arguments:
            n - the size of the groups to return
        """
        return izip_longest(*[iter(iterable)] * n, fillvalue=None)

    # noinspection PyTypeChecker
    def _whatisthing(self, thing):
        """
        Bucket the thing it gets passed into the list of items VT supports
        Returns a sting or "unknown"

        Keyword arguments:
            thing - a string to identify
        """
        if isinstance(thing, list):
            thing = thing[0]
        #per the API, bulk requests must be of the same type
        #ignore that you can intersperse scan IDs and hashes for now
        #...although, does that actually matter given the API semantics?

        elif os.path.isfile(thing):
            #thing==filename
            return "file"

        #implied failure case, thing is neither a list or a file, so we assume string
        if not isinstance(thing, basestring):
            return "unknown"

        #Is a hash
        if all(i in "1234567890abcdef" for i in str(thing).lower()) and len(thing) in [32, 40, 64]:
            return "hash"

        # Is IP address
        if all(i in "1234567890." for i in thing) and len(thing) <= 15:
            return "ip"

        # Is domain name
        if "." in thing and "/" not in thing:
            return "domain"

        #Is scan ID
        if self._SCAN_ID_RE.match(thing):
            return "scanid"

        # Is URL ?
        if urlparse.urlparse(thing).scheme:
            return "url"

        return "unknown"

class VirusTotal2Report(object):
    def __init__(self, obj, parent, thing_id, query):

        super(VirusTotal2Report, self).__init__()

        self.scan = parent
        self._json = obj
        self.type = thing_id
        self.query = query

        #initial API calls return response_code = 1
        #we expect -2, as the scan is queued, so we update to get what we think we should have,
        self.update()

    def __repr__(self):
        return "<VirusTotal2 report %s (%s)>" % (
            self.id,
            self.status,
        )

    def __iter__(self):
        if self.type == "ip":
            for resolution in self.resolutions.iteritems():
                yield resolution
        elif self.type == "domain":
            for resolution in self.resolutions.iteritems():
                yield resolution
        elif self.type == "url":
            for scanner, report in self.scans.iteritems():
                yield (scanner, report["result"])
        else:
            for antivirus, report in self.scans.iteritems():
                yield (
                    (antivirus, report["version"], report["update"]),
                    report["result"],
                )

    def __getattr__(self, attr):
        item = {
            "id": "resource",
            "status": "verbose_msg",
        }.get(attr, attr)

        try:
            return self._json[item]

        except KeyError:
            raise AttributeError(attr)

    def update(self):
        """
        Re-query the Virustotal API for new results on the current object.  If the current object is listed as
        not in VirusTotal (can be the case with IPs or domains), this function does nothing.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT
        """
        if self.response_code == 0:
            #it wasn't there the first time.  why try again?
            # or we already have complete results.  why update without a rescan?
            return

        if self.type in ("ip", "domain"):
            data = self.scan.retrieve(self.query, raw=True)
        elif self.type == "file":
            data = self.scan.retrieve(self.scan_id, thing_type="hash", raw=True)
        else:
            data = self.scan.retrieve(self.scan_id, thing_type=self.type, raw=True)

        try:
            self._json = json.loads(data)
        except:
            raise TypeError

    def rescan(self):
        #only applies to files and URLs
        """
        Requests a rescan of the current file.  This API only works for reports that have been generated from files or
          hashes.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT
        """
        if self.type in ("file", "hash"):
            data = self.scan.retrieve(self.scan_id, thing_type="hash", raw=True, rescan=True)
        else:
            raise TypeError("cannot rescan type "+self.type)

        try:
            self._json = json.loads(data)
        except:
            raise TypeError

    def wait(self):
        """
        Wait until the Virustotal API is done scanning the current object.  If the current object is listed as not in
        VirusTotal (can be the case with IPs or domains), or we already have results this function returns immediately.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT (it would pass through from the update() function)
        """
        interval = 60

        self.update()
        while self.response_code not in (1, 0):
            time.sleep(interval)
            self.update()<commit_msg>`os.isfile` expects a string only<commit_after>#!/usr/bin/env python

import threading
from itertools import izip_longest
import os
import urlparse
import re
import json
import time
import urllib2
import urllib
import hashlib
import requests

class VirusTotal2(object):
    _SCAN_ID_RE = re.compile(r"^[a-fA-F0-9]{64}-[0-9]{10}$")

    def __init__(self, api_key, limit_per_min=None):
        limit_per_min = limit_per_min if limit_per_min is not None else 4

        super(VirusTotal2, self).__init__()

        self.api_key = api_key
        self.limit_per_min = limit_per_min
        self.limits = []
        self.limit_lock = threading.Lock()

    #we only scan files and URLs
    def scan(self, thing, thing_type=None, raw=False, rescan=False):
        """
        Submit a file to or URL to VirusTotal for scanning.
        Returns a VirusTotal2Report object

        Keyword arguments:
         thing - a file name on the local system or a URL or list of URLs
         thing_type - Optional, a hint to the function as to what you are sending it
         raw - Optional, if True return the raw JSON output from VT

        Raises a TypeError if it gets something other than a file or URL/list of URLs
        Raises an TypeError if VirusTotal returns something we can't parse.
        """
        #identify the thing
        thing_id = self._whatisthing(thing)
        if thing_type is None:
            thing_type = thing_id

        data = {"apikey": self.api_key}

        #set up and execute the query based on what the thing is
        #we can only take URLs and Files for scan()
        if thing_type == "url":
            endpoint = "https://www.virustotal.com/vtapi/v2/url/scan"
            if isinstance(thing, list):
                data["url"] = "\n".join(thing)
            else:
                data["url"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()

        elif thing_type == "file" and rescan is False:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/scan"
            with open(thing, 'rb') as f:
                file_contents = f.read()

            self._limit_call_handler()
            result = requests.post(endpoint, data=data, files={"file": (os.path.basename(thing), file_contents)}).text
        elif thing_type == "file" and rescan is True:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/rescan"
            fh = open(thing, 'rb')
            content = fh.read()
            data["resource"] = hashlib.sha256(content).hexdigest()

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()
        elif thing_type == "hash" and rescan is True:
            endpoint = "https://www.virustotal.com/vtapi/v2/file/rescan"
            if isinstance(thing, list):
                data["resource"] = ", ".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))
            self._limit_call_handler()
            result = urllib2.urlopen(req).read()
        elif thing_type == "hash" and rescan is not True:
            raise TypeError("Hahses can only be re-scanned, please set rescan=True")
        else:
            raise TypeError("Unable to scan type '"+thing_type+".")

        #should we just return raw JSON?
        if raw:
            return result

        return self._generate_report(result, thing_id, thing)

    def retrieve(self, thing, thing_type=None, raw=False):
        """
        Retrieve a report from VirusTotal based on a hash, IP, domain, file or URL.  NOTE: URLs must include the scheme
         (e.g. http://)
        Returns a VirusTotal2Report object

        Keyword arguments:
         thing - a file name on the local system, a URL or list of URLs,
            an IP or list of IPs, a domain or list of domains, a hash or list of hashes
         thing_type - Optional, a hint to the function as to what you are sending it
         raw - Optional, if True return the raw JSON output from VT

        Raises a TypeError if it gets something other than a filename, URL, IP domain or hash
        Raises an TypeError if VirusTotal returns something we can't parse.
        """
        #trust the user-supplied type over the automatic identification
        thing_id = self._whatisthing(thing)
        if thing_type is None:
            thing_type = thing_id

        data = {"apikey": self.api_key}

        if thing_type == "url":
            endpoint = "http://www.virustotal.com/vtapi/v2/url/report"
            if isinstance(thing, list):
                data["resource"] = "\n".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == "ip":
            endpoint = "http://www.virustotal.com/vtapi/v2/ip-address/report"
            #IPs don't support bulk queries
            if isinstance(thing, list):
                raise TypeError
            data["ip"] = thing

            req = urllib2.Request("%s?%s" % (endpoint, urllib.urlencode([(k, v) for k, v in data.items()])))

        elif thing_type == "file":
            endpoint = "http://www.virustotal.com/vtapi/v2/file/report"
            hashes = []
            if not isinstance(thing, list):
                thing = [thing]

            for f in thing:
                fh = open(f, 'rb')
                content = fh.read()
                hashval = hashlib.sha256(content).hexdigest()
                hashes.append(hashval)

            data["resource"] = ", ".join(hashes)

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == 'domain':
            endpoint = "http://www.virustotal.com/vtapi/v2/domain/report"
            #domains don't support bulk queries
            if isinstance(thing, list):
                raise TypeError
            data["domain"] = thing

            req = urllib2.Request("%s?%s" % (endpoint, urllib.urlencode([(k, v) for k, v in data.items()])))

        elif thing_type == 'hash':
            endpoint = "http://www.virustotal.com/vtapi/v2/file/report"
            if isinstance(thing, list):
                data["resource"] = ", ".join(thing)
            else:
                data["resource"] = thing

            req = urllib2.Request(endpoint, urllib.urlencode(data))

        elif thing_type == "scanid":
            #TODO ???
            raise TypeError

        else:
            raise TypeError("Unable to scan type '"+thing_type+".")

        #blocks until we won't violate the 4 queries per min rule
        self._limit_call_handler()

        #result here is a json string
        result = urllib2.urlopen(req).read()

        #should we just return raw JSON?
        if raw:
            return result

        return self._generate_report(result, thing_id, thing)

    def _generate_report(self, result, thing_id, thing):
        """
        Generate a VirusTotal2Report object based on the passed JSON
        Returns a VirusTotal2Report object

        Keyword arguments:
         result - a JSON string to parse into a report.
         thing - the item we're reporting on
         thing_id - what kind of item thing is

        Raises an TypeError if report is something we can't parse.
        """
        report = []

        if isinstance(result, basestring):
            try:
                obj = json.loads(result)
                if isinstance(obj, dict):
                    #one result
                    report.append(VirusTotal2Report(obj, self, thing_id, thing))
                else:
                    #obj is a list
                    for (i, rep) in enumerate(obj):
                        report.append(VirusTotal2Report(rep, self, thing_id, thing[i]))
            except:
                raise TypeError("VT String unparsable: "+str(result))
        else:
            raise TypeError("VT String unparsable: "+result)

        return report if len(report) > 1 else report[0]

    def _limit_call_handler(self):
        """
        Ensure we don't exceed the 4 requests a minute limit by leveraging a thread lock

        Keyword arguments:
            None
        """
        with self.limit_lock:
            if self.limit_per_min <= 0:
                return

            now = time.time()
            self.limits = [l for l in self.limits if l > now]
            self.limits.append(now + 60)

            if len(self.limits) >= self.limit_per_min:
                time.sleep(self.limits[0] - now)

    @staticmethod
    def _grouped(iterable, n):
        """
        take a list of items and return a list of groups of size n.  Fill any missing values at the end with None

        Keyword arguments:
            n - the size of the groups to return
        """
        return izip_longest(*[iter(iterable)] * n, fillvalue=None)

    # noinspection PyTypeChecker
    def _whatisthing(self, thing):
        """
        Bucket the thing it gets passed into the list of items VT supports
        Returns a sting or "unknown"

        Keyword arguments:
            thing - a string to identify
        """
        if isinstance(thing, list):
            thing = thing[0]
        #per the API, bulk requests must be of the same type
        #ignore that you can intersperse scan IDs and hashes for now
        #...although, does that actually matter given the API semantics?

        elif isinstance(thing,str) and os.path.isfile(thing):
            #thing==filename
            return "file"

        #implied failure case, thing is neither a list or a file, so we assume string
        if not isinstance(thing, basestring):
            return "unknown"

        #Is a hash
        if all(i in "1234567890abcdef" for i in str(thing).lower()) and len(thing) in [32, 40, 64]:
            return "hash"

        # Is IP address
        if all(i in "1234567890." for i in thing) and len(thing) <= 15:
            return "ip"

        # Is domain name
        if "." in thing and "/" not in thing:
            return "domain"

        #Is scan ID
        if self._SCAN_ID_RE.match(thing):
            return "scanid"

        # Is URL ?
        if urlparse.urlparse(thing).scheme:
            return "url"

        return "unknown"

class VirusTotal2Report(object):
    def __init__(self, obj, parent, thing_id, query):

        super(VirusTotal2Report, self).__init__()

        self.scan = parent
        self._json = obj
        self.type = thing_id
        self.query = query

        #initial API calls return response_code = 1
        #we expect -2, as the scan is queued, so we update to get what we think we should have,
        self.update()

    def __repr__(self):
        return "<VirusTotal2 report %s (%s)>" % (
            self.id,
            self.status,
        )

    def __iter__(self):
        if self.type == "ip":
            for resolution in self.resolutions.iteritems():
                yield resolution
        elif self.type == "domain":
            for resolution in self.resolutions.iteritems():
                yield resolution
        elif self.type == "url":
            for scanner, report in self.scans.iteritems():
                yield (scanner, report["result"])
        else:
            for antivirus, report in self.scans.iteritems():
                yield (
                    (antivirus, report["version"], report["update"]),
                    report["result"],
                )

    def __getattr__(self, attr):
        item = {
            "id": "resource",
            "status": "verbose_msg",
        }.get(attr, attr)

        try:
            return self._json[item]

        except KeyError:
            raise AttributeError(attr)

    def update(self):
        """
        Re-query the Virustotal API for new results on the current object.  If the current object is listed as
        not in VirusTotal (can be the case with IPs or domains), this function does nothing.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT
        """
        if self.response_code == 0:
            #it wasn't there the first time.  why try again?
            # or we already have complete results.  why update without a rescan?
            return

        if self.type in ("ip", "domain"):
            data = self.scan.retrieve(self.query, raw=True)
        elif self.type == "file":
            data = self.scan.retrieve(self.scan_id, thing_type="hash", raw=True)
        else:
            data = self.scan.retrieve(self.scan_id, thing_type=self.type, raw=True)

        try:
            self._json = json.loads(data)
        except:
            raise TypeError

    def rescan(self):
        #only applies to files and URLs
        """
        Requests a rescan of the current file.  This API only works for reports that have been generated from files or
          hashes.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT
        """
        if self.type in ("file", "hash"):
            data = self.scan.retrieve(self.scan_id, thing_type="hash", raw=True, rescan=True)
        else:
            raise TypeError("cannot rescan type "+self.type)

        try:
            self._json = json.loads(data)
        except:
            raise TypeError

    def wait(self):
        """
        Wait until the Virustotal API is done scanning the current object.  If the current object is listed as not in
        VirusTotal (can be the case with IPs or domains), or we already have results this function returns immediately.

        Keyword arguments:
            none

        Raises:
            TypeError if we don't get JSON back from VT (it would pass through from the update() function)
        """
        interval = 60

        self.update()
        while self.response_code not in (1, 0):
            time.sleep(interval)
            self.update()<|endoftext|>


<commit_before>import unittest

import mock
import numpy

import cupy
try:
    import cupy.cuda.cudnn as libcudnn
    cudnn_enabled = True
    modes = [
        libcudnn.CUDNN_ACTIVATION_SIGMOID,
        libcudnn.CUDNN_ACTIVATION_RELU,
        libcudnn.CUDNN_ACTIVATION_TANH,
    ]
except ImportError:
    cudnn_enabled = False
    modes = []
import cupy.cudnn
from cupy import testing

@unittest.skipUnless(
    cudnn_enabled and libcudnn.getVersion() >= 3000,
    'cuDNN >= 3.0 is supported')
@testing.parameterize(*testing.product({
    'dtype': [numpy.float32, numpy.float64],
    'mode': modes,
}))
class TestCudnnActivation(unittest.TestCase):

    def setUp(self):
        self.x = testing.shaped_arange((3, 4), cupy, self.dtype)
        self.y = testing.shaped_arange((3, 4), cupy, self.dtype)
        self.g = testing.shaped_arange((3, 4), cupy, self.dtype)

    def test_activation_forward_version(self):
        if libcudnn.getVersion() >= 4000:
            patch = 'cupy.cuda.cudnn.activationForward_v4'
        else:
            patch = 'cupy.cuda.cudnn.activationForward_v3'
        with mock.patch(patch) as func:
            cupy.cudnn.activation_forward(self.x, self.mode)
            self.assertEqual(func.called, True)

    def test_activation_backward_version(self):
        if libcudnn.getVersion() >= 4000:
            patch = 'cupy.cuda.cudnn.activationBackward_v4'
        else:
            patch = 'cupy.cuda.cudnn.activationBakward_v3'
        with mock.patch(patch) as func:
            cupy.cudnn.activation_backward(self.x, self.y, self.g, self.mode)
            self.assertEqual(func.called, True)
<commit_msg>Fix test<commit_after>import unittest

import mock
import numpy

import cupy
try:
    import cupy.cuda.cudnn as libcudnn
    cudnn_enabled = True
    modes = [
        libcudnn.CUDNN_ACTIVATION_SIGMOID,
        libcudnn.CUDNN_ACTIVATION_RELU,
        libcudnn.CUDNN_ACTIVATION_TANH,
    ]
    import cupy.cudnn
except ImportError:
    cudnn_enabled = False
    modes = []
from cupy import testing

@testing.parameterize(*testing.product({
    'dtype': [numpy.float32, numpy.float64],
    'mode': modes,
}))
@unittest.skipUnless(
    cudnn_enabled and libcudnn.getVersion() >= 3000,
    'cuDNN >= 3.0 is supported')
class TestCudnnActivation(unittest.TestCase):

    def setUp(self):
        self.x = testing.shaped_arange((3, 4), cupy, self.dtype)
        self.y = testing.shaped_arange((3, 4), cupy, self.dtype)
        self.g = testing.shaped_arange((3, 4), cupy, self.dtype)

    def test_activation_forward_version(self):
        if libcudnn.getVersion() >= 4000:
            patch = 'cupy.cuda.cudnn.activationForward_v4'
        else:
            patch = 'cupy.cuda.cudnn.activationForward_v3'
        with mock.patch(patch) as func:
            cupy.cudnn.activation_forward(self.x, self.mode)
            self.assertEqual(func.called, True)

    def test_activation_backward_version(self):
        if libcudnn.getVersion() >= 4000:
            patch = 'cupy.cuda.cudnn.activationBackward_v4'
        else:
            patch = 'cupy.cuda.cudnn.activationBakward_v3'
        with mock.patch(patch) as func:
            cupy.cudnn.activation_backward(self.x, self.y, self.g, self.mode)
            self.assertEqual(func.called, True)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (c) 2015-2016 The Khronos Group Inc.
# Copyright (c) 2015-2016 Valve Corporation
# Copyright (c) 2015-2016 LunarG, Inc.
# Copyright (c) 2015-2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: Chia-I Wu <olv@lunarg.com>
# Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
# Author: Jon Ashburn <jon@lunarg.com>
# Author: Gwan-gyeong Mun <kk.moon@samsung.com>
# Author: Mark Lobodzinski <mark@lunarg.com>

import sys

import vulkan

def generate_get_proc_addr_check(name):
    return "    if (!%s || %s[0] != 'v' || %s[1] != 'k')\n" \
           "        return NULL;" % ((name,) * 3)

class Subcommand(object):
    def __init__(self, argv):
        self.argv = argv
        self.headers = vulkan.headers
        self.protos = vulkan.protos
        self.outfile = None

    def run(self):
        if self.outfile:
            with open(self.outfile, "w") as outfile:
                outfile.write(self.generate())
        else:
            print(self.generate())

    def generate(self):
        copyright = self.generate_copyright()
        header = self.generate_header()
        body = self.generate_body()
        footer = self.generate_footer()

        contents = []
        if copyright:
            contents.append(copyright)
        if header:
            contents.append(header)
        if body:
            contents.append(body)
        if footer:
            contents.append(footer)

        return "\n\n".join(contents)

    def generate_copyright(self):
        return """/* THIS FILE IS GENERATED.  DO NOT EDIT. */

/*
 * Copyright (c) 2015-2016 The Khronos Group Inc.
 * Copyright (c) 2015-2016 Valve Corporation
 * Copyright (c) 2015-2016 LunarG, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
 */"""

    def generate_header(self):
        return "\n".join(["#include <" + h + ">" for h in self.headers])

    def generate_body(self):
        pass

    def generate_footer(self):
        pass

class DispatchTableOpsSubcommand(Subcommand):
    def __init__(self, argv):
        self.argv = argv
        self.headers = vulkan.headers_all
        self.protos = vulkan.protos_all
        self.outfile = None

    def run(self):
        if len(self.argv) < 1:
            print("DispatchTableOpsSubcommand: <prefix> unspecified")
            return

        self.prefix = self.argv[0]

        if len(self.argv) > 2:
            print("DispatchTableOpsSubcommand: <prefix> [outfile]")
            return

        if len(self.argv) == 2:
            self.outfile = self.argv[1]

        super(DispatchTableOpsSubcommand, self).run()

    def generate_header(self):
        return "\n".join(["#include <vulkan/vulkan.h>",
                          "#include <vulkan/vk_layer.h>",
                          "#include <string.h>"])

    def _generate_init_dispatch(self, type):
        stmts = []
        func = []
        if type == "device":
            # GPA has to be first one and uses wrapped object
            stmts.append("    memset(table, 0, sizeof(*table));")
            stmts.append("    // Core device function pointers")
            stmts.append("    table->GetDeviceProcAddr = (PFN_vkGetDeviceProcAddr) gpa(device, \"vkGetDeviceProcAddr\");")

            for proto in self.protos:
                if proto.name == "CreateInstance" or proto.name == "EnumerateInstanceExtensionProperties" or \
                  proto.name == "EnumerateInstanceLayerProperties" or proto.params[0].ty == "VkInstance" or \
                  proto.params[0].ty == "VkPhysicalDevice" or proto.name == "GetDeviceProcAddr":
                    continue
                stmts.append("    table->%s = (PFN_vk%s) gpa(device, \"vk%s\");" % (proto.name, proto.name, proto.name))
            func.append("static inline void %s_init_device_dispatch_table(VkDevice device,"
                % self.prefix)
            func.append("%s                                               VkLayerDispatchTable *table,"
                % (" " * len(self.prefix)))
            func.append("%s                                               PFN_vkGetDeviceProcAddr gpa)"
                % (" " * len(self.prefix)))
        else:
            stmts.append("    memset(table, 0, sizeof(*table));")
            stmts.append("    // Core instance function pointers")
            stmts.append("    table->GetInstanceProcAddr = (PFN_vkGetInstanceProcAddr) gpa(instance, \"vkGetInstanceProcAddr\");")

            for proto in self.protos:
                if proto.params[0].ty != "VkInstance" and proto.params[0].ty != "VkPhysicalDevice" or \
                  proto.name == "CreateDevice" or proto.name == "GetInstanceProcAddr":
                    continue
                # Protect platform-dependent APIs with #ifdef
                if 'KHR' in proto.name and 'Win32' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_WIN32_KHR")
                if 'KHR' in proto.name and 'Xlib' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_XLIB_KHR")
                if 'KHR' in proto.name and 'Xcb' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_XCB_KHR")
                if 'KHR' in proto.name and 'Mir' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_MIR_KHR")
                if 'KHR' in proto.name and 'Wayland' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_WAYLAND_KHR")
                if 'KHR' in proto.name and 'Android' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_ANDROID_KHR")
                # Output dispatch table entry
                stmts.append("    table->%s = (PFN_vk%s) gpa(instance, \"vk%s\");" %
                      (proto.name, proto.name, proto.name))
                # If entry was protected by an #ifdef, close with a #endif
                if 'KHR' in proto.name and 'Win32' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_WIN32_KHR")
                if 'KHR' in proto.name and 'Xlib' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_XLIB_KHR")
                if 'KHR' in proto.name and 'Xcb' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_XCB_KHR")
                if 'KHR' in proto.name and 'Mir' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_MIR_KHR")
                if 'KHR' in proto.name and 'Wayland' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_WAYLAND_KHR")
                if 'KHR' in proto.name and 'Android' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_ANDROID_KHR")
            func.append("static inline void %s_init_instance_dispatch_table(" % self.prefix)
            func.append("%s        VkInstance instance," % (" " * len(self.prefix)))
            func.append("%s        VkLayerInstanceDispatchTable *table," % (" " * len(self.prefix)))
            func.append("%s        PFN_vkGetInstanceProcAddr gpa)" % (" " * len(self.prefix)))
        func.append("{")
        func.append("%s" % "\n".join(stmts))
        func.append("}")

        return "\n".join(func)

    def generate_body(self):
        body = [self._generate_init_dispatch("device"),
                self._generate_init_dispatch("instance")]

        return "\n\n".join(body)

class WinDefFileSubcommand(Subcommand):
    def run(self):
        library_exports = {
                "all": [],
                "icd": [
                    "vk_icdGetInstanceProcAddr",
                ],
                "layer": [
                    "vkGetInstanceProcAddr",
                    "vkGetDeviceProcAddr",
                    "vkEnumerateInstanceLayerProperties",
                    "vkEnumerateInstanceExtensionProperties"
                ],
                "layer_multi": [
                    "multi2GetInstanceProcAddr",
                    "multi1GetDeviceProcAddr"
                ]
        }

        if len(self.argv) < 2 or len(self.argv) > 3 or self.argv[1] not in library_exports:
            print("WinDefFileSubcommand: <library-name> {%s} [outfile]" %
                    "|".join(library_exports.keys()))
            return

        self.library = self.argv[0]
        if self.library == "VkLayer_multi":
            self.exports = library_exports["layer_multi"]
        else:
            self.exports = library_exports[self.argv[1]]

        if len(self.argv) == 3:
            self.outfile = self.argv[2]

        super(WinDefFileSubcommand, self).run()

    def generate_copyright(self):
        return """; THIS FILE IS GENERATED.  DO NOT EDIT.

;;;; Begin Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Vulkan
;
; Copyright (c) 2015-2016 The Khronos Group Inc.
; Copyright (c) 2015-2016 Valve Corporation
; Copyright (c) 2015-2016 LunarG, Inc.
;
; Licensed under the Apache License, Version 2.0 (the "License");
; you may not use this file except in compliance with the License.
; You may obtain a copy of the License at
;
;     http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
; distributed under the License is distributed on an "AS IS" BASIS,
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
; See the License for the specific language governing permissions and
; limitations under the License.
;
;  Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
;;;;  End Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"""

    def generate_header(self):
        return "; The following is required on Windows, for exporting symbols from the DLL"

    def generate_body(self):
        body = []

        body.append("LIBRARY " + self.library)
        body.append("EXPORTS")

        for proto in self.exports:
            if self.library != "VkLayerSwapchain" or proto != "vkEnumerateInstanceExtensionProperties" and proto != "vkEnumerateInstanceLayerProperties":
                body.append( proto)

        return "\n".join(body)

def main():
    wsi = {
            "Win32",
            "Android",
            "Xcb",
            "Xlib",
            "Wayland",
            "Mir",
            "Display",
    }
    subcommands = {
            "dispatch-table-ops": DispatchTableOpsSubcommand,
            "win-def-file": WinDefFileSubcommand,
    }

    if len(sys.argv) < 3 or sys.argv[1] not in wsi or sys.argv[2] not in subcommands:
        print("Usage: %s <wsi> <subcommand> [options]" % sys.argv[0])
        print
        print("Available sucommands are: %s" % " ".join(subcommands))
        exit(1)

    subcmd = subcommands[sys.argv[2]](sys.argv[3:])
    subcmd.run()

if __name__ == "__main__":
    main()
<commit_msg>scripts: Re-added AllPlatforms wsi option<commit_after>#!/usr/bin/env python3
#
# Copyright (c) 2015-2016 The Khronos Group Inc.
# Copyright (c) 2015-2016 Valve Corporation
# Copyright (c) 2015-2016 LunarG, Inc.
# Copyright (c) 2015-2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Author: Chia-I Wu <olv@lunarg.com>
# Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
# Author: Jon Ashburn <jon@lunarg.com>
# Author: Gwan-gyeong Mun <kk.moon@samsung.com>
# Author: Mark Lobodzinski <mark@lunarg.com>

import sys

import vulkan

def generate_get_proc_addr_check(name):
    return "    if (!%s || %s[0] != 'v' || %s[1] != 'k')\n" \
           "        return NULL;" % ((name,) * 3)

class Subcommand(object):
    def __init__(self, argv):
        self.argv = argv
        self.headers = vulkan.headers
        self.protos = vulkan.protos
        self.outfile = None

    def run(self):
        if self.outfile:
            with open(self.outfile, "w") as outfile:
                outfile.write(self.generate())
        else:
            print(self.generate())

    def generate(self):
        copyright = self.generate_copyright()
        header = self.generate_header()
        body = self.generate_body()
        footer = self.generate_footer()

        contents = []
        if copyright:
            contents.append(copyright)
        if header:
            contents.append(header)
        if body:
            contents.append(body)
        if footer:
            contents.append(footer)

        return "\n\n".join(contents)

    def generate_copyright(self):
        return """/* THIS FILE IS GENERATED.  DO NOT EDIT. */

/*
 * Copyright (c) 2015-2016 The Khronos Group Inc.
 * Copyright (c) 2015-2016 Valve Corporation
 * Copyright (c) 2015-2016 LunarG, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
 */"""

    def generate_header(self):
        return "\n".join(["#include <" + h + ">" for h in self.headers])

    def generate_body(self):
        pass

    def generate_footer(self):
        pass

class DispatchTableOpsSubcommand(Subcommand):
    def __init__(self, argv):
        self.argv = argv
        self.headers = vulkan.headers_all
        self.protos = vulkan.protos_all
        self.outfile = None

    def run(self):
        if len(self.argv) < 1:
            print("DispatchTableOpsSubcommand: <prefix> unspecified")
            return

        self.prefix = self.argv[0]

        if len(self.argv) > 2:
            print("DispatchTableOpsSubcommand: <prefix> [outfile]")
            return

        if len(self.argv) == 2:
            self.outfile = self.argv[1]

        super(DispatchTableOpsSubcommand, self).run()

    def generate_header(self):
        return "\n".join(["#include <vulkan/vulkan.h>",
                          "#include <vulkan/vk_layer.h>",
                          "#include <string.h>"])

    def _generate_init_dispatch(self, type):
        stmts = []
        func = []
        if type == "device":
            # GPA has to be first one and uses wrapped object
            stmts.append("    memset(table, 0, sizeof(*table));")
            stmts.append("    // Core device function pointers")
            stmts.append("    table->GetDeviceProcAddr = (PFN_vkGetDeviceProcAddr) gpa(device, \"vkGetDeviceProcAddr\");")

            for proto in self.protos:
                if proto.name == "CreateInstance" or proto.name == "EnumerateInstanceExtensionProperties" or \
                  proto.name == "EnumerateInstanceLayerProperties" or proto.params[0].ty == "VkInstance" or \
                  proto.params[0].ty == "VkPhysicalDevice" or proto.name == "GetDeviceProcAddr":
                    continue
                stmts.append("    table->%s = (PFN_vk%s) gpa(device, \"vk%s\");" % (proto.name, proto.name, proto.name))
            func.append("static inline void %s_init_device_dispatch_table(VkDevice device,"
                % self.prefix)
            func.append("%s                                               VkLayerDispatchTable *table,"
                % (" " * len(self.prefix)))
            func.append("%s                                               PFN_vkGetDeviceProcAddr gpa)"
                % (" " * len(self.prefix)))
        else:
            stmts.append("    memset(table, 0, sizeof(*table));")
            stmts.append("    // Core instance function pointers")
            stmts.append("    table->GetInstanceProcAddr = (PFN_vkGetInstanceProcAddr) gpa(instance, \"vkGetInstanceProcAddr\");")

            for proto in self.protos:
                if proto.params[0].ty != "VkInstance" and proto.params[0].ty != "VkPhysicalDevice" or \
                  proto.name == "CreateDevice" or proto.name == "GetInstanceProcAddr":
                    continue
                # Protect platform-dependent APIs with #ifdef
                if 'KHR' in proto.name and 'Win32' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_WIN32_KHR")
                if 'KHR' in proto.name and 'Xlib' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_XLIB_KHR")
                if 'KHR' in proto.name and 'Xcb' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_XCB_KHR")
                if 'KHR' in proto.name and 'Mir' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_MIR_KHR")
                if 'KHR' in proto.name and 'Wayland' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_WAYLAND_KHR")
                if 'KHR' in proto.name and 'Android' in proto.name:
                    stmts.append("#ifdef VK_USE_PLATFORM_ANDROID_KHR")
                # Output dispatch table entry
                stmts.append("    table->%s = (PFN_vk%s) gpa(instance, \"vk%s\");" %
                      (proto.name, proto.name, proto.name))
                # If entry was protected by an #ifdef, close with a #endif
                if 'KHR' in proto.name and 'Win32' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_WIN32_KHR")
                if 'KHR' in proto.name and 'Xlib' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_XLIB_KHR")
                if 'KHR' in proto.name and 'Xcb' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_XCB_KHR")
                if 'KHR' in proto.name and 'Mir' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_MIR_KHR")
                if 'KHR' in proto.name and 'Wayland' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_WAYLAND_KHR")
                if 'KHR' in proto.name and 'Android' in proto.name:
                    stmts.append("#endif // VK_USE_PLATFORM_ANDROID_KHR")
            func.append("static inline void %s_init_instance_dispatch_table(" % self.prefix)
            func.append("%s        VkInstance instance," % (" " * len(self.prefix)))
            func.append("%s        VkLayerInstanceDispatchTable *table," % (" " * len(self.prefix)))
            func.append("%s        PFN_vkGetInstanceProcAddr gpa)" % (" " * len(self.prefix)))
        func.append("{")
        func.append("%s" % "\n".join(stmts))
        func.append("}")

        return "\n".join(func)

    def generate_body(self):
        body = [self._generate_init_dispatch("device"),
                self._generate_init_dispatch("instance")]

        return "\n\n".join(body)

class WinDefFileSubcommand(Subcommand):
    def run(self):
        library_exports = {
                "all": [],
                "icd": [
                    "vk_icdGetInstanceProcAddr",
                ],
                "layer": [
                    "vkGetInstanceProcAddr",
                    "vkGetDeviceProcAddr",
                    "vkEnumerateInstanceLayerProperties",
                    "vkEnumerateInstanceExtensionProperties"
                ],
                "layer_multi": [
                    "multi2GetInstanceProcAddr",
                    "multi1GetDeviceProcAddr"
                ]
        }

        if len(self.argv) < 2 or len(self.argv) > 3 or self.argv[1] not in library_exports:
            print("WinDefFileSubcommand: <library-name> {%s} [outfile]" %
                    "|".join(library_exports.keys()))
            return

        self.library = self.argv[0]
        if self.library == "VkLayer_multi":
            self.exports = library_exports["layer_multi"]
        else:
            self.exports = library_exports[self.argv[1]]

        if len(self.argv) == 3:
            self.outfile = self.argv[2]

        super(WinDefFileSubcommand, self).run()

    def generate_copyright(self):
        return """; THIS FILE IS GENERATED.  DO NOT EDIT.

;;;; Begin Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Vulkan
;
; Copyright (c) 2015-2016 The Khronos Group Inc.
; Copyright (c) 2015-2016 Valve Corporation
; Copyright (c) 2015-2016 LunarG, Inc.
;
; Licensed under the Apache License, Version 2.0 (the "License");
; you may not use this file except in compliance with the License.
; You may obtain a copy of the License at
;
;     http://www.apache.org/licenses/LICENSE-2.0
;
; Unless required by applicable law or agreed to in writing, software
; distributed under the License is distributed on an "AS IS" BASIS,
; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
; See the License for the specific language governing permissions and
; limitations under the License.
;
;  Author: Courtney Goeltzenleuchter <courtney@LunarG.com>
;;;;  End Copyright Notice ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"""

    def generate_header(self):
        return "; The following is required on Windows, for exporting symbols from the DLL"

    def generate_body(self):
        body = []

        body.append("LIBRARY " + self.library)
        body.append("EXPORTS")

        for proto in self.exports:
            if self.library != "VkLayerSwapchain" or proto != "vkEnumerateInstanceExtensionProperties" and proto != "vkEnumerateInstanceLayerProperties":
                body.append( proto)

        return "\n".join(body)

def main():
    wsi = {
            "Win32",
            "Android",
            "Xcb",
            "Xlib",
            "Wayland",
            "Mir",
            "Display",
            "AllPlatforms",
    }
    subcommands = {
            "dispatch-table-ops": DispatchTableOpsSubcommand,
            "win-def-file": WinDefFileSubcommand,
    }

    if len(sys.argv) < 3 or sys.argv[1] not in wsi or sys.argv[2] not in subcommands:
        print("Usage: %s <wsi> <subcommand> <option>" % sys.argv[0])
        print
        print("Available wsi are: %s" % " ".join(wsi))
        print("Available subcommands are: %s" % " ".join(subcommands))
        exit(1)

    subcmd = subcommands[sys.argv[2]](sys.argv[3:])
    subcmd.run()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""Test how opacity is handled for SVG."""

import pytest

from ...testing_utils import assert_no_logs

# TODO: xfail tests fail because of GhostScript and are supposed to work with
# real PDF files.

opacity_source = '''
  <style>
    @page { size: 9px }
    svg { display: block }
  </style>
  <svg width="9px" height="9px" xmlns="http://www.w3.org/2000/svg">%s</svg>'''

@assert_no_logs
def test_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="rgb(127, 255, 127)" fill="rgb(127, 127, 255)" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" opacity="0.5" />
        ''',
    )

@assert_no_logs
def test_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" fill-opacity="0.5" />
        ''',
    )

@pytest.mark.xfail
@assert_no_logs
def test_stroke_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" opacity="0.5" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" stroke-opacity="0.5" />
        ''',
    )

@pytest.mark.xfail
@assert_no_logs
def test_stroke_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" opacity="0.5" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue"
                  stroke-opacity="0.5" fill-opacity="0.5" />
        ''',
    )

@pytest.mark.xfail
@assert_no_logs
def test_pattern_gradient_stroke_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <defs>
              <linearGradient id="grad" x1="0" y1="0" x2="0" y2="1"
                              gradientUnits="objectBoundingBox">
                <stop stop-color="black" offset="42.86%"></stop>
                <stop stop-color="green" offset="42.86%"></stop>
              </linearGradient>
              <pattern id="pat" x="0" y="0" width="2" height="2"
                       patternUnits="userSpaceOnUse"
                       patternContentUnits="userSpaceOnUse">
                <rect x="0" y="0" width="1" height="1" fill="blue" />
                <rect x="0" y="1" width="1" height="1" fill="red" />
                <rect x="1" y="0" width="1" height="1" fill="red" />
                <rect x="1" y="1" width="1" height="1" fill="blue" />
              </pattern>
            </defs>
            <rect x="2" y="2" width="5" height="5"
                  fill="url(#pat)" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="url(#grad)" fill="transparent" opacity="0.5" />
        ''',
        '''
            <defs>
              <linearGradient id="grad" x1="0" y1="0" x2="0" y2="1"
                              gradientUnits="objectBoundingBox">
                <stop stop-color="black" offset="42.86%"></stop>
                <stop stop-color="green" offset="42.86%"></stop>
              </linearGradient>
              <pattern id="pat" x="0" y="0" width="2" height="2"
                       patternUnits="userSpaceOnUse"
                       patternContentUnits="userSpaceOnUse">
                <rect x="0" y="0" width="1" height="1" fill="blue" />
                <rect x="0" y="1" width="1" height="1" fill="red" />
                <rect x="1" y="0" width="1" height="1" fill="red" />
                <rect x="1" y="1" width="1" height="1" fill="blue" />
              </pattern>
            </defs>
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="url(#grad)" fill="url(#pat)"
                  stroke-opacity="0.5" fill-opacity="0.5" />
        ''',
    )
<commit_msg>Remove xfail for passing tests about opacity<commit_after>"""Test how opacity is handled for SVG."""

from ...testing_utils import assert_no_logs

opacity_source = '''
  <style>
    @page { size: 9px }
    svg { display: block }
  </style>
  <svg width="9px" height="9px" xmlns="http://www.w3.org/2000/svg">%s</svg>'''

@assert_no_logs
def test_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="rgb(127, 255, 127)" fill="rgb(127, 127, 255)" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" opacity="0.5" />
        ''',
    )

@assert_no_logs
def test_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" fill-opacity="0.5" />
        ''',
    )

@assert_no_logs
def test_stroke_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" opacity="0.5" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue" stroke-opacity="0.5" />
        ''',
    )

@assert_no_logs
def test_stroke_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <rect x="2" y="2" width="5" height="5"
                  fill="blue" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="transparent" opacity="0.5" />
        ''',
        '''
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="lime" fill="blue"
                  stroke-opacity="0.5" fill-opacity="0.5" />
        ''',
    )

@assert_no_logs
def test_pattern_gradient_stroke_fill_opacity(assert_same_renderings):
    assert_same_renderings(
        '''
            <defs>
              <linearGradient id="grad" x1="0" y1="0" x2="0" y2="1"
                              gradientUnits="objectBoundingBox">
                <stop stop-color="black" offset="42.86%"></stop>
                <stop stop-color="green" offset="42.86%"></stop>
              </linearGradient>
              <pattern id="pat" x="0" y="0" width="2" height="2"
                       patternUnits="userSpaceOnUse"
                       patternContentUnits="userSpaceOnUse">
                <rect x="0" y="0" width="1" height="1" fill="blue" />
                <rect x="0" y="1" width="1" height="1" fill="red" />
                <rect x="1" y="0" width="1" height="1" fill="red" />
                <rect x="1" y="1" width="1" height="1" fill="blue" />
              </pattern>
            </defs>
            <rect x="2" y="2" width="5" height="5"
                  fill="url(#pat)" opacity="0.5" />
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="url(#grad)" fill="transparent" opacity="0.5" />
        ''',
        '''
            <defs>
              <linearGradient id="grad" x1="0" y1="0" x2="0" y2="1"
                              gradientUnits="objectBoundingBox">
                <stop stop-color="black" offset="42.86%"></stop>
                <stop stop-color="green" offset="42.86%"></stop>
              </linearGradient>
              <pattern id="pat" x="0" y="0" width="2" height="2"
                       patternUnits="userSpaceOnUse"
                       patternContentUnits="userSpaceOnUse">
                <rect x="0" y="0" width="1" height="1" fill="blue" />
                <rect x="0" y="1" width="1" height="1" fill="red" />
                <rect x="1" y="0" width="1" height="1" fill="red" />
                <rect x="1" y="1" width="1" height="1" fill="blue" />
              </pattern>
            </defs>
            <rect x="2" y="2" width="5" height="5" stroke-width="2"
                  stroke="url(#grad)" fill="url(#pat)"
                  stroke-opacity="0.5" fill-opacity="0.5" />
        ''',
    )
<|endoftext|>