<commit_before>#!/usr/bin/env python

'''
Peach CI Cloud Generic Integration Runner
Copyright (c) 2016 Peach Fuzzer, LLC

This script is used when the build/test environment only
has outbound network access to Peach API Security.  This
script will create a network tunnel using SSH to allow
communication two and from the proxy.

This script provides generic integration with CI systems by
running a command that returns non-zero when testing did not pass.
The vast majority of CI systems support this method of integration.

If a specific integration is offered for your CI system that is
preferred over this generic integration.
'''

from __future__ import print_function
import logging, os, sys, re

## Configuration

# JUnit
try:
    junit = os.environ["PEACH_JUNIT"]
except:
    junit = None

# Exit code when testing passed
try:
    exit_code_ok = int(os.environ["PEACH_EXIT_CODE_OK"])
except:
    exit_code_ok = 0

# Exit code when testing failed
try:
    exit_code_failure = int(os.environ["PEACH_EXIT_CODE_FAILURE"])
except:
    exit_code_failure = 1

# Exit code when error occurred during testing
try:
    exit_code_error = int(os.environ["PEACH_EXIT_CODE_ERROR"])
except:
    exit_code_error = 100

# SSH Identify file path
try:
    peach_ssh_user = os.environ["PEACH_SSH_USER"]
except:
    print("Error, missing PEACH_SSH_USER environment variable.")
    sys.exit(exit_code_error)

# SSH Identify file path
try:
    peach_ssh_id = os.environ["PEACH_SSH_ID"]
except:
    print("Error, missing PEACH_SSH_ID environment variable.")
    sys.exit(exit_code_error)

# SSH Host
try:
    peach_ssh_host = os.environ["PEACH_SSH_HOST"]
except:
    print("Error, missing PEACH_SSH_HOST environment variable.")
    sys.exit(exit_code_error)

# SSH Port
try:
    peach_ssh_port = int(os.environ["PEACH_SSH_PORT"])
except:
    print("Error, missing PEACH_SSH_PORT environment variable.")
    sys.exit(exit_code_error)

# Test automation launch script
# Command line or None to disable
try:
    automation_cmd = os.environ["PEACH_AUTOMATION_CMD"]
except:
    print("Error, missing PEACH_AUTOMATION_CMD environment variable.")
    sys.exit(exit_code_error)

# Configuration to start
project_config = None

try:
    project_config_file = os.environ["PEACH_CONFIG"]
except:
    print("Error, missing PEACH_CONFIG environment variable.")
    sys.exit(exit_code_error)

try:
    profile = os.environ["PEACH_PROFILE"]
except:
    print("Error, missing PEACH_PROFILE environment variable.")
    sys.exit(exit_code_error)

# Peach API url
peach_api = "http://127.0.0.1:5000"

# Peach UI URL
try:
    peach_ui = os.environ["PEACH_UI"]
except:
    peach_ui = peach_api

# Verbose output
try:
    verbose = bool(os.environ["PEACH_VERBOSE"])
except:
    verbose = False

# Enable logging to syslog
try:
    syslog_host = os.environ["PEACH_SYSLOG_HOST"]

    try:
        syslog_port = int(os.environ["PEACH_SYSLOG_PORT"])
    except:
        syslog_port = 514

    syslog_enabled = True
    syslog_level = logging.ERROR
    syslog_level = logging.INFO

except:
    syslog_enabled = False
    syslog_host = 'logserver.foobar.com'
    syslog_port = 514
    syslog_level = logging.ERROR
    syslog_level = logging.INFO

###############################################################
## DO NOT EDIT BELOW THIS LINE
###############################################################

import os

try:
    import peachproxy
except:
    print("Detected missing dependency 'peachproxy' python module.")
    print("Install: pip install peachproxy")
    exit(exit_code_error)

try:
    from requests import get, post, delete
except:
    print("Detected missing dependency 'requests' python module.")
    print("Install: pip install requests")
    exit(exit_code_error)

try:
    import psutil
except:
    print("Detected missing dependency 'psutil' python module.")
    print("Install: pip install psutil")
    exit(exit_code_error)

import requests, json, sys
import subprocess, signal
import logging, logging.handlers
from time import sleep
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

logger = logging.getLogger(__name__)

logger.setLevel(syslog_level)
logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] Peach Web CI: %(message)s")

if syslog_enabled:
    syslogHandler = logging.handlers.SysLogHandler(address=(syslog_host, syslog_port))
    syslogHandler.setFormatter(logFormatter)
    logger.addHandler(syslogHandler)

consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
logger.addHandler(consoleHandler)

logger.info("Peach Web CI Cloud Generic Starting")

logger.info(" ")
logger.info("  Peach API Security UI: %s", peach_ui)
logger.info(" ")

logger.info("  automation_cmd: %s", automation_cmd)
logger.info("  exit_code_ok: %d", exit_code_ok)
logger.info("  exit_code_failure: %d", exit_code_failure)
logger.info("  exit_code_error: %d", exit_code_error)
logger.info("  peach_api: %s", peach_api)
logger.info("  project_config: %s", project_config)
logger.info("  project_config_file: %s", project_config_file)
logger.info("  junit: %s", junit)
logger.info("  syslog_enabled: %s", syslog_enabled )
if syslog_enabled:
    logger.info("  syslog_host: %s", syslog_host)
    logger.info("  syslog_port: %d", syslog_port)

test_process = None
peach_jobid = None

ssh_ctl_api = "peach-api.ssh.ctl"
ssh_ctl_proxy = "peach-proxy.ssh.ctl"
ssh_ctl_target = "peach-target.ssh.ctl"

# Load configuration file if provided
if not project_config and project_config_file:
    with open(project_config_file) as fd:
        project_config = fd.read()

try:
    from subprocess import DEVNULL
except ImportError:
    import os
    DEVNULL = open(os.devnull, 'wb')

def kill_proc_tree(pid, including_parent=True):
    '''Try and kill the pid's process tree
    '''

    parent = psutil.Process(pid)
    children = parent.children(recursive=True)
    for child in children:
        child.kill()
    psutil.wait_procs(children, timeout=5)
    if including_parent:
        parent.kill()
        parent.wait(5)

def shutdown():
    '''Perform clean shutdown of CI script
    '''

    if test_process:
        try:
            kill_proc_tree(test_process.pid)
            kill_proc_tree(test_process.pid)
            while not test_process.poll():
                os.killpg(os.getpgid(test_process.pid), signal.SIGTERM)
                test_process.terminate()
                test_process.kill()
                test_process.wait()
        except:
            pass

    if peach_jobid:
        peachproxy.session_teardown()

    # Close down any SSH tunnels
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_api, peach_ssh_host, peach_ssh_port))
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_proxy, peach_ssh_host, peach_ssh_port))
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_target, peach_ssh_host, peach_ssh_port))

def eexit(code):
    '''Close all processes and exit with 'code'
    '''

    logger.info("eexit(%d)", code)

    shutdown()

    exit(code)

import atexit
@atexit.register
def goodbye():
    '''Make sure we close out cleanly
    '''

    shutdown()

# Start SSH to Peach API

# -f  Requests ssh to go to background just before command execution
# -N  Do not execute a remote command.
# -T  Disable pseudo-terminal allocation
# -M  Places the ssh client into master mode for connection sharing
# -S  Specifies the location of a control socket for connection sharing
# -i  Selects a file from which the identity (private key) for public key authentication is read

logger.info("Port 5000")
os.system("ssh -f -i %s -p %d -N -T -M -L 5000:127.0.0.1:5000 -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, ssh_ctl_api, peach_ssh_user, peach_ssh_host))
logger.info("Port 7777/8888")
os.system("ssh -f -i %s -p %d -N -T -M -R 8888:127.0.0.1:7777 -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, ssh_ctl_target, peach_ssh_user, peach_ssh_host))

# Start job
try:
    logger.info("Starting session")
    peach_jobid = None

    peachproxy.session_setup(project_config, profile)
    peach_jobid = str(peachproxy.session_id())
    peach_proxy = str(peachproxy.proxy_url())

    proxy_port = int(re.search(":\d+", peach_proxy).group(0))
    peach_proxy = "http://127.0.0.1:%d" % proxy_port

except Exception as ex:
    logger.critical("Error starting session: %s", ex)
    eexit(exit_code_failure)

# Start SSH to Peach Proxy

logger.info("Poroxy port")
os.system("ssh -f -i %s -p %d -N -T -M -L %d:127.0.0.1:%d -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, proxy_port, proxy_port, ssh_ctl_proxy, peach_ssh_user, peach_ssh_host))

try:

    # Launch test automation

    if automation_cmd:

        logger.info("Launching test automation")
        logger.info("  PEACH_SESSIONID: %s", peach_jobid)
        logger.info("  PEACH_API: %s", peach_api)
        logger.info("  PEACH_PROXY: %s", peach_proxy)

        try:

            test_env = os.environ.copy()
            test_env["PEACH_SESSIONID"] = peach_jobid
            test_env["PEACH_API"] = peach_api
            test_env["PEACH_PROXY"] = peach_proxy

            if verbose:
                test_process = subprocess.Popen(
                    automation_cmd.split(),
                    env = test_env,
                )
            else:
                test_process = subprocess.Popen(
                    automation_cmd.split(),
                    env = test_env,
                    stdin=subprocess.PIPE, stdout=DEVNULL, stderr=subprocess.STDOUT
                )

            if not test_process:
                logger.critical("Unable to start test automation")
                eexit(exit_code_error)

            sleep(1)
            if test_process.poll():
                logger.critical("Unable to start test automation")
                eexit(exit_code_error)

        except Exception as ex:
            logger.critical("Error starting test automation: %s", ex)
            eexit(exit_code_error)

    else:
        logger.critical("No automated configured.  Currently required.")
        eexit(exit_code_error)

    # Wait for fuzzing to end

    logger.info("Waiting for job to complete")

    test_process.wait()

finally:
    logger.info("Ending session")

    ret = peachproxy.session_teardown()

    if junit:
        with open(junit, "wb+") as fd:
            fd.write(peachproxy.junit_xml())

    if ret:
        peach_jobid = None
        logger.info("Session Teardown indicated failures during testing.")
        eexit(exit_code_failure)

    eexit(exit_code_ok)

# end
<commit_msg>logging<commit_after>#!/usr/bin/env python

'''
Peach CI Cloud Generic Integration Runner
Copyright (c) 2016 Peach Fuzzer, LLC

This script is used when the build/test environment only
has outbound network access to Peach API Security.  This
script will create a network tunnel using SSH to allow
communication two and from the proxy.

This script provides generic integration with CI systems by
running a command that returns non-zero when testing did not pass.
The vast majority of CI systems support this method of integration.

If a specific integration is offered for your CI system that is
preferred over this generic integration.
'''

from __future__ import print_function
import logging, os, sys, re

## Configuration

# JUnit
try:
    junit = os.environ["PEACH_JUNIT"]
except:
    junit = None

# Exit code when testing passed
try:
    exit_code_ok = int(os.environ["PEACH_EXIT_CODE_OK"])
except:
    exit_code_ok = 0

# Exit code when testing failed
try:
    exit_code_failure = int(os.environ["PEACH_EXIT_CODE_FAILURE"])
except:
    exit_code_failure = 1

# Exit code when error occurred during testing
try:
    exit_code_error = int(os.environ["PEACH_EXIT_CODE_ERROR"])
except:
    exit_code_error = 100

# SSH Identify file path
try:
    peach_ssh_user = os.environ["PEACH_SSH_USER"]
except:
    print("Error, missing PEACH_SSH_USER environment variable.")
    sys.exit(exit_code_error)

# SSH Identify file path
try:
    peach_ssh_id = os.environ["PEACH_SSH_ID"]
except:
    print("Error, missing PEACH_SSH_ID environment variable.")
    sys.exit(exit_code_error)

# SSH Host
try:
    peach_ssh_host = os.environ["PEACH_SSH_HOST"]
except:
    print("Error, missing PEACH_SSH_HOST environment variable.")
    sys.exit(exit_code_error)

# SSH Port
try:
    peach_ssh_port = int(os.environ["PEACH_SSH_PORT"])
except:
    print("Error, missing PEACH_SSH_PORT environment variable.")
    sys.exit(exit_code_error)

# Test automation launch script
# Command line or None to disable
try:
    automation_cmd = os.environ["PEACH_AUTOMATION_CMD"]
except:
    print("Error, missing PEACH_AUTOMATION_CMD environment variable.")
    sys.exit(exit_code_error)

# Configuration to start
project_config = None

try:
    project_config_file = os.environ["PEACH_CONFIG"]
except:
    print("Error, missing PEACH_CONFIG environment variable.")
    sys.exit(exit_code_error)

try:
    profile = os.environ["PEACH_PROFILE"]
except:
    print("Error, missing PEACH_PROFILE environment variable.")
    sys.exit(exit_code_error)

# Peach API url
peach_api = "http://127.0.0.1:5001"

# Peach UI URL
try:
    peach_ui = os.environ["PEACH_UI"]
except:
    peach_ui = peach_api

# Verbose output
try:
    verbose = bool(os.environ["PEACH_VERBOSE"])
except:
    verbose = False

# Enable logging to syslog
try:
    syslog_host = os.environ["PEACH_SYSLOG_HOST"]

    try:
        syslog_port = int(os.environ["PEACH_SYSLOG_PORT"])
    except:
        syslog_port = 514

    syslog_enabled = True
    syslog_level = logging.ERROR
    syslog_level = logging.INFO

except:
    syslog_enabled = False
    syslog_host = 'logserver.foobar.com'
    syslog_port = 514
    syslog_level = logging.ERROR
    syslog_level = logging.INFO

###############################################################
## DO NOT EDIT BELOW THIS LINE
###############################################################

import os

try:
    import peachproxy
except:
    print("Detected missing dependency 'peachproxy' python module.")
    print("Install: pip install peachproxy")
    exit(exit_code_error)

try:
    from requests import get, post, delete
except:
    print("Detected missing dependency 'requests' python module.")
    print("Install: pip install requests")
    exit(exit_code_error)

try:
    import psutil
except:
    print("Detected missing dependency 'psutil' python module.")
    print("Install: pip install psutil")
    exit(exit_code_error)

import requests, json, sys
import subprocess, signal
import logging, logging.handlers
from time import sleep
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

logger = logging.getLogger(__name__)

logger.setLevel(syslog_level)
logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] Peach Web CI: %(message)s")

if syslog_enabled:
    syslogHandler = logging.handlers.SysLogHandler(address=(syslog_host, syslog_port))
    syslogHandler.setFormatter(logFormatter)
    logger.addHandler(syslogHandler)

consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
logger.addHandler(consoleHandler)

logger.info("Peach Web CI Cloud Generic Starting")

logger.info(" ")
logger.info("  Peach API Security UI: %s", peach_ui)
logger.info(" ")

logger.info("  automation_cmd: %s", automation_cmd)
logger.info("  exit_code_ok: %d", exit_code_ok)
logger.info("  exit_code_failure: %d", exit_code_failure)
logger.info("  exit_code_error: %d", exit_code_error)
logger.info("  peach_api: %s", peach_api)
logger.info("  project_config: %s", project_config)
logger.info("  project_config_file: %s", project_config_file)
logger.info("  junit: %s", junit)
logger.info("  syslog_enabled: %s", syslog_enabled )
if syslog_enabled:
    logger.info("  syslog_host: %s", syslog_host)
    logger.info("  syslog_port: %d", syslog_port)

test_process = None
peach_jobid = None

ssh_ctl_api = "peach-api.ssh.ctl"
ssh_ctl_proxy = "peach-proxy.ssh.ctl"
ssh_ctl_target = "peach-target.ssh.ctl"

# Load configuration file if provided
if not project_config and project_config_file:
    with open(project_config_file) as fd:
        project_config = fd.read()

try:
    from subprocess import DEVNULL
except ImportError:
    import os
    DEVNULL = open(os.devnull, 'wb')

def kill_proc_tree(pid, including_parent=True):
    '''Try and kill the pid's process tree
    '''

    parent = psutil.Process(pid)
    children = parent.children(recursive=True)
    for child in children:
        child.kill()
    psutil.wait_procs(children, timeout=5)
    if including_parent:
        parent.kill()
        parent.wait(5)

def shutdown():
    '''Perform clean shutdown of CI script
    '''

    if test_process:
        try:
            kill_proc_tree(test_process.pid)
            kill_proc_tree(test_process.pid)
            while not test_process.poll():
                os.killpg(os.getpgid(test_process.pid), signal.SIGTERM)
                test_process.terminate()
                test_process.kill()
                test_process.wait()
        except:
            pass

    if peach_jobid:
        peachproxy.session_teardown()

    # Close down any SSH tunnels
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_api, peach_ssh_host, peach_ssh_port))
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_proxy, peach_ssh_host, peach_ssh_port))
    os.system("ssh -T -O \"exit\" -S %s %s:%d" % (ssh_ctl_target, peach_ssh_host, peach_ssh_port))

def eexit(code):
    '''Close all processes and exit with 'code'
    '''

    logger.info("eexit(%d)", code)

    shutdown()

    exit(code)

import atexit
@atexit.register
def goodbye():
    '''Make sure we close out cleanly
    '''

    shutdown()

# Start SSH to Peach API

# -f  Requests ssh to go to background just before command execution
# -N  Do not execute a remote command.
# -T  Disable pseudo-terminal allocation
# -M  Places the ssh client into master mode for connection sharing
# -S  Specifies the location of a control socket for connection sharing
# -i  Selects a file from which the identity (private key) for public key authentication is read

logger.info("Port 5000")
os.system("ssh -f -i %s -p %d -N -T -M -L 5001:127.0.0.1:5000 -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, ssh_ctl_api, peach_ssh_user, peach_ssh_host))
logger.info("Port 7777/8888")
os.system("ssh -f -i %s -p %d -N -T -M -R 8888:127.0.0.1:7777 -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, ssh_ctl_target, peach_ssh_user, peach_ssh_host))

# Start job
try:
    logger.info("Starting session")
    peach_jobid = None

    peachproxy.session_setup(project_config, profile)
    peach_jobid = str(peachproxy.session_id())
    peach_proxy = str(peachproxy.proxy_url())

    proxy_port = int(re.search(":\d+", peach_proxy).group(0))
    peach_proxy = "http://127.0.0.1:%d" % proxy_port

except Exception as ex:
    logger.critical("Error starting session: %s", ex)
    eexit(exit_code_failure)

# Start SSH to Peach Proxy

logger.info("Poroxy port")
os.system("ssh -f -i %s -p %d -N -T -M -L %d:127.0.0.1:%d -S %s %s@%s" % (peach_ssh_id, peach_ssh_port, proxy_port, proxy_port, ssh_ctl_proxy, peach_ssh_user, peach_ssh_host))

try:

    # Launch test automation

    if automation_cmd:

        logger.info("Launching test automation")
        logger.info("  PEACH_SESSIONID: %s", peach_jobid)
        logger.info("  PEACH_API: %s", peach_api)
        logger.info("  PEACH_PROXY: %s", peach_proxy)

        try:

            test_env = os.environ.copy()
            test_env["PEACH_SESSIONID"] = peach_jobid
            test_env["PEACH_API"] = peach_api
            test_env["PEACH_PROXY"] = peach_proxy

            if verbose:
                test_process = subprocess.Popen(
                    automation_cmd.split(),
                    env = test_env,
                )
            else:
                test_process = subprocess.Popen(
                    automation_cmd.split(),
                    env = test_env,
                    stdin=subprocess.PIPE, stdout=DEVNULL, stderr=subprocess.STDOUT
                )

            if not test_process:
                logger.critical("Unable to start test automation")
                eexit(exit_code_error)

            sleep(1)
            if test_process.poll():
                logger.critical("Unable to start test automation")
                eexit(exit_code_error)

        except Exception as ex:
            logger.critical("Error starting test automation: %s", ex)
            eexit(exit_code_error)

    else:
        logger.critical("No automated configured.  Currently required.")
        eexit(exit_code_error)

    # Wait for fuzzing to end

    logger.info("Waiting for job to complete")

    test_process.wait()

finally:
    logger.info("Ending session")

    ret = peachproxy.session_teardown()

    if junit:
        with open(junit, "wb+") as fd:
            fd.write(peachproxy.junit_xml())

    if ret:
        peach_jobid = None
        logger.info("Session Teardown indicated failures during testing.")
        eexit(exit_code_failure)

    eexit(exit_code_ok)

# end
<|endoftext|>


<commit_before>
import logging
import time

from archinfo.arch_soot import (ArchSoot, SootAddressDescriptor,
                                SootMethodDescriptor)

from .. import Backend
from ...errors import CLEError

try:
    import pysoot
    from pysoot.lifter import Lifter
except ImportError:
    pysoot = None

l = logging.getLogger(name=__name__)

class Soot(Backend):
    """
    The basis backend for lifting and loading bytecode from JARs and APKs to Soot IR.
    """

    def __init__(self, path, entry_point=None, entry_point_params=(), input_format=None,
                 additional_jars=None, additional_jar_roots=None,
                 jni_libs_ld_path=None, jni_libs=None,
                 android_sdk=None, **kwargs):

        if not pysoot:
            raise ImportError('Cannot import PySoot. The Soot backend requires PySoot.')

        if kwargs.get('has_memory', False):
            raise CLEError('The parameter "has_memory" must be False for Soot backend.')

        super(Soot, self).__init__(path, has_memory=False, **kwargs)

        # load the classes
        l.debug("Lifting to Soot IR ...")
        start_time = time.time()
        pysoot_lifter = Lifter(path,
                               input_format=input_format,
                               android_sdk=android_sdk,
                               additional_jars=additional_jars,
                               additional_jar_roots=additional_jar_roots)
        end_time = time.time()
        l.debug("Lifting completed in %ds", round(end_time - start_time, 2))
        self._classes = pysoot_lifter.classes

        # find entry method
        if entry_point:
            try:
                ep_method = self.get_soot_method(entry_point, params=entry_point_params)
                ep_method_descriptor = SootMethodDescriptor.from_soot_method(ep_method)
                self._entry = SootAddressDescriptor(ep_method_descriptor, 0, 0)
                l.debug("Entry point set to %s", self._entry)
            except CLEError:
                l.warning("Couldn't find entry point %s.", entry_point)
                self._entry = None

        self.os = 'javavm'
        self.rebase_addr = None
        self.set_arch(ArchSoot())

        if jni_libs:
            # native libraries are getting loaded by adding them as a dependency of this object
            self.deps += [jni_libs] if type(jni_libs) in (str, bytes) else jni_libs
            # if available, add additional load path(s)
            if jni_libs_ld_path:
                path_list = [jni_libs_ld_path] if type(jni_libs_ld_path) in (str, bytes) else jni_libs_ld_path
                self.extra_load_path += path_list
            self.jni_support = True
        else:
            self.jni_support = False

    @property
    def max_addr(self):
        # FIXME: This is a hack to satisfy checks elsewhere that max_addr must be greater than min_addr
        return self.min_addr + 1

    @property
    def entry(self):
        return self._entry

    @property
    def classes(self):
        return self._classes

    def get_soot_class(self, cls_name, none_if_missing=False):
        """
        Get Soot class object.

        :param str cls_name: Name of the class.
        :return:             The class object.
        :rtype:              pysoot.soot.SootClass
        """
        try:
            return self._classes[cls_name]
        except KeyError:
            if none_if_missing:
                return None
            else:
                raise CLEError('Class "%s" does not exist.' % cls_name)

    def get_soot_method(self, thing, class_name=None, params=(), none_if_missing=False):
        """
        Get Soot method object.

        :param thing:           Descriptor or the method, or name of the method.
        :param str class_name:  Name of the class. If not specified, class name can be parsed from method_name.
        :return:                Soot method that satisfy the criteria.
        """

        # Step 1: Parse input
        if isinstance(thing, SootMethodDescriptor):
            method_description = {
                'class_name': thing.class_name,
                'name': thing.name,
                'params': thing.params,
            }

        elif isinstance(thing, (str, bytes)):
            method_name = thing

            # if class_name is not set, parse it from the method name
            if class_name is None:
                last_dot = method_name.rfind('.')
                if last_dot >= 0:
                    class_name = method_name[:last_dot]
                    method_name = method_name[last_dot + 1:]
                else:
                    raise ValueError('Cannot parse class name from method %s.' % method_name)

            method_description = {
                'class_name': class_name,
                'name': method_name,
                'params': params,
            }

        else:
            raise TypeError('Unsupported type "%s" for the first argument.' % thing)

        # Step 2: Load class containing the method
        try:
            cls = self.get_soot_class(method_description['class_name'])
        except CLEError:
            if none_if_missing:
                return None
            else:
                raise

        # Step 3: Get all methods matching the description
        methods = [soot_method for soot_method in cls.methods
                   if self._description_matches_soot_method(soot_method, **method_description)]

        if not methods:
            if none_if_missing:
                return None
            else:
                raise CLEError('Method with description %s does not exist in class %s.'
                               % (method_description, method_description['class_name']))

        if len(methods) > 1:
            # Warn if we found several matching methods
            l.warning('Method with description %s is ambiguous in class %s.',
                       method_description, method_description['class_name'])

        return methods[0]

    @staticmethod
    def _description_matches_soot_method(soot_method, name=None, class_name=None, params=()):
        if name       and soot_method.name != name:              return False
        if class_name and soot_method.class_name != class_name:  return False
        if soot_method.params != params:          return False
        return True

    @property
    def main_methods(self):
        """
        Find all Main methods in this binary.

        :return: All main methods in each class.
        :rtype:  iterator
        """
        for cls in self.classes.values():
            for method in cls.methods:
                if method.name == "main":  # TODO: Check more attributes
                    yield method

    @staticmethod
    def is_zip_archive(stream):
        identstring = stream.read(4)
        stream.seek(0)
        return identstring.startswith(b'\x50\x4b\x03\x04')
<commit_msg>Fix test if file is a zip archive (#170)<commit_after>
import logging
import time

from archinfo.arch_soot import (ArchSoot, SootAddressDescriptor,
                                SootMethodDescriptor)

from .. import Backend
from ...errors import CLEError

try:
    import pysoot
    from pysoot.lifter import Lifter
except ImportError:
    pysoot = None

l = logging.getLogger(name=__name__)

class Soot(Backend):
    """
    The basis backend for lifting and loading bytecode from JARs and APKs to Soot IR.
    """

    def __init__(self, path, entry_point=None, entry_point_params=(), input_format=None,
                 additional_jars=None, additional_jar_roots=None,
                 jni_libs_ld_path=None, jni_libs=None,
                 android_sdk=None, **kwargs):

        if not pysoot:
            raise ImportError('Cannot import PySoot. The Soot backend requires PySoot.')

        if kwargs.get('has_memory', False):
            raise CLEError('The parameter "has_memory" must be False for Soot backend.')

        super(Soot, self).__init__(path, has_memory=False, **kwargs)

        # load the classes
        l.debug("Lifting to Soot IR ...")
        start_time = time.time()
        pysoot_lifter = Lifter(path,
                               input_format=input_format,
                               android_sdk=android_sdk,
                               additional_jars=additional_jars,
                               additional_jar_roots=additional_jar_roots)
        end_time = time.time()
        l.debug("Lifting completed in %ds", round(end_time - start_time, 2))
        self._classes = pysoot_lifter.classes

        # find entry method
        if entry_point:
            try:
                ep_method = self.get_soot_method(entry_point, params=entry_point_params)
                ep_method_descriptor = SootMethodDescriptor.from_soot_method(ep_method)
                self._entry = SootAddressDescriptor(ep_method_descriptor, 0, 0)
                l.debug("Entry point set to %s", self._entry)
            except CLEError:
                l.warning("Couldn't find entry point %s.", entry_point)
                self._entry = None

        self.os = 'javavm'
        self.rebase_addr = None
        self.set_arch(ArchSoot())

        if jni_libs:
            # native libraries are getting loaded by adding them as a dependency of this object
            self.deps += [jni_libs] if type(jni_libs) in (str, bytes) else jni_libs
            # if available, add additional load path(s)
            if jni_libs_ld_path:
                path_list = [jni_libs_ld_path] if type(jni_libs_ld_path) in (str, bytes) else jni_libs_ld_path
                self.extra_load_path += path_list
            self.jni_support = True
        else:
            self.jni_support = False

    @property
    def max_addr(self):
        # FIXME: This is a hack to satisfy checks elsewhere that max_addr must be greater than min_addr
        return self.min_addr + 1

    @property
    def entry(self):
        return self._entry

    @property
    def classes(self):
        return self._classes

    def get_soot_class(self, cls_name, none_if_missing=False):
        """
        Get Soot class object.

        :param str cls_name: Name of the class.
        :return:             The class object.
        :rtype:              pysoot.soot.SootClass
        """
        try:
            return self._classes[cls_name]
        except KeyError:
            if none_if_missing:
                return None
            else:
                raise CLEError('Class "%s" does not exist.' % cls_name)

    def get_soot_method(self, thing, class_name=None, params=(), none_if_missing=False):
        """
        Get Soot method object.

        :param thing:           Descriptor or the method, or name of the method.
        :param str class_name:  Name of the class. If not specified, class name can be parsed from method_name.
        :return:                Soot method that satisfy the criteria.
        """

        # Step 1: Parse input
        if isinstance(thing, SootMethodDescriptor):
            method_description = {
                'class_name': thing.class_name,
                'name': thing.name,
                'params': thing.params,
            }

        elif isinstance(thing, (str, bytes)):
            method_name = thing

            # if class_name is not set, parse it from the method name
            if class_name is None:
                last_dot = method_name.rfind('.')
                if last_dot >= 0:
                    class_name = method_name[:last_dot]
                    method_name = method_name[last_dot + 1:]
                else:
                    raise ValueError('Cannot parse class name from method %s.' % method_name)

            method_description = {
                'class_name': class_name,
                'name': method_name,
                'params': params,
            }

        else:
            raise TypeError('Unsupported type "%s" for the first argument.' % thing)

        # Step 2: Load class containing the method
        try:
            cls = self.get_soot_class(method_description['class_name'])
        except CLEError:
            if none_if_missing:
                return None
            else:
                raise

        # Step 3: Get all methods matching the description
        methods = [soot_method for soot_method in cls.methods
                   if self._description_matches_soot_method(soot_method, **method_description)]

        if not methods:
            if none_if_missing:
                return None
            else:
                raise CLEError('Method with description %s does not exist in class %s.'
                               % (method_description, method_description['class_name']))

        if len(methods) > 1:
            # Warn if we found several matching methods
            l.warning('Method with description %s is ambiguous in class %s.',
                       method_description, method_description['class_name'])

        return methods[0]

    @staticmethod
    def _description_matches_soot_method(soot_method, name=None, class_name=None, params=()):
        if name       and soot_method.name != name:              return False
        if class_name and soot_method.class_name != class_name:  return False
        if soot_method.params != params:          return False
        return True

    @property
    def main_methods(self):
        """
        Find all Main methods in this binary.

        :return: All main methods in each class.
        :rtype:  iterator
        """
        for cls in self.classes.values():
            for method in cls.methods:
                if method.name == "main":  # TODO: Check more attributes
                    yield method

    @staticmethod
    def is_zip_archive(stream):
        stream.seek(0)
        identstring = stream.read(4)
        stream.seek(0)
        return identstring.startswith(b'\x50\x4b\x03\x04')
<|endoftext|>


<commit_before>import pytest
import numpy as np
from numpy.testing import assert_allclose

from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, RepeatVector, TimeDistributed
from keras.layers import Input
from keras import optimizers
from keras import objectives
from keras import metrics
from keras.utils.test_utils import keras_test
from keras.models import save_model, load_model

@keras_test
def test_sequential_model_saving():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_sequential_model_saving_2():
    # test with funkier config
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(RepeatVector(3))
    model.add(TimeDistributed(Dense(3)))
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy],
                  sample_weight_mode='temporal')
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_sequential_model_saving_3():
    # test with custom optimizer, loss
    custom_opt = optimizers.rmsprop
    custom_loss = objectives.mse
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss=custom_loss, optimizer=custom_opt(), metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname,
                       custom_objects={'custom_opt': custom_opt,
                                       'custom_loss': custom_loss})
    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_fuctional_model_saving():
    input = Input(shape=(3,))
    x = Dense(2)(input)
    output = Dense(3)(x)

    model = Model(input, output)
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy])
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname)
    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_saving_without_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)

@keras_test
def test_saving_right_after_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
    model.model._make_train_function()

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)

if __name__ == '__main__':
    pytest.main([__file__])
<commit_msg>Clean up test files<commit_after>import pytest
import os
import numpy as np
from numpy.testing import assert_allclose

from keras.models import Model, Sequential
from keras.layers import Dense, Dropout, RepeatVector, TimeDistributed
from keras.layers import Input
from keras import optimizers
from keras import objectives
from keras import metrics
from keras.utils.test_utils import keras_test
from keras.models import save_model, load_model

@keras_test
def test_sequential_model_saving():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    os.remove(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_sequential_model_saving_2():
    # test with funkier config
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(RepeatVector(3))
    model.add(TimeDistributed(Dense(3)))
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy],
                  sample_weight_mode='temporal')
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    new_model = load_model(fname)
    os.remove(fname)

    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

    # test that new updates are the same with both models
    x = np.random.random((1, 3))
    y = np.random.random((1, 3, 3))
    model.train_on_batch(x, y)
    new_model.train_on_batch(x, y)
    out = model.predict(x)
    out2 = new_model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_sequential_model_saving_3():
    # test with custom optimizer, loss
    custom_opt = optimizers.rmsprop
    custom_loss = objectives.mse
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss=custom_loss, optimizer=custom_opt(), metrics=['acc'])

    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname,
                       custom_objects={'custom_opt': custom_opt,
                                       'custom_loss': custom_loss})
    os.remove(fname)

    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_fuctional_model_saving():
    input = Input(shape=(3,))
    x = Dense(2)(input)
    output = Dense(3)(x)

    model = Model(input, output)
    model.compile(loss=objectives.MSE,
                  optimizer=optimizers.RMSprop(lr=0.0001),
                  metrics=[metrics.categorical_accuracy])
    x = np.random.random((1, 3))
    y = np.random.random((1, 3))
    model.train_on_batch(x, y)

    out = model.predict(x)
    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)

    model = load_model(fname)
    os.remove(fname)

    out2 = model.predict(x)
    assert_allclose(out, out2, atol=1e-05)

@keras_test
def test_saving_without_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)
    os.remove(fname)

@keras_test
def test_saving_right_after_compilation():
    model = Sequential()
    model.add(Dense(2, input_dim=3))
    model.add(Dense(3))
    model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
    model.model._make_train_function()

    fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
    save_model(model, fname)
    model = load_model(fname)
    os.remove(fname)

if __name__ == '__main__':
    pytest.main([__file__])
<|endoftext|>


<commit_before># Copyright (c) 2010 gocept gmbh & co. kg
# See also LICENSE.txt

import mock
import unittest2
import zeit.cms.testing
import zeit.content.article.testing

class GalleryTest(unittest2.TestCase,
                  zeit.cms.testing.BrowserAssertions):

    layer = zeit.content.article.testing.TestBrowserLayer

    expected_type = 'gallery'
    attribute = 'href'

    def setUp(self):
        from zope.testbrowser.testing import Browser
        self.browser = browser = Browser()
        browser.addHeader('Authorization', 'Basic user:userpw')
        browser.open('http://localhost:8080/++skin++vivi/repository/online'
                     '/2007/01/Somalia/@@checkout')
        self.article_url = browser.url
        browser.open('@@contents')
        self.contents_url = browser.url
        browser.open(self.contents_url)

    def setup_content(self):
        """Create a gallery (requires folder with images)"""
        from zeit.content.gallery.browser.testing import add_folder, add_image
        browser = self.browser
        browser.open('http://localhost/++skin++cms/repository/online/2007/01')
        _ = add_folder(browser, 'gallery')
        _ = add_image(browser, '01.jpg')
        _ = add_image(browser, '02.jpg')
        browser.getLink('01').click()
        menu = browser.getControl(name='add_menu')
        menu.displayValue = ['Gallery']
        browser.open(menu.value[0])
        browser.getControl('File name').value = 'island'
        browser.getControl('Title').value = 'Auf den Spuren der Elfen'
        browser.getControl('Ressort').displayValue = ['Reisen']
        browser.getControl(name="form.image_folder").value = (
            'http://xml.zeit.de/online/2007/01/gallery')
        browser.getControl(name='form.authors.0.').value = 'Hans Sachs'
        browser.getControl(name="form.actions.add").click()
        browser.getLink('Checkin').click()
        self.content_id = 'http://xml.zeit.de/online/2007/01/island'
        browser.open(self.contents_url)

    def get_article(self, with_empty_block=False):
        import transaction
        article = self.layer.setup.getRootFolder()[
            'workingcopy']['zope.user']['Somalia']
        for p in article.xml.findall('//division/*'):
            p.getparent().remove(p)
        if with_empty_block:
            article.xml.body.division[self.expected_type] = ''
            article.xml.body.division[self.expected_type].set(
                '{http://namespaces.zeit.de/CMS/cp}__name__', 'blockname')
        article._p_changed = True
        transaction.commit()
        return article

    def test_no_reference_should_render(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open('editable-body/@@contents')
        self.assert_ellipsis(
            """<...
            <div ...class="block type-%s...""" % self.expected_type)

    def test_empty_block_should_be_landing_zone(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId=%s' %
            self.content_id)
        self.assert_json(
            {'signals': [{'args': [
                'blockname',
                'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/blockname/@@contents'],
                'name': 'reload',
                'when': None}]})
        self.browser.open('@@contents')
        self.assert_ellipsis(
            """<div ...class="block type-%s...""" % self.expected_type)

    def test_only_specific_type_should_be_droppable(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        with self.assertRaises(Exception):
            self.browser.open(
                'editable-body/blockname/@@set_reference?uniqueId='
                'http://xml.zeit.de/testcontent')
        # NOTE: the error message should be improved
        self.assert_ellipsis(
            "ConstraintNotSatisfied(<zeit.cms.repository.unknown."
            "PersistentUnknownResource object at 0x...>)")

    def test_droping_type_on_landing_zone_creates_block(self):
        article = self.get_article()
        self.setup_content()
        self.assert_ellipsis(
            """...cms:drop-url="http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@article-landing-zone-drop"...""")
        uuid4 = mock.Mock(side_effect=lambda: uuid4.call_count)
        with mock.patch('uuid.uuid4', new=uuid4):
            self.browser.open(
                'editable-body/@@article-landing-zone-drop?uniqueId=%s' %
                self.content_id)
        data = self.assert_json(
            {'signals': [{'args': ['1'], 'name': 'added', 'when': 'after-reload'},
                         {'args': ['editable-body',
                                   'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@contents'],
                          'name': 'reload',
                          'when': None}]})
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """<...
                <div ...class="block type-%s...""" % self.expected_type)
        # Each block has its own landing zone:
        id = data['signals'][0]['args'][0]
        with mock.patch('uuid.uuid4', new=uuid4):
            self.browser.open(
                'editable-body/%s/@@article-landing-zone-drop?uniqueId=%s' % (
                    id, self.content_id))
        self.assert_json(
            {'signals': [{'args': ['2'], 'name': 'added', 'when': 'after-reload'},
                         {'args': ['editable-body',
                                   'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@contents'],
                          'name': 'reload',
                          'when': None}]})

    def test_checkin_should_work_with_empty_block(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.article_url)
        # Assertion is that no error is raised
        self.browser.handleErrors = False
        self.browser.getLink('Checkin').click()

    def test_should_be_visible_in_read_only_mode(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId=%s' %
            self.content_id)
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        self.browser.open('@@contents')
        self.assert_ellipsis(
            """<div ...class="block type-%s...""" % self.expected_type)

    def test_reference_should_be_stored_by_set_ref(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.assertEqual(
            self.content_id,
            article.xml.body.division[self.expected_type].get(self.attribute))

    def test_reference_should_be_stored_after_checkin(self):
        import zeit.cms.interfaces
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        with zeit.cms.testing.site(self.layer.setup.getRootFolder()):
            article = zeit.cms.interfaces.ICMSContent(article.uniqueId)
        self.assertEqual(
            self.content_id,
            article.xml.body.division[self.expected_type].get(self.attribute))

    def test_empty_block_should_not_provide_drop_in_readonly_mode(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        self.browser.open('@@contents')
        self.assert_ellipsis(
            '<div ...class="block type-{0}...No content referenced...'.format(
                self.expected_type))

    def test_contents_should_not_have_cache_control_header(self):
        self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.browser.open('@@contents')
        self.assertNotIn('Cache-Control', self.browser.headers)

class InfoboxTest(GalleryTest):

    expected_type = 'infobox'

    def setup_content(self):
        from zeit.content.infobox.infobox import Infobox
        import transaction
        ib = Infobox()
        ib.supertitle = u'infobox title'
        root = self.layer.setup.getRootFolder()
        with zeit.cms.testing.site(root):
            root['repository']['infobox'] = ib
        self.content_id = 'http://xml.zeit.de/infobox'
        transaction.commit()

class PortraitboxTest(GalleryTest):

    expected_type = 'portraitbox'

    def setup_content(self):
        from zeit.content.portraitbox.portraitbox import Portraitbox
        import transaction
        import zeit.cms.testing
        pb = Portraitbox()
        root = self.layer.setup.getRootFolder()
        with zeit.cms.testing.site(root):
            with zeit.cms.testing.interaction():
                root['repository']['pb'] = pb
        self.content_id = 'http://xml.zeit.de/pb'
        transaction.commit()

    def test_layout_should_be_editable(self):
        self.setup_content()
        article = self.get_article(with_empty_block=True)
        article.xml.body.division[self.expected_type].set(
            'href', self.content_id)
        article._p_changed = True
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'short', 'wide'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['wide']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="wide ...""")

class ImageTest(GalleryTest):

    expected_type = 'image'
    attribute = 'src'

    def setup_content(self):
        self.content_id = 'http://xml.zeit.de/2006/DSC00109_2.JPG'

    def test_only_specific_type_should_be_droppable(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        with self.assertRaises(Exception):
            self.browser.open(
                'editable-body/blockname/@@set_reference?uniqueId='
                'http://xml.zeit.de/testcontent')
        # NOTE: the error message should be improved
        self.assert_ellipsis("ComponentLookupError...")

    def test_layout_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'small', 'large', 'Infobox', 'Hochkant'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['large']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="large ...""")

    def test_empty_block_should_not_provide_drop_in_readonly_mode(self):
        # Disable inherited test because empty images are removed during
        # checkin. This will change hopefully once #8194 is implemented.
        pass

class VideoTest(GalleryTest):

    layer = zeit.content.article.testing.ArticleBrightcoveLayer
    expected_type = 'video'

    def setup_content(self):
        self.content_id = 'http://video.zeit.de/video/1234'

    def test_layout_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'small', 'with info', 'large', 'double'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['large']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="large ...""")

    def test_two_videos_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.browser.getControl('Video', index=0).value = (
            'http://video.zeit.de/video/1234')
        self.browser.getControl('Video 2').value = (
            'http://video.zeit.de/video/6789')
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...
               ......""")
<commit_msg>Fix test, TestContentType has a proper type declaration now<commit_after># Copyright (c) 2010 gocept gmbh & co. kg
# See also LICENSE.txt

import mock
import unittest2
import zeit.cms.testing
import zeit.content.article.testing

class GalleryTest(unittest2.TestCase,
                  zeit.cms.testing.BrowserAssertions):

    layer = zeit.content.article.testing.TestBrowserLayer

    expected_type = 'gallery'
    attribute = 'href'

    def setUp(self):
        from zope.testbrowser.testing import Browser
        self.browser = browser = Browser()
        browser.addHeader('Authorization', 'Basic user:userpw')
        browser.open('http://localhost:8080/++skin++vivi/repository/online'
                     '/2007/01/Somalia/@@checkout')
        self.article_url = browser.url
        browser.open('@@contents')
        self.contents_url = browser.url
        browser.open(self.contents_url)

    def setup_content(self):
        """Create a gallery (requires folder with images)"""
        from zeit.content.gallery.browser.testing import add_folder, add_image
        browser = self.browser
        browser.open('http://localhost/++skin++cms/repository/online/2007/01')
        _ = add_folder(browser, 'gallery')
        _ = add_image(browser, '01.jpg')
        _ = add_image(browser, '02.jpg')
        browser.getLink('01').click()
        menu = browser.getControl(name='add_menu')
        menu.displayValue = ['Gallery']
        browser.open(menu.value[0])
        browser.getControl('File name').value = 'island'
        browser.getControl('Title').value = 'Auf den Spuren der Elfen'
        browser.getControl('Ressort').displayValue = ['Reisen']
        browser.getControl(name="form.image_folder").value = (
            'http://xml.zeit.de/online/2007/01/gallery')
        browser.getControl(name='form.authors.0.').value = 'Hans Sachs'
        browser.getControl(name="form.actions.add").click()
        browser.getLink('Checkin').click()
        self.content_id = 'http://xml.zeit.de/online/2007/01/island'
        browser.open(self.contents_url)

    def get_article(self, with_empty_block=False):
        import transaction
        article = self.layer.setup.getRootFolder()[
            'workingcopy']['zope.user']['Somalia']
        for p in article.xml.findall('//division/*'):
            p.getparent().remove(p)
        if with_empty_block:
            article.xml.body.division[self.expected_type] = ''
            article.xml.body.division[self.expected_type].set(
                '{http://namespaces.zeit.de/CMS/cp}__name__', 'blockname')
        article._p_changed = True
        transaction.commit()
        return article

    def test_no_reference_should_render(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open('editable-body/@@contents')
        self.assert_ellipsis(
            """<...
            <div ...class="block type-%s...""" % self.expected_type)

    def test_empty_block_should_be_landing_zone(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId=%s' %
            self.content_id)
        self.assert_json(
            {'signals': [{'args': [
                'blockname',
                'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/blockname/@@contents'],
                'name': 'reload',
                'when': None}]})
        self.browser.open('@@contents')
        self.assert_ellipsis(
            """<div ...class="block type-%s...""" % self.expected_type)

    def test_only_specific_type_should_be_droppable(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        with self.assertRaises(Exception):
            self.browser.open(
                'editable-body/blockname/@@set_reference?uniqueId='
                'http://xml.zeit.de/testcontent')
        # NOTE: the error message should be improved
        self.assert_ellipsis(
            "ConstraintNotSatisfied(<zeit.cms.testcontenttype..."
            "object at 0x...>)")

    def test_droping_type_on_landing_zone_creates_block(self):
        article = self.get_article()
        self.setup_content()
        self.assert_ellipsis(
            """...cms:drop-url="http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@article-landing-zone-drop"...""")
        uuid4 = mock.Mock(side_effect=lambda: uuid4.call_count)
        with mock.patch('uuid.uuid4', new=uuid4):
            self.browser.open(
                'editable-body/@@article-landing-zone-drop?uniqueId=%s' %
                self.content_id)
        data = self.assert_json(
            {'signals': [{'args': ['1'], 'name': 'added', 'when': 'after-reload'},
                         {'args': ['editable-body',
                                   'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@contents'],
                          'name': 'reload',
                          'when': None}]})
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """<...
                <div ...class="block type-%s...""" % self.expected_type)
        # Each block has its own landing zone:
        id = data['signals'][0]['args'][0]
        with mock.patch('uuid.uuid4', new=uuid4):
            self.browser.open(
                'editable-body/%s/@@article-landing-zone-drop?uniqueId=%s' % (
                    id, self.content_id))
        self.assert_json(
            {'signals': [{'args': ['2'], 'name': 'added', 'when': 'after-reload'},
                         {'args': ['editable-body',
                                   'http://localhost:8080/++skin++vivi/workingcopy/zope.user/Somalia/editable-body/@@contents'],
                          'name': 'reload',
                          'when': None}]})

    def test_checkin_should_work_with_empty_block(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.article_url)
        # Assertion is that no error is raised
        self.browser.handleErrors = False
        self.browser.getLink('Checkin').click()

    def test_should_be_visible_in_read_only_mode(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId=%s' %
            self.content_id)
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        self.browser.open('@@contents')
        self.assert_ellipsis(
            """<div ...class="block type-%s...""" % self.expected_type)

    def test_reference_should_be_stored_by_set_ref(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.assertEqual(
            self.content_id,
            article.xml.body.division[self.expected_type].get(self.attribute))

    def test_reference_should_be_stored_after_checkin(self):
        import zeit.cms.interfaces
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        with zeit.cms.testing.site(self.layer.setup.getRootFolder()):
            article = zeit.cms.interfaces.ICMSContent(article.uniqueId)
        self.assertEqual(
            self.content_id,
            article.xml.body.division[self.expected_type].get(self.attribute))

    def test_empty_block_should_not_provide_drop_in_readonly_mode(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.article_url)
        self.browser.getLink('Checkin').click()
        self.browser.open('@@contents')
        self.assert_ellipsis(
            '<div ...class="block type-{0}...No content referenced...'.format(
                self.expected_type))

    def test_contents_should_not_have_cache_control_header(self):
        self.get_article(with_empty_block=True)
        self.setup_content()
        self.browser.open(
            'editable-body/blockname/@@set_reference?uniqueId={0}'.format(
                self.content_id))
        self.browser.open('@@contents')
        self.assertNotIn('Cache-Control', self.browser.headers)

class InfoboxTest(GalleryTest):

    expected_type = 'infobox'

    def setup_content(self):
        from zeit.content.infobox.infobox import Infobox
        import transaction
        ib = Infobox()
        ib.supertitle = u'infobox title'
        root = self.layer.setup.getRootFolder()
        with zeit.cms.testing.site(root):
            root['repository']['infobox'] = ib
        self.content_id = 'http://xml.zeit.de/infobox'
        transaction.commit()

class PortraitboxTest(GalleryTest):

    expected_type = 'portraitbox'

    def setup_content(self):
        from zeit.content.portraitbox.portraitbox import Portraitbox
        import transaction
        import zeit.cms.testing
        pb = Portraitbox()
        root = self.layer.setup.getRootFolder()
        with zeit.cms.testing.site(root):
            with zeit.cms.testing.interaction():
                root['repository']['pb'] = pb
        self.content_id = 'http://xml.zeit.de/pb'
        transaction.commit()

    def test_layout_should_be_editable(self):
        self.setup_content()
        article = self.get_article(with_empty_block=True)
        article.xml.body.division[self.expected_type].set(
            'href', self.content_id)
        article._p_changed = True
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'short', 'wide'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['wide']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="wide ...""")

class ImageTest(GalleryTest):

    expected_type = 'image'
    attribute = 'src'

    def setup_content(self):
        self.content_id = 'http://xml.zeit.de/2006/DSC00109_2.JPG'

    def test_only_specific_type_should_be_droppable(self):
        article = self.get_article(with_empty_block=True)
        self.setup_content()
        with self.assertRaises(Exception):
            self.browser.open(
                'editable-body/blockname/@@set_reference?uniqueId='
                'http://xml.zeit.de/testcontent')
        # NOTE: the error message should be improved
        self.assert_ellipsis("ComponentLookupError...")

    def test_layout_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'small', 'large', 'Infobox', 'Hochkant'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['large']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="large ...""")

    def test_empty_block_should_not_provide_drop_in_readonly_mode(self):
        # Disable inherited test because empty images are removed during
        # checkin. This will change hopefully once #8194 is implemented.
        pass

class VideoTest(GalleryTest):

    layer = zeit.content.article.testing.ArticleBrightcoveLayer
    expected_type = 'video'

    def setup_content(self):
        self.content_id = 'http://video.zeit.de/video/1234'

    def test_layout_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.assertEqual(
            ['(no value)', 'small', 'with info', 'large', 'double'],
            self.browser.getControl('Layout').displayOptions)
        self.browser.getControl('Layout').displayValue = ['large']
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        # The layout is used as a css-class:
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...<div ...class="large ...""")

    def test_two_videos_should_be_editable(self):
        article = self.get_article(with_empty_block=True)
        self.browser.open(self.contents_url)
        self.browser.getLink('Edit').click()
        self.browser.getControl('Video', index=0).value = (
            'http://video.zeit.de/video/1234')
        self.browser.getControl('Video 2').value = (
            'http://video.zeit.de/video/6789')
        self.browser.getControl('Apply').click()
        self.assert_ellipsis("<...self.close()...")
        self.browser.open(self.contents_url)
        self.assert_ellipsis(
            """...
               ......""")
<|endoftext|>


<commit_before>from __future__ import with_statement # with statement in Python 2.5

# mex_builder.py:  Matlab extension builder
#
# by Marc Joliet, marcec@gmx.de, 2010.04.22
# (loosely based on mex.py by Joe VanAndel, vanandel@ucar.edu, 2010/1/15)
#
# This is a little SCons builder tool that finds Matlabs installation directory
# and sets various variables of the build environment. To do that, it starts a
# Matlab subprocess that issues a few commands that print to a file, and parses
# the file.  Furthermore, it adds a pseudo-builder method to the environment
# that wraps the SharedLibrary builder.

import os, tempfile, subprocess as subp, sys, pickle

vars_file = '.matlab_vars_cache'

def load_matlab_vars(env):
    """Load various Matlab specific variables from a cache file via the pickle
    module.
    """

    with open( vars_file, 'r' ) as f:
        p = pickle.Unpickler(f)
        env['MATLAB'] = p.load()

def cache_matlab_vars(matlab_vars):
    """Store various Matlab specific variables in a cache file via the pickle
    module.
    """

    # create the file if it doesn't exist
    if not os.path.isfile(vars_file):
        os.mknod(vars_file)

    # store the objects in a file
    with open(vars_file, 'w') as f:
        p = pickle.Pickler(f)
        p.dump(matlab_vars)

def gen_matlab_env(env, **kwargs):
    """Obtain various Matlab specific variables and put them in env['MATLAB'].
    """

    if not os.path.isfile(vars_file):
        tmp_file, tmp_file_name = tempfile.mkstemp()
        # As per Python tempfile documentation, Windows doesn't support multiple
        # processes accessing the same file, so close it immediately.
        os.close(tmp_file)

        # Invoke matlab, method of doing so taken from the mlabwrap setup.py.  The
        # usage of '10' as a newline char is needed because... maybe Python
        # universal newlines translate to newlines Matlab doesn't like? I dunno, but
        # in the Matlab command line '\n' works, but not in this script, even with
        # escapes or as a raw string.
        matlab_cmd = "fid = fopen('%s', 'wt');" % tmp_file_name + \
                r"fprintf(fid, '%s%c%s%c%s%c', mexext, 10, matlabroot, 10, computer, 10, version, 10);" + \
                "fclose(fid);quit;"
        cmd_line = ['matlab', '-nodesktop', '-nosplash', '-r', matlab_cmd]
        if os.name == "nt":
            cmd_line[-1] = '"' + cmd_line[-1] + '"'
            cmd_line += ['-wait'] # stop Matlab from forking

        try:
            # output to pipe to suppress output on Unix
            subp.check_call(cmd_line, stdout=subp.PIPE)
        except BaseException, e:
            # PEP 352 can't go ahead quickly enough, stupid args tuple. I want the
            # message attribute back!
            print >> sys.stderr, "Error:", ', '.join([repr(i) for i in e.args])
            os.remove(tmp_file_name)
            exit("Error calling Matlab, exiting.")

        # read lines from file and remove newline chars
        with open(tmp_file_name) as tmp_file:
            lines = [l.strip('\n') for l in tmp_file.readlines()]
            os.remove(tmp_file_name)

        matlab_root = lines[1]
        matlab_arch = lines[2].lower()
        matlab_ver, matlab_release  = lines[3].split()
        if matlab_arch == 'pcwin':
            matlab_arch = 'win32'

        env['MATLAB'] = {
            "MEX_EXT":  "." + lines[0],
            "ROOT":     matlab_root,
            "ARCH":     matlab_arch,
            "VERSION":  matlab_ver,
            "RELEASE":  matlab_release.strip('()'),
            "SRC":      os.sep.join([matlab_root, 'extern', 'src']),
            "INCLUDE":  os.sep.join([matlab_root, 'extern', 'include']),
            "LIB_DIR":  [os.sep.join([matlab_root, 'bin', matlab_arch])]
        }

        if matlab_arch == 'win32':
            env['MATLAB']['LIB_DIR'] += \
                    [os.sep.join([matlab_root, 'extern', 'lib', 'win32', 'microsoft'])]
            # TODO: test WINDOWS_INSERT_DEF option
            # env.Replace(WINDOWS_INSERT_DEF=True)

        print "Caching Matlab vars..."
        cache_matlab_vars(env['MATLAB'])
    else:
        print "Loading Matlab vars from cache..."
        load_matlab_vars(env)

def mex_builder(env, target, source, only_deps=False, make_def=True):
    """A Mex pseudo-builder for SCons that wraps the SharedLibrary builder.

       This pseudo-builder merely inserts some library dependencies, source file
       dependencies and the compiler expected by Matlab.  We don't return the
       targets and sources, since they only change on Unix, where we don't do
       anything but build anyway.

       The only_deps option is for Windows, where a MSVS Solution file might be
       wanted, in which case only the dependencies are added to the environment.
       """

    # don't touch the original environment unless the dependencies are wanted;
    # this is supposed to prevent unwanted side effects (compiler options) when
    # adding Mex programs to existing environments
    if not only_deps:
        env = env.Clone()

    source   = list(source)
    platform = env['PLATFORM']

    # define operating system independent options and dependencies
    if platform == "win32":
        # Matlab doesn't follow the Windows standard and adds a 'lib' prefix anyway
        env.Append(LIBS = ["libmex", "libmx"])
    else:
        env.Append(LIBS = ["mex", "mx"])

    # this tells SCons where to find mexversion.c
    env.Repository(env["MATLAB"]["SRC"])

    # OS dependent stuff, we assume GCC on Unix like platforms
    if platform == "posix":
        # add "exceptions" option, without which any mex function that raises an
        # exception (e.g., mexErrMsgTxt()) causes Matlab to crash
        env.Append(CCFLAGS=["-fexceptions", "-pthread"])

        # the need for mexversion.c was removed in Matlab version 7.9
        if env['MATLAB']['RELEASE'] < "R2009a":
            mexversion = env.Clone()
            # give each Mex file its own mexversion object (prevents warnings
            # from SCons and makes sure the same compiler options are used)
            mexversion_obj = mexversion.SharedObject("mexversion_"+target,
                                                     "mexversion.c")
            source.append(mexversion_obj)
    elif platform == "win32":
        env.Append(WINDOWS_INSERT_MANIFEST = True)
        # TODO: test WINDOWS_INSERT_DEF option
        # def_file = env.Textfile(target+".def", \
        #     source=["LIBRARY " + [s for s in source if target in s],
        #             "EXPORTS mexFunction"])

        # source += [def_file]
    elif platform == "darwin":
        env.Append(CCFLAGS="-fexceptions -pthread")
    else:
        exit("Oops, not a supported platform.")

    env.Append(CPPDEFINES = ["MATLAB_MEX_FILE"],
               CPPPATH    = [env['MATLAB']['INCLUDE']],
               LIBPATH    = [env['MATLAB']['LIB_DIR']])

    # add compile target: return the node object (or None, if only the deps are
    # requested)
    if not only_deps:
        return env.SharedLibrary(target, source,
                                 SHLIBPREFIX="",
                                 SHLIBSUFFIX=env['MATLAB']['MEX_EXT'])
    else:
        return None

def generate(env, **kwargs):
    gen_matlab_env(env, mex=kwargs.get('mex', False))

    # defines a pseudo-builder that internally calls the SharedLibrary builder
    env.AddMethod(mex_builder, "Mex")

def exists(env):
    # FIXME: Why is this function not called? Calling exit() here does nothing!
    if not env.WhereIs("matlab"):
        return False
    return True
<commit_msg>mex_builder.py: refactor if block.<commit_after>from __future__ import with_statement # with statement in Python 2.5

# mex_builder.py:  Matlab extension builder
#
# by Marc Joliet, marcec@gmx.de, 2010.04.22
# (loosely based on mex.py by Joe VanAndel, vanandel@ucar.edu, 2010/1/15)
#
# This is a little SCons builder tool that finds Matlabs installation directory
# and sets various variables of the build environment. To do that, it starts a
# Matlab subprocess that issues a few commands that print to a file, and parses
# the file.  Furthermore, it adds a pseudo-builder method to the environment
# that wraps the SharedLibrary builder.

import os, tempfile, subprocess as subp, sys, pickle

vars_file = '.matlab_vars_cache'

def load_matlab_vars(env):
    """Load various Matlab specific variables from a cache file via the pickle
    module.
    """

    with open( vars_file, 'r' ) as f:
        p = pickle.Unpickler(f)
        env['MATLAB'] = p.load()

def cache_matlab_vars(matlab_vars):
    """Store various Matlab specific variables in a cache file via the pickle
    module.
    """

    # create the file if it doesn't exist
    if not os.path.isfile(vars_file):
        os.mknod(vars_file)

    # store the objects in a file
    with open(vars_file, 'w') as f:
        p = pickle.Pickler(f)
        p.dump(matlab_vars)

def gen_matlab_env(env, **kwargs):
    """Obtain various Matlab specific variables and put them in env['MATLAB'].
    """

    if os.path.isfile(vars_file):
        print "Loading Matlab vars from cache..."
        load_matlab_vars(env)
        return

    tmp_file, tmp_file_name = tempfile.mkstemp()
    # As per Python tempfile documentation, Windows doesn't support multiple
    # processes accessing the same file, so close it immediately.
    os.close(tmp_file)

    # Invoke matlab, method of doing so taken from the mlabwrap setup.py.  The
    # usage of '10' as a newline char is needed because... maybe Python
    # universal newlines translate to newlines Matlab doesn't like? I dunno, but
    # in the Matlab command line '\n' works, but not in this script, even with
    # escapes or as a raw string.
    matlab_cmd = "fid = fopen('%s', 'wt');" % tmp_file_name + \
            r"fprintf(fid, '%s%c%s%c%s%c', mexext, 10, matlabroot, 10, computer, 10, version, 10);" + \
            "fclose(fid);quit;"
    cmd_line = ['matlab', '-nodesktop', '-nosplash', '-r', matlab_cmd]
    if os.name == "nt":
        cmd_line[-1] = '"' + cmd_line[-1] + '"'
        cmd_line += ['-wait'] # stop Matlab from forking

    try:
        # output to pipe to suppress output on Unix
        subp.check_call(cmd_line, stdout=subp.PIPE)
    except BaseException, e:
        # PEP 352 can't go ahead quickly enough, stupid args tuple. I want the
        # message attribute back!
        print >> sys.stderr, "Error:", ', '.join([repr(i) for i in e.args])
        os.remove(tmp_file_name)
        exit("Error calling Matlab, exiting.")

    # read lines from file and remove newline chars
    with open(tmp_file_name) as tmp_file:
        lines = [l.strip('\n') for l in tmp_file.readlines()]
        os.remove(tmp_file_name)

    matlab_root = lines[1]
    matlab_arch = lines[2].lower()
    matlab_ver, matlab_release  = lines[3].split()
    if matlab_arch == 'pcwin':
        matlab_arch = 'win32'

    env['MATLAB'] = {
        "MEX_EXT":  "." + lines[0],
        "ROOT":     matlab_root,
        "ARCH":     matlab_arch,
        "VERSION":  matlab_ver,
        "RELEASE":  matlab_release.strip('()'),
        "SRC":      os.sep.join([matlab_root, 'extern', 'src']),
        "INCLUDE":  os.sep.join([matlab_root, 'extern', 'include']),
        "LIB_DIR":  [os.sep.join([matlab_root, 'bin', matlab_arch])]
    }

    if matlab_arch == 'win32':
        env['MATLAB']['LIB_DIR'] += \
                [os.sep.join([matlab_root, 'extern', 'lib', 'win32', 'microsoft'])]
        # TODO: test WINDOWS_INSERT_DEF option
        # env.Replace(WINDOWS_INSERT_DEF=True)

    print "Caching Matlab vars..."
    cache_matlab_vars(env['MATLAB'])

def mex_builder(env, target, source, only_deps=False, make_def=True):
    """A Mex pseudo-builder for SCons that wraps the SharedLibrary builder.

       This pseudo-builder merely inserts some library dependencies, source file
       dependencies and the compiler expected by Matlab.  We don't return the
       targets and sources, since they only change on Unix, where we don't do
       anything but build anyway.

       The only_deps option is for Windows, where a MSVS Solution file might be
       wanted, in which case only the dependencies are added to the environment.
       """

    # don't touch the original environment unless the dependencies are wanted;
    # this is supposed to prevent unwanted side effects (compiler options) when
    # adding Mex programs to existing environments
    if not only_deps:
        env = env.Clone()

    source   = list(source)
    platform = env['PLATFORM']

    # define operating system independent options and dependencies
    if platform == "win32":
        # Matlab doesn't follow the Windows standard and adds a 'lib' prefix anyway
        env.Append(LIBS = ["libmex", "libmx"])
    else:
        env.Append(LIBS = ["mex", "mx"])

    # this tells SCons where to find mexversion.c
    env.Repository(env["MATLAB"]["SRC"])

    # OS dependent stuff, we assume GCC on Unix like platforms
    if platform == "posix":
        # add "exceptions" option, without which any mex function that raises an
        # exception (e.g., mexErrMsgTxt()) causes Matlab to crash
        env.Append(CCFLAGS=["-fexceptions", "-pthread"])

        # the need for mexversion.c was removed in Matlab version 7.9
        if env['MATLAB']['RELEASE'] < "R2009a":
            mexversion = env.Clone()
            # give each Mex file its own mexversion object (prevents warnings
            # from SCons and makes sure the same compiler options are used)
            mexversion_obj = mexversion.SharedObject("mexversion_"+target,
                                                     "mexversion.c")
            source.append(mexversion_obj)
    elif platform == "win32":
        env.Append(WINDOWS_INSERT_MANIFEST = True)
        # TODO: test WINDOWS_INSERT_DEF option
        # def_file = env.Textfile(target+".def", \
        #     source=["LIBRARY " + [s for s in source if target in s],
        #             "EXPORTS mexFunction"])

        # source += [def_file]
    elif platform == "darwin":
        env.Append(CCFLAGS="-fexceptions -pthread")
    else:
        exit("Oops, not a supported platform.")

    env.Append(CPPDEFINES = ["MATLAB_MEX_FILE"],
               CPPPATH    = [env['MATLAB']['INCLUDE']],
               LIBPATH    = [env['MATLAB']['LIB_DIR']])

    # add compile target: return the node object (or None, if only the deps are
    # requested)
    if not only_deps:
        return env.SharedLibrary(target, source,
                                 SHLIBPREFIX="",
                                 SHLIBSUFFIX=env['MATLAB']['MEX_EXT'])
    else:
        return None

def generate(env, **kwargs):
    gen_matlab_env(env, mex=kwargs.get('mex', False))

    # defines a pseudo-builder that internally calls the SharedLibrary builder
    env.AddMethod(mex_builder, "Mex")

def exists(env):
    # FIXME: Why is this function not called? Calling exit() here does nothing!
    if not env.WhereIs("matlab"):
        return False
    return True
<|endoftext|>