<commit_before>import json
from awsclpy import AWSCLPy

class AmiEc2(object):
    def __init__(self, **kwrags):
        self.__quiet = kwrags.get('quiet', False)
        self.__recipe = kwrags['recipe']
        self.__awscli = AWSCLPy(quiet=self.__quiet,
                                **self.__recipe.awscli_args.__dict__)

    def instantiate(self):
        security_group = self.__recipe.security_groups
        if not security_group:
            self.__create_security_group()
            security_group = self.security_group

        key_name = self.__recipe.key_name
        if not key_name:
            self.__generate_key_pair()
            key_name = self.key_name

        instance_profile = self.__recipe.iam_instance_profile
        iam_roles = self.__recipe.iam_roles

        if instance_profile:
            instance_profile_arn = instance_profile.arn
            instance_profile_name = instance_profile.name
        elif isinstance(iam_roles, list):
            instance_profile_arn, instance_profile_name = \
                self.__create_iam_instance_profile(iam_roles)
        else:
            instance_profile_arn = instance_profile_name = None

        iam_instance_profile = []

        if instance_profile_arn:
            iam_instance_profile.append(
                '='.join(['Arn', instance_profile_arn])
            )

        if instance_profile_name:
            iam_instance_profile.append(
                '='.join(['Name', instance_profile_name])
            )

        if iam_instance_profile:
            iam_instance_profile = [
                '--iam-instance-profile',
                ','.join(iam_instance_profile)
            ]

        associate_public_ip_address = \
            '--associate-public-ip-address' \
            if self.__recipe.associate_public_ip \
            else '--no-associate-public-ip-address'

        instance = self.__awscli.ec2(
            'run-instances',
            '--image-id', self.__recipe.base_ami,
            '--key-name', key_name,
            '--security-group-ids', security_group,
            '--instance-type', self.__recipe.instance_type,
            '--subnet-id', self.__recipe.subnet_id,
            associate_public_ip_address,
            iam_instance_profile
            )

        self.__instance = instance['Instances'][0]

        self.tag(self.__instance['InstanceId'], self.__recipe.ec2_tags)

        self.__describe_instance()

    def grab_existing_instance(self, ec2_id):
        self.__describe_instance(ec2_id)

    def terminate(self):
        self.__awscli.ec2('terminate-instances',
                          '--instance-ids', self.__instance['InstanceId'])

        if hasattr(self, 'security_group'):
            self.wait_until_terminated()
            self.__delete_security_group()

        if hasattr(self, 'key_name'):
            self.__delete_key_pair()

        if hasattr(self, 'iam_instance_profile'):
            self.__delete_iam_instance_profile()

    def wait_until_running(self):
        self.__awscli.ec2('wait', 'instance-running',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_healthy(self):
        self.__awscli.ec2('wait', 'instance-status-ok',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_terminated(self):
        self.__awscli.ec2('wait', 'instance-terminated',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_image_available(self):
        self.__awscli.ec2('wait', 'image-available',
                          '--image-ids', self.__image['ImageId'])

    def get_hostname(self):
        if self.__instance.get('PublicDnsName'):
            return self.__instance['PublicDnsName']
        else:
            if self.__instance.get('PublicIpAddress'):
                return self.__instance['PublicIpAddress']
            elif self.__instance.get('PrivateIpAddress'):
                return self.__instance['PrivateIpAddress']

    def get_username(self):
        return self.__recipe.ssh_username

    def tag(self, resource, tags):
        tags = ["Key=%s,Value=%s" % (key, value) for key, value in
                tags.iteritems()]

        self.__awscli.ec2('create-tags',
                          '--resources', resource,
                          '--tags', tags)

    def create_image(self):
        self.__image = self.__awscli.ec2(
            'create-image',
            '--instance-id', self.__instance['InstanceId'],
            '--name', self.__recipe.ami_tags.Name,
            '--reboot')

        ami_permissions = self.__recipe.ami_permissions

        if ami_permissions:
            self.__share_image(ami_permissions)

        if not self.__image:
            raise Exception('Image creation for instance %s failed.' %
                            self.__instance['InstanceId'])

        self.tag(self.__image['ImageId'], self.__recipe.ami_tags)

        return self.__image['ImageId']

    def __share_image(self, account_ids):
        permissions = {'Add': []}

        for account_id in account_ids:
            permissions['Add'].append({'UserId': str(account_id)})

        self.wait_until_image_available()
        self.__awscli.ec2('modify-image-attribute',
                          '--image-id', self.__image['ImageId'],
                          '--launch-permission', json.dumps(permissions))

    def __describe_instance(self, specific_id):
        if specific_id:
            instance = self.__awscli.ec2('describe-instances',
                                         '--instance-ids',
                                         specific_id)
        else:
            self.wait_until_running()
            instance = self.__awscli.ec2('describe-instances',
                                         '--instance-ids',
                                         self.__instance['InstanceId'])

        self.__instance = instance['Reservations'][0]['Instances'][0]

    def __get_vpc_id(self):
        subnet = self.__awscli.ec2('describe-subnets',
                                   '--subnet-ids', self.__recipe.subnet_id)

        return subnet['Subnets'][0]['VpcId']

    def __create_security_group(self):
        vpc_id = self.__get_vpc_id()

        security_group = self.__awscli.ec2(
            'create-security-group',
            '--group-name', self.__recipe.ec2_tags.Name,
            '--description', 'Allows temporary SSH access to the box.',
            '--vpc-id', vpc_id)

        self.__awscli.ec2('authorize-security-group-ingress',
                          '--group-id', security_group['GroupId'],
                          '--protocol', 'tcp',
                          '--port', 22,
                          '--cidr', '0.0.0.0/0')

        self.__awscli.ec2('authorize-security-group-egress',
                          '--group-id', security_group['GroupId'],
                          '--protocol', 'tcp',
                          '--port', '0-65535',
                          '--cidr', '0.0.0.0/0')

        self.security_group = security_group['GroupId']

    def __delete_security_group(self):
        self.__awscli.ec2('delete-security-group',
                          '--group-id', self.security_group)

    def __generate_key_pair(self):
        # TODO: generate keypair if not provided
        self.key_name = None

    def __delete_key_pair(self):
        # TODO: delete key pair if autogenerated
        pass

    def __create_iam_instance_profile(self, iam_roles):
        iam_instance_profile = self.__awscli.iam(
            'create-instance-profile',
            '--instance-profile-name', 'AmiBaker')

        self.iam_instance_profile = iam_instance_profile['InstanceProfile']

        for role in iam_roles:
            self.__awscli.iam('add-role-to-instance-profile',
                              '--instance-profile-name', 'AmiBaker',
                              '--role-name', role)

        return (self.iam_instance_profile['InstanceProfileName'],
                self.iam_instance_profile['Arn'])

    def __delete_iam_instance_profile(self):
        self.__awscli.iam('delete-instance-profile',
                          '--instance-profile-name', 'AmiBaker')

        self.iam_instance_profile = None
<commit_msg>make that optional<commit_after>import json
from awsclpy import AWSCLPy

class AmiEc2(object):
    def __init__(self, **kwrags):
        self.__quiet = kwrags.get('quiet', False)
        self.__recipe = kwrags['recipe']
        self.__awscli = AWSCLPy(quiet=self.__quiet,
                                **self.__recipe.awscli_args.__dict__)

    def instantiate(self):
        security_group = self.__recipe.security_groups
        if not security_group:
            self.__create_security_group()
            security_group = self.security_group

        key_name = self.__recipe.key_name
        if not key_name:
            self.__generate_key_pair()
            key_name = self.key_name

        instance_profile = self.__recipe.iam_instance_profile
        iam_roles = self.__recipe.iam_roles

        if instance_profile:
            instance_profile_arn = instance_profile.arn
            instance_profile_name = instance_profile.name
        elif isinstance(iam_roles, list):
            instance_profile_arn, instance_profile_name = \
                self.__create_iam_instance_profile(iam_roles)
        else:
            instance_profile_arn = instance_profile_name = None

        iam_instance_profile = []

        if instance_profile_arn:
            iam_instance_profile.append(
                '='.join(['Arn', instance_profile_arn])
            )

        if instance_profile_name:
            iam_instance_profile.append(
                '='.join(['Name', instance_profile_name])
            )

        if iam_instance_profile:
            iam_instance_profile = [
                '--iam-instance-profile',
                ','.join(iam_instance_profile)
            ]

        associate_public_ip_address = \
            '--associate-public-ip-address' \
            if self.__recipe.associate_public_ip \
            else '--no-associate-public-ip-address'

        instance = self.__awscli.ec2(
            'run-instances',
            '--image-id', self.__recipe.base_ami,
            '--key-name', key_name,
            '--security-group-ids', security_group,
            '--instance-type', self.__recipe.instance_type,
            '--subnet-id', self.__recipe.subnet_id,
            associate_public_ip_address,
            iam_instance_profile
            )

        self.__instance = instance['Instances'][0]

        self.tag(self.__instance['InstanceId'], self.__recipe.ec2_tags)

        self.__describe_instance()

    def grab_existing_instance(self, ec2_id):
        self.__describe_instance(ec2_id)

    def terminate(self):
        self.__awscli.ec2('terminate-instances',
                          '--instance-ids', self.__instance['InstanceId'])

        if hasattr(self, 'security_group'):
            self.wait_until_terminated()
            self.__delete_security_group()

        if hasattr(self, 'key_name'):
            self.__delete_key_pair()

        if hasattr(self, 'iam_instance_profile'):
            self.__delete_iam_instance_profile()

    def wait_until_running(self):
        self.__awscli.ec2('wait', 'instance-running',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_healthy(self):
        self.__awscli.ec2('wait', 'instance-status-ok',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_terminated(self):
        self.__awscli.ec2('wait', 'instance-terminated',
                          '--instance-ids', self.__instance['InstanceId'])

    def wait_until_image_available(self):
        self.__awscli.ec2('wait', 'image-available',
                          '--image-ids', self.__image['ImageId'])

    def get_hostname(self):
        if self.__instance.get('PublicDnsName'):
            return self.__instance['PublicDnsName']
        else:
            if self.__instance.get('PublicIpAddress'):
                return self.__instance['PublicIpAddress']
            elif self.__instance.get('PrivateIpAddress'):
                return self.__instance['PrivateIpAddress']

    def get_username(self):
        return self.__recipe.ssh_username

    def tag(self, resource, tags):
        tags = ["Key=%s,Value=%s" % (key, value) for key, value in
                tags.iteritems()]

        self.__awscli.ec2('create-tags',
                          '--resources', resource,
                          '--tags', tags)

    def create_image(self):
        self.__image = self.__awscli.ec2(
            'create-image',
            '--instance-id', self.__instance['InstanceId'],
            '--name', self.__recipe.ami_tags.Name,
            '--reboot')

        ami_permissions = self.__recipe.ami_permissions

        if ami_permissions:
            self.__share_image(ami_permissions)

        if not self.__image:
            raise Exception('Image creation for instance %s failed.' %
                            self.__instance['InstanceId'])

        self.tag(self.__image['ImageId'], self.__recipe.ami_tags)

        return self.__image['ImageId']

    def __share_image(self, account_ids):
        permissions = {'Add': []}

        for account_id in account_ids:
            permissions['Add'].append({'UserId': str(account_id)})

        self.wait_until_image_available()
        self.__awscli.ec2('modify-image-attribute',
                          '--image-id', self.__image['ImageId'],
                          '--launch-permission', json.dumps(permissions))

    def __describe_instance(self, specific_id=None):
        if specific_id:
            instance = self.__awscli.ec2('describe-instances',
                                         '--instance-ids',
                                         specific_id)
        else:
            self.wait_until_running()
            instance = self.__awscli.ec2('describe-instances',
                                         '--instance-ids',
                                         self.__instance['InstanceId'])

        self.__instance = instance['Reservations'][0]['Instances'][0]

    def __get_vpc_id(self):
        subnet = self.__awscli.ec2('describe-subnets',
                                   '--subnet-ids', self.__recipe.subnet_id)

        return subnet['Subnets'][0]['VpcId']

    def __create_security_group(self):
        vpc_id = self.__get_vpc_id()

        security_group = self.__awscli.ec2(
            'create-security-group',
            '--group-name', self.__recipe.ec2_tags.Name,
            '--description', 'Allows temporary SSH access to the box.',
            '--vpc-id', vpc_id)

        self.__awscli.ec2('authorize-security-group-ingress',
                          '--group-id', security_group['GroupId'],
                          '--protocol', 'tcp',
                          '--port', 22,
                          '--cidr', '0.0.0.0/0')

        self.__awscli.ec2('authorize-security-group-egress',
                          '--group-id', security_group['GroupId'],
                          '--protocol', 'tcp',
                          '--port', '0-65535',
                          '--cidr', '0.0.0.0/0')

        self.security_group = security_group['GroupId']

    def __delete_security_group(self):
        self.__awscli.ec2('delete-security-group',
                          '--group-id', self.security_group)

    def __generate_key_pair(self):
        # TODO: generate keypair if not provided
        self.key_name = None

    def __delete_key_pair(self):
        # TODO: delete key pair if autogenerated
        pass

    def __create_iam_instance_profile(self, iam_roles):
        iam_instance_profile = self.__awscli.iam(
            'create-instance-profile',
            '--instance-profile-name', 'AmiBaker')

        self.iam_instance_profile = iam_instance_profile['InstanceProfile']

        for role in iam_roles:
            self.__awscli.iam('add-role-to-instance-profile',
                              '--instance-profile-name', 'AmiBaker',
                              '--role-name', role)

        return (self.iam_instance_profile['InstanceProfileName'],
                self.iam_instance_profile['Arn'])

    def __delete_iam_instance_profile(self):
        self.__awscli.iam('delete-instance-profile',
                          '--instance-profile-name', 'AmiBaker')

        self.iam_instance_profile = None
<|endoftext|>


<commit_before>#!/usr/bin/python
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# All Rights Reserved
#

import errno
import logging
import os, sys
import math
import md5
import re
import urllib
import signal
import shutil
import tempfile
import time

from conary import callbacks
from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from conary.lib import util

from jobmaster.util import logCall, getIP

SWAP_SIZE = 268435456 # 256 MB in bytes
TAGSCRIPT_GROWTH = 20971520 # 20MB in bytes
CYLINDERSIZE = 516096
SECTORS         = 63
HEADS           = 16

###########
# all image building related functions are not class members to
# enforce prevention of side effects
###########

def md5sum(s):
    m = md5.new()
    m.update(s)
    return m.hexdigest()

def roundUpSize(size):
    # 13% accounts for reserved block and inode consumption
    size = int(math.ceil((size + TAGSCRIPT_GROWTH + SWAP_SIZE) / 0.87))
    # now round up to next cylinder size
    return size + ((CYLINDERSIZE - (size % CYLINDERSIZE)) % CYLINDERSIZE)

def createDir(d):
    if not os.path.exists(d):
        createDir(os.path.split(d)[0])
        os.mkdir(d)

def mkBlankFile(fn, size, sparse = True):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    if sparse:
        f.seek(size - 1)
        f.write(chr(0))
    else:
        for i in range(size / 512):
            f.write(512 * chr(0))
        f.write((size % 512) * chr(0))
    f.close()

def createFile(fn, contents):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    f.write(contents)
    f.close()

def writeConaryRc(d, mirrorUrl = ''):
    # write the conaryrc file
    conaryrcFile = open(os.path.join(d, 'etc', 'conaryrc'), "w")
    if mirrorUrl:
        type, url = urllib.splittype(mirrorUrl)
        relativeLink = ''
        if not type:
            type = 'http'
        if not url.startswith('//'):
            url = '//' + url
        if not urllib.splithost(url)[1]:
            relativeLink = '/conaryrc'
        mirrorUrl = type + ':' + url + relativeLink
        print >> conaryrcFile, 'includeConfigFile ' + mirrorUrl
    print >> conaryrcFile, "pinTroves kernel.*"
    print >> conaryrcFile, "includeConfigFile /etc/conary/config.d/*"
    conaryrcFile.close()

def createTemporaryRoot(fakeRoot):
    for d in ('etc', 'etc/sysconfig', 'etc/sysconfig/network-scripts',
              'boot/grub', 'tmp', 'proc', 'sys', 'root', 'var'):
        util.mkdirChain(os.path.join(fakeRoot, d))

def fsOddsNEnds(d):
    createFile(os.path.join(d, 'etc', 'fstab'),
               '\n'.join(('LABEL=/ / ext3 defaults 1 1',
                          'none /dev/pts devpts gid=5,mode=620 0 0',
                          'none /dev/shm tmpfs defaults 0 0',
                          'none /proc proc defaults 0 0',
                          'none /sys sysfs defaults 0 0',
                          '/var/swap swap swap defaults 0 0\n')))
    #create a swap file
    mkBlankFile(os.path.join(d, 'var', 'swap'), SWAP_SIZE, sparse = False)
    logCall('/sbin/mkswap %s >/dev/null 2>&1' % \
                  os.path.join(d, 'var', 'swap'))

    util.copytree(os.path.join(d, 'usr', 'share', 'grub', '*', '*'), \
                      os.path.join(d, 'boot', 'grub'))

    #copy the files needed by grub and set up the links
    grubContents = \
        '\n'.join(('default=0',
                   'timeout=0',
                   'title rBuilder Job Slave (template)',
                   '    root (hd0,0)',
                   '    kernel /boot/vmlinuz-template ro root=LABEL=/ quiet',
                   '    initrd /boot/initrd-template.img\n'))
    createFile(os.path.join(d, 'boot', 'grub', 'grub.conf'), grubContents)
    os.symlink('grub.conf', os.path.join(d, 'boot', 'grub', 'menu.1st'))

    #Add the other miscellaneous files needed
    createFile(os.path.join(d, 'etc', 'hosts'),
               '127.0.0.1       localhost.localdomain   localhost\n')
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network'),
               '\n'.join(('NETWORKING=yes',
                          'HOSTNAME=localhost.localdomain\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network-scripts',
                            'ifcfg-eth0.template'),
               '\n'.join(('DEVICE=eth0',
                          'BOOTPROTO=static',
                          'IPADDR=%(ipaddr)s',
                          'GATEWAY=%(masterip)s',
                          'ONBOOT=yes',
                          'TYPE=Ethernet\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'keyboard'),
               '\n'.join(('KEYBOARDTYPE="pc"',
                          'KEYTABLE="us"\n')))
    writeConaryRc(d)

def getRunningKernel():
    p = os.popen('uname -r')
    data = p.read()
    p.close()

    m = re.match('[\d.-]*', data)
    ver = m.group()[:-1]
    p = os.popen('conary q kernel:runtime --full-versions --flavors | grep %s' % ver)
    return p.read().strip()

def signalHandler(*args, **kwargs):
    # change signals into exceptions
    raise RuntimeError('process killed')

class ImageCache(object):
    def __init__(self, cachePath, masterCfg):
        self.cachePath = cachePath
        self.masterCfg = masterCfg
        util.mkdirChain(self.cachePath)

        self.conarycfg = conarycfg.ConaryConfiguration(True)

        self.cc = conaryclient.ConaryClient(self.conarycfg)
        self.nc = self.cc.getRepos()

        self.tmpPath = os.path.join(os.path.split(cachePath)[0], 'tmp')

    def startBuildingImage(self, hash, output = True):
        # this function is designed to block if an image is being built already
        lockPath = os.path.join(self.cachePath, hash + '.lock')
        done = False
        while not done:
            while os.path.exists(lockPath):
                if output:
                    logging.info('Waiting for building lock: %s' % lockPath)
                    output = False
                time.sleep(1)
            try:
                os.mkdir(lockPath)
                logging.info('Acquired slave building lock: %s' % lockPath)
                done = True
            except OSError, e:
                if e.errno != errno.EEXIST:
                    raise

    def stopBuildingImage(self, hash):
        lockPath = os.path.join(self.cachePath, hash + '.lock')
        logging.info('Releasing slave building lock: %s' % lockPath)
        os.rmdir(lockPath)

    def deleteAllImages(self):
        # this is for clearing the cache, eg. needed if entitlements changed
        for image in os.listdir(self.cachePath):
            os.unlink(os.path.join(self.cachePath, image))

    def haveImage(self, troveSpec):
        return os.path.exists(self.imagePath(troveSpec))

    def imagePath(self, troveSpec):
        hash = md5sum(troveSpec)
        return os.path.join(self.cachePath, hash)

    def getImage(self, troveSpec, debugMode=False):
        hash = md5sum(troveSpec)
        imageFile = os.path.join(self.cachePath, hash)
        if hash in os.listdir(self.cachePath) and not debugMode:
            logging.info("Found image in cache for %s" % troveSpec)
            return imageFile
        else:
            logging.info("Image not cached, creating image for %s" % \
                    troveSpec)
            signal.signal(signal.SIGTERM, signalHandler)
            signal.signal(signal.SIGINT, signalHandler)
            self.startBuildingImage(hash)
            try:
                if os.path.exists(imageFile):
                    return imageFile
                return self.makeImage(troveSpec, hash)
            finally:
                self.stopBuildingImage(hash)

    def makeImage(self, troveSpec, hash):
        spec_n, spec_v, spec_f = cmdline.parseTroveSpec(troveSpec)
        n, v, f = self.nc.findTrove(None, (spec_n, spec_v, spec_f), self.conarycfg.flavor)[0]

        trv = self.nc.getTrove(n, v, f, withFiles = False)

        size = trv.getSize()
        size = roundUpSize(size)

        fd, fn = tempfile.mkstemp(dir = self.tmpPath)
        os.close(fd)

        fd, tagScript = tempfile.mkstemp(prefix = "tagscript",
                                         dir = self.tmpPath)
        os.close(fd)

        mntDir = tempfile.mkdtemp(dir = self.tmpPath)
        client = None
        try:
            mkBlankFile(fn, size)

            # run mke2fs on blank image
            logCall('mkfs -t ext2 -F -L / %s %d' % \
                          (fn, size / 1024))
            logCall('tune2fs -m 0 -i 0 -c 0 %s' % fn)

            logCall('mount -o loop %s %s' % (fn, mntDir))

            createTemporaryRoot(mntDir)
            logCall('mount -t proc none %s' % os.path.join(mntDir, 'proc'))
            logCall('mount -t sysfs none %s' % os.path.join(mntDir, 'sys'))

            # Prepare conary client
            cfg = conarycfg.ConaryConfiguration(True)
            if self.masterCfg.conaryProxy:
                proxy = self.masterCfg.conaryProxy
                if proxy == 'self':
                    proxy = 'http://%s/' % getIP()
                cfg.conaryProxy['http']  = proxy
                cfg.conaryProxy['https'] = proxy
            cfg.root = mntDir
            client = conaryclient.ConaryClient(cfg)
            client.setUpdateCallback(UpdateCallback())

            kernelSpec = getRunningKernel()
            k_n, k_v, k_f = cmdline.parseTroveSpec(kernelSpec)

            # Install jobslave root and kernel
            job = client.newUpdateJob()
            logging.info('Preparing update job')
            client.prepareUpdateJob(job, (
                (n,   (None, None), (v,   f),    True), # root
                (k_n, (None, None), (k_v, k_f),  True), # kernel
                ))
            logging.info('Applying update job')
            client.applyUpdateJob(job, tagScript=tagScript)

            # Create various filesystem pieces
            fsOddsNEnds(mntDir)

            # Assemble tag script and run it
            outScript = os.path.join(mntDir, 'root', 'conary-tag-script')
            outScriptInRoot = os.path.join('', 'root', 'conary-tag-script')
            outScriptOutput = os.path.join('', 'root',
                'conary-tag-script.output')

            tagScriptFile = open(tagScript, 'r')
            outScriptFile = open(outScript, 'w')
            outScriptFile.write('/sbin/ldconfig\n')
            for line in tagScriptFile:
                if line.startswith('/sbin/ldconfig'):
                    continue
                outScriptFile.write(line)
            tagScriptFile.close()
            outScriptFile.close()

            os.unlink(tagScript)

            util.execute("chroot %s bash -c 'sh -x %s > %s 2>&1'" % (
                    mntDir, outScriptInRoot, outScriptOutput))

            # the preload wrapper isn't working yet, work around until we know why
            #safeEnv = {"LD_PRELOAD": "/usr/lib/jobmaster/chrootsafe_wrapper.so"}

            # authconfig can whack the domainname in certain circumstances
            oldDomainname = os.popen('domainname').read().strip() # save old domainname
            logCall("chroot %s /usr/sbin/authconfig --kickstart --enablemd5 --enableshadow --disablecache" % mntDir)
            # Only restore the domain if it was set in the first place.
            if oldDomainname:
                logCall("domainname %s" % oldDomainname)

            logCall("chroot %s /usr/sbin/usermod -p '' root" % mntDir)
            logCall('grubby --remove-kernel=/boot/vmlinuz-template --config-file=%s' % os.path.join(mntDir, 'boot', 'grub', 'grub.conf'))
        finally:
            if client:
                client.close()
                del job
            logCall('umount %s' % os.path.join(mntDir, 'proc'))
            logCall('umount %s' % os.path.join(mntDir, 'sys'))
            logCall('sync')
            logCall('umount %s' % mntDir)
            logCall('sync')
            util.rmtree(mntDir, ignore_errors = True)
        shutil.move(fn, os.path.join(self.cachePath, hash))
        return os.path.join(self.cachePath, hash)

class UpdateCallback(callbacks.UpdateCallback):
    def eatMe(*P, **K):
        pass

    tagHandlerOutput = troveScriptOutput = troveScriptFailure = eatMe

    # TODO: maybe log something useful occasionally when loglevel >= DEBUG
<commit_msg>Create a new conaryclient for each slave built so that the config may be changed without restarting the jobmaster.<commit_after>#!/usr/bin/python
#
# Copyright (c) 2004-2006 rPath, Inc.
#
# All Rights Reserved
#

import errno
import logging
import os, sys
import math
import md5
import re
import urllib
import signal
import shutil
import tempfile
import time

from conary import callbacks
from conary import conarycfg
from conary import conaryclient
from conary.conaryclient import cmdline
from conary.lib import util

from jobmaster.util import logCall, getIP

SWAP_SIZE = 268435456 # 256 MB in bytes
TAGSCRIPT_GROWTH = 20971520 # 20MB in bytes
CYLINDERSIZE = 516096
SECTORS         = 63
HEADS           = 16

###########
# all image building related functions are not class members to
# enforce prevention of side effects
###########

def md5sum(s):
    m = md5.new()
    m.update(s)
    return m.hexdigest()

def roundUpSize(size):
    # 13% accounts for reserved block and inode consumption
    size = int(math.ceil((size + TAGSCRIPT_GROWTH + SWAP_SIZE) / 0.87))
    # now round up to next cylinder size
    return size + ((CYLINDERSIZE - (size % CYLINDERSIZE)) % CYLINDERSIZE)

def createDir(d):
    if not os.path.exists(d):
        createDir(os.path.split(d)[0])
        os.mkdir(d)

def mkBlankFile(fn, size, sparse = True):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    if sparse:
        f.seek(size - 1)
        f.write(chr(0))
    else:
        for i in range(size / 512):
            f.write(512 * chr(0))
        f.write((size % 512) * chr(0))
    f.close()

def createFile(fn, contents):
    createDir(os.path.split(fn)[0])
    f = open(fn, 'w')
    f.write(contents)
    f.close()

def writeConaryRc(d, mirrorUrl = ''):
    # write the conaryrc file
    conaryrcFile = open(os.path.join(d, 'etc', 'conaryrc'), "w")
    if mirrorUrl:
        type, url = urllib.splittype(mirrorUrl)
        relativeLink = ''
        if not type:
            type = 'http'
        if not url.startswith('//'):
            url = '//' + url
        if not urllib.splithost(url)[1]:
            relativeLink = '/conaryrc'
        mirrorUrl = type + ':' + url + relativeLink
        print >> conaryrcFile, 'includeConfigFile ' + mirrorUrl
    print >> conaryrcFile, "pinTroves kernel.*"
    print >> conaryrcFile, "includeConfigFile /etc/conary/config.d/*"
    conaryrcFile.close()

def createTemporaryRoot(fakeRoot):
    for d in ('etc', 'etc/sysconfig', 'etc/sysconfig/network-scripts',
              'boot/grub', 'tmp', 'proc', 'sys', 'root', 'var'):
        util.mkdirChain(os.path.join(fakeRoot, d))

def fsOddsNEnds(d):
    createFile(os.path.join(d, 'etc', 'fstab'),
               '\n'.join(('LABEL=/ / ext3 defaults 1 1',
                          'none /dev/pts devpts gid=5,mode=620 0 0',
                          'none /dev/shm tmpfs defaults 0 0',
                          'none /proc proc defaults 0 0',
                          'none /sys sysfs defaults 0 0',
                          '/var/swap swap swap defaults 0 0\n')))
    #create a swap file
    mkBlankFile(os.path.join(d, 'var', 'swap'), SWAP_SIZE, sparse = False)
    logCall('/sbin/mkswap %s >/dev/null 2>&1' % \
                  os.path.join(d, 'var', 'swap'))

    util.copytree(os.path.join(d, 'usr', 'share', 'grub', '*', '*'), \
                      os.path.join(d, 'boot', 'grub'))

    #copy the files needed by grub and set up the links
    grubContents = \
        '\n'.join(('default=0',
                   'timeout=0',
                   'title rBuilder Job Slave (template)',
                   '    root (hd0,0)',
                   '    kernel /boot/vmlinuz-template ro root=LABEL=/ quiet',
                   '    initrd /boot/initrd-template.img\n'))
    createFile(os.path.join(d, 'boot', 'grub', 'grub.conf'), grubContents)
    os.symlink('grub.conf', os.path.join(d, 'boot', 'grub', 'menu.1st'))

    #Add the other miscellaneous files needed
    createFile(os.path.join(d, 'etc', 'hosts'),
               '127.0.0.1       localhost.localdomain   localhost\n')
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network'),
               '\n'.join(('NETWORKING=yes',
                          'HOSTNAME=localhost.localdomain\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'network-scripts',
                            'ifcfg-eth0.template'),
               '\n'.join(('DEVICE=eth0',
                          'BOOTPROTO=static',
                          'IPADDR=%(ipaddr)s',
                          'GATEWAY=%(masterip)s',
                          'ONBOOT=yes',
                          'TYPE=Ethernet\n')))
    createFile(os.path.join(d, 'etc', 'sysconfig', 'keyboard'),
               '\n'.join(('KEYBOARDTYPE="pc"',
                          'KEYTABLE="us"\n')))
    writeConaryRc(d)

def getRunningKernel():
    p = os.popen('uname -r')
    data = p.read()
    p.close()

    m = re.match('[\d.-]*', data)
    ver = m.group()[:-1]
    p = os.popen('conary q kernel:runtime --full-versions --flavors | grep %s' % ver)
    return p.read().strip()

def signalHandler(*args, **kwargs):
    # change signals into exceptions
    raise RuntimeError('process killed')

class ImageCache(object):
    def __init__(self, cachePath, masterCfg):
        self.cachePath = cachePath
        self.masterCfg = masterCfg
        util.mkdirChain(self.cachePath)

        self.tmpPath = os.path.join(os.path.split(cachePath)[0], 'tmp')

    def startBuildingImage(self, hash, output = True):
        # this function is designed to block if an image is being built already
        lockPath = os.path.join(self.cachePath, hash + '.lock')
        done = False
        while not done:
            while os.path.exists(lockPath):
                if output:
                    logging.info('Waiting for building lock: %s' % lockPath)
                    output = False
                time.sleep(1)
            try:
                os.mkdir(lockPath)
                logging.info('Acquired slave building lock: %s' % lockPath)
                done = True
            except OSError, e:
                if e.errno != errno.EEXIST:
                    raise

    def stopBuildingImage(self, hash):
        lockPath = os.path.join(self.cachePath, hash + '.lock')
        logging.info('Releasing slave building lock: %s' % lockPath)
        os.rmdir(lockPath)

    def deleteAllImages(self):
        # this is for clearing the cache, eg. needed if entitlements changed
        for image in os.listdir(self.cachePath):
            os.unlink(os.path.join(self.cachePath, image))

    def haveImage(self, troveSpec):
        return os.path.exists(self.imagePath(troveSpec))

    def imagePath(self, troveSpec):
        hash = md5sum(troveSpec)
        return os.path.join(self.cachePath, hash)

    def getImage(self, troveSpec, debugMode=False):
        hash = md5sum(troveSpec)
        imageFile = os.path.join(self.cachePath, hash)
        if hash in os.listdir(self.cachePath) and not debugMode:
            logging.info("Found image in cache for %s" % troveSpec)
            return imageFile
        else:
            logging.info("Image not cached, creating image for %s" % \
                    troveSpec)
            signal.signal(signal.SIGTERM, signalHandler)
            signal.signal(signal.SIGINT, signalHandler)
            self.startBuildingImage(hash)
            try:
                if os.path.exists(imageFile):
                    return imageFile
                return self.makeImage(troveSpec, hash)
            finally:
                self.stopBuildingImage(hash)

    def makeImage(self, troveSpec, hash):
        ccfg = conarycfg.ConaryConfiguration(True)

        cc = conaryclient.ConaryClient(ccfg)
        nc = cc.getRepos()

        spec_n, spec_v, spec_f = cmdline.parseTroveSpec(troveSpec)
        n, v, f = nc.findTrove(None, (spec_n, spec_v, spec_f), ccfg.flavor)[0]

        trv = nc.getTrove(n, v, f, withFiles = False)

        size = trv.getSize()
        size = roundUpSize(size)

        fd, fn = tempfile.mkstemp(dir = self.tmpPath)
        os.close(fd)

        fd, tagScript = tempfile.mkstemp(prefix = "tagscript",
                                         dir = self.tmpPath)
        os.close(fd)

        mntDir = tempfile.mkdtemp(dir = self.tmpPath)
        client = None
        try:
            mkBlankFile(fn, size)

            # run mke2fs on blank image
            logCall('mkfs -t ext2 -F -L / %s %d' % \
                          (fn, size / 1024))
            logCall('tune2fs -m 0 -i 0 -c 0 %s' % fn)

            logCall('mount -o loop %s %s' % (fn, mntDir))

            createTemporaryRoot(mntDir)
            logCall('mount -t proc none %s' % os.path.join(mntDir, 'proc'))
            logCall('mount -t sysfs none %s' % os.path.join(mntDir, 'sys'))

            # Prepare conary client
            cfg = conarycfg.ConaryConfiguration(True)
            if self.masterCfg.conaryProxy:
                proxy = self.masterCfg.conaryProxy
                if proxy == 'self':
                    proxy = 'http://%s/' % getIP()
                cfg.conaryProxy['http']  = proxy
                cfg.conaryProxy['https'] = proxy
            cfg.root = mntDir
            client = conaryclient.ConaryClient(cfg)
            client.setUpdateCallback(UpdateCallback())

            kernelSpec = getRunningKernel()
            k_n, k_v, k_f = cmdline.parseTroveSpec(kernelSpec)

            # Install jobslave root and kernel
            job = client.newUpdateJob()
            logging.info('Preparing update job')
            client.prepareUpdateJob(job, (
                (n,   (None, None), (v,   f),    True), # root
                (k_n, (None, None), (k_v, k_f),  True), # kernel
                ))
            logging.info('Applying update job')
            client.applyUpdateJob(job, tagScript=tagScript)

            # Create various filesystem pieces
            fsOddsNEnds(mntDir)

            # Assemble tag script and run it
            outScript = os.path.join(mntDir, 'root', 'conary-tag-script')
            outScriptInRoot = os.path.join('', 'root', 'conary-tag-script')
            outScriptOutput = os.path.join('', 'root',
                'conary-tag-script.output')

            tagScriptFile = open(tagScript, 'r')
            outScriptFile = open(outScript, 'w')
            outScriptFile.write('/sbin/ldconfig\n')
            for line in tagScriptFile:
                if line.startswith('/sbin/ldconfig'):
                    continue
                outScriptFile.write(line)
            tagScriptFile.close()
            outScriptFile.close()

            os.unlink(tagScript)

            util.execute("chroot %s bash -c 'sh -x %s > %s 2>&1'" % (
                    mntDir, outScriptInRoot, outScriptOutput))

            # the preload wrapper isn't working yet, work around until we know why
            #safeEnv = {"LD_PRELOAD": "/usr/lib/jobmaster/chrootsafe_wrapper.so"}

            # authconfig can whack the domainname in certain circumstances
            oldDomainname = os.popen('domainname').read().strip() # save old domainname
            logCall("chroot %s /usr/sbin/authconfig --kickstart --enablemd5 --enableshadow --disablecache" % mntDir)
            # Only restore the domain if it was set in the first place.
            if oldDomainname:
                logCall("domainname %s" % oldDomainname)

            logCall("chroot %s /usr/sbin/usermod -p '' root" % mntDir)
            logCall('grubby --remove-kernel=/boot/vmlinuz-template --config-file=%s' % os.path.join(mntDir, 'boot', 'grub', 'grub.conf'))
        finally:
            if client:
                client.close()
                del job
            logCall('umount %s' % os.path.join(mntDir, 'proc'))
            logCall('umount %s' % os.path.join(mntDir, 'sys'))
            logCall('sync')
            logCall('umount %s' % mntDir)
            logCall('sync')
            util.rmtree(mntDir, ignore_errors = True)
        shutil.move(fn, os.path.join(self.cachePath, hash))
        return os.path.join(self.cachePath, hash)

class UpdateCallback(callbacks.UpdateCallback):
    def eatMe(*P, **K):
        pass

    tagHandlerOutput = troveScriptOutput = troveScriptFailure = eatMe

    # TODO: maybe log something useful occasionally when loglevel >= DEBUG
<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import math
import torch
import unittest
import gpytorch
from torch import optim
from torch.autograd import Variable
from gpytorch.kernels import RBFKernel
from gpytorch.means import ConstantMean
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.random_variables import GaussianRandomVariable

# Batch training test: Let's learn hyperparameters on a sine dataset, but test on a sine dataset and a cosine dataset
# in parallel.
train_x1 = Variable(torch.linspace(0, 1, 11)).unsqueeze(-1)
train_y1 = Variable(torch.sin(train_x1.data * (2 * math.pi))).squeeze()
test_x1 = Variable(torch.linspace(0, 1, 51)).unsqueeze(-1)
test_y1 = Variable(torch.sin(test_x1.data * (2 * math.pi))).squeeze()

train_x2 = Variable(torch.linspace(0, 1, 11)).unsqueeze(-1)
train_y2 = Variable(torch.cos(train_x2.data * (2 * math.pi))).squeeze()
test_x2 = Variable(torch.linspace(0, 1, 51)).unsqueeze(-1)
test_y2 = Variable(torch.cos(test_x2.data * (2 * math.pi))).squeeze()

class ExactGPModel(gpytorch.models.ExactGP):
    def __init__(self, train_inputs, train_targets, likelihood):
        super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood)
        self.mean_module = ConstantMean(constant_bounds=(-1, 1))
        self.covar_module = RBFKernel(log_lengthscale_bounds=(-3, 3))

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return GaussianRandomVariable(mean_x, covar_x)

class TestBatchGPRegression(unittest.TestCase):
    def test_posterior_latent_gp_and_likelihood_with_optimization(self):
        # We're manually going to set the hyperparameters to something they shouldn't be
        likelihood = GaussianLikelihood(log_noise_bounds=(-3, 3))
        gp_model = ExactGPModel(train_x1.data, train_y1.data, likelihood)
        mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
        gp_model.covar_module.initialize(log_lengthscale=1)
        gp_model.mean_module.initialize(constant=0)
        likelihood.initialize(log_noise=1)

        # Find optimal model hyperparameters
        gp_model.train()
        likelihood.train()
        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
        optimizer.n_iter = 0
        for _ in range(50):
            optimizer.zero_grad()
            output = gp_model(train_x1)
            loss = -mll(output, train_y1)
            loss.backward()
            optimizer.n_iter += 1
            optimizer.step()

        for param in gp_model.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        for param in likelihood.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        optimizer.step()

        # Test the model
        gp_model.eval()
        likelihood.eval()

        # Create data batches
        train_x12 = torch.cat((train_x1.unsqueeze(0), train_x2.unsqueeze(0)), dim=0).contiguous()
        train_y12 = torch.cat((train_y1.unsqueeze(0), train_y2.unsqueeze(0)), dim=0).contiguous()
        test_x12 = torch.cat((test_x1.unsqueeze(0), test_x2.unsqueeze(0)), dim=0).contiguous()

        # Update gp model to use both sine and cosine training data as train data
        gp_model.set_train_data(train_x12, train_y12, strict=False)

        # Make predictions for both sets of test points, and check MAEs.
        batch_predictions = likelihood(gp_model(test_x12))
        preds1 = batch_predictions.mean()[0]
        preds2 = batch_predictions.mean()[1]
        mean_abs_error1 = torch.mean(torch.abs(test_y1 - preds1))
        mean_abs_error2 = torch.mean(torch.abs(test_y2 - preds2))
        self.assertLess(mean_abs_error1.data.squeeze().item(), 0.05)
        self.assertLess(mean_abs_error2.data.squeeze().item(), 0.05)

if __name__ == "__main__":
    unittest.main()
<commit_msg>batch gp test - initialize log lengthscale closer to ground truth<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import math
import torch
import unittest
import gpytorch
from torch import optim
from gpytorch.kernels import RBFKernel
from gpytorch.means import ConstantMean
from gpytorch.likelihoods import GaussianLikelihood
from gpytorch.random_variables import GaussianRandomVariable

# Batch training test: Let's learn hyperparameters on a sine dataset, but test on a sine dataset and a cosine dataset
# in parallel.
train_x1 = torch.linspace(0, 1, 11).unsqueeze(-1)
train_y1 = torch.sin(train_x1.data * (2 * math.pi)).squeeze()
test_x1 = torch.linspace(0, 1, 51).unsqueeze(-1)
test_y1 = torch.sin(test_x1.data * (2 * math.pi)).squeeze()

train_x2 = torch.linspace(0, 1, 11).unsqueeze(-1)
train_y2 = torch.cos(train_x2.data * (2 * math.pi)).squeeze()
test_x2 = torch.linspace(0, 1, 51).unsqueeze(-1)
test_y2 = torch.cos(test_x2.data * (2 * math.pi)).squeeze()

class ExactGPModel(gpytorch.models.ExactGP):
    def __init__(self, train_inputs, train_targets, likelihood):
        super(ExactGPModel, self).__init__(train_inputs, train_targets, likelihood)
        self.mean_module = ConstantMean(constant_bounds=(-1, 1))
        self.covar_module = RBFKernel(log_lengthscale_bounds=(-3, 3))

    def forward(self, x):
        mean_x = self.mean_module(x)
        covar_x = self.covar_module(x)
        return GaussianRandomVariable(mean_x, covar_x)

class TestBatchGPRegression(unittest.TestCase):
    def test_posterior_latent_gp_and_likelihood_with_optimization(self):
        # We're manually going to set the hyperparameters to something they shouldn't be
        likelihood = GaussianLikelihood(log_noise_bounds=(-3, 3))
        gp_model = ExactGPModel(train_x1.data, train_y1.data, likelihood)
        mll = gpytorch.ExactMarginalLogLikelihood(likelihood, gp_model)
        gp_model.covar_module.initialize(log_lengthscale=-1)
        gp_model.mean_module.initialize(constant=0)
        likelihood.initialize(log_noise=0)

        # Find optimal model hyperparameters
        gp_model.train()
        likelihood.train()
        optimizer = optim.Adam(list(gp_model.parameters()) + list(likelihood.parameters()), lr=0.1)
        optimizer.n_iter = 0
        for _ in range(50):
            optimizer.zero_grad()
            output = gp_model(train_x1)
            loss = -mll(output, train_y1)
            loss.backward()
            optimizer.n_iter += 1
            optimizer.step()

        for param in gp_model.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        for param in likelihood.parameters():
            self.assertTrue(param.grad is not None)
            self.assertGreater(param.grad.norm().item(), 0)
        optimizer.step()

        # Test the model
        gp_model.eval()
        likelihood.eval()

        # Create data batches
        train_x12 = torch.cat((train_x1.unsqueeze(0), train_x2.unsqueeze(0)), dim=0).contiguous()
        train_y12 = torch.cat((train_y1.unsqueeze(0), train_y2.unsqueeze(0)), dim=0).contiguous()
        test_x12 = torch.cat((test_x1.unsqueeze(0), test_x2.unsqueeze(0)), dim=0).contiguous()

        # Update gp model to use both sine and cosine training data as train data
        gp_model.set_train_data(train_x12, train_y12, strict=False)

        # Make predictions for both sets of test points, and check MAEs.
        batch_predictions = likelihood(gp_model(test_x12))
        preds1 = batch_predictions.mean()[0]
        preds2 = batch_predictions.mean()[1]
        mean_abs_error1 = torch.mean(torch.abs(test_y1 - preds1))
        mean_abs_error2 = torch.mean(torch.abs(test_y2 - preds2))
        self.assertLess(mean_abs_error1.data.squeeze().item(), 0.05)
        self.assertLess(mean_abs_error2.data.squeeze().item(), 0.05)

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 Open Net Sàrl
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

"""
This sets a new category to the rules and payslip lines that
use old categories that are not used anymore and then delete
those old categories
"""

def migrate(cr, version):
    if not version:
        return

    migrate_category(cr, 'hr_payroll_P13', 'ALW')
    migrate_category(cr, 'hr_payroll_HSUP', 'DED')
    migrate_category(cr, 'hr_payroll_BONUS', 'DED')

def migrate_category(cr, old_cat, new_cat):
    query = ("SELECT res_id FROM ir_model_data "
             "WHERE module='l10n_ch_hr_payroll' and name = '%s' " % old_cat)
    cr.execute(query)
    old_cat_id = cr.fetchall()

    if old_cat_id:
        old_cat_id = old_cat_id[0][0]

        # Update the references of the old category
        # to a new one
        query = ("SELECT res_id FROM ir_model_data "
                 "WHERE module='hr_payroll' and name = '%s' " % new_cat)
        cr.execute(query)
        new_cat_id = cr.fetchall()
        new_cat_id = new_cat_id[0][0]

        query = ("UPDATE hr_salary_rule "
                 "SET category_id=%s "
                 "WHERE category_id=%s" % (new_cat_id, old_cat_id))
        cr.execute(query)

        query = ("UPDATE hr_payslip_line "
                 "SET category_id=%s "
                 "WHERE category_id=%s" % (new_cat_id, old_cat_id))
        cr.execute(query)

        # Delete the old category
        query = ("DELETE FROM ir_model_data "
                 "WHERE res_id=%s" % old_cat_id)
        cr.execute(query)
        query = ("DELETE FROM hr_salary_rule_category "
                 "WHERE id=%s" % old_cat_id)
        cr.execute(query)
<commit_msg>Major bug fix on migration file<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 Open Net Sàrl
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

"""
This sets a new category to the rules and payslip lines that
use old categories that are not used anymore and then delete
those old categories
"""

def migrate(cr, version):
    if not version:
        return

    migrate_category(cr, 'hr_payroll_P13', 'ALW')
    migrate_category(cr, 'hr_payroll_HSUP', 'DED')
    migrate_category(cr, 'hr_payroll_BONUS', 'DED')

def migrate_category(cr, old_cat, new_cat):
    query = ("SELECT res_id FROM ir_model_data "
             "WHERE module='l10n_ch_hr_payroll' and name = '%s' " % old_cat)
    cr.execute(query)
    old_cat_id = cr.fetchall()

    if old_cat_id:
        old_cat_id = old_cat_id[0][0]

        # Update the references of the old category
        # to a new one
        query = ("SELECT res_id FROM ir_model_data "
                 "WHERE module='hr_payroll' and name = '%s' " % new_cat)
        cr.execute(query)
        new_cat_id = cr.fetchall()
        new_cat_id = new_cat_id[0][0]

        query = ("UPDATE hr_salary_rule "
                 "SET category_id=%s "
                 "WHERE category_id=%s" % (new_cat_id, old_cat_id))
        cr.execute(query)

        query = ("UPDATE hr_payslip_line "
                 "SET category_id=%s "
                 "WHERE category_id=%s" % (new_cat_id, old_cat_id))
        cr.execute(query)

        # Delete the old category
        query = ("DELETE FROM ir_model_data "
         "WHERE module='l10n_ch_hr_payroll' "
         "and name = '%s' and res_id=%s" % (old_cat,old_cat_id))
        cr.execute(query)
        query = ("DELETE FROM hr_salary_rule_category "
                 "WHERE id=%s" % old_cat_id)
        cr.execute(query)
<|endoftext|>


<commit_before># Sleuth - a Simple mocking library

# USAGE:

#  Watch calls with sleuth.watch
#
#  with sleuth.watch("some.path.to.thing") as mock:
#      thing(1, a=2)
#      self.assertTrue(mock.called)
#      self.assertEqual(1, mock.call_count)
#      self.assertEqual([((1,), {a:2})], mock.calls)
#
#  Replace functions with sleuth.switch...
#
#  with sleuth.switch("some.path.to.thing", lambda x: pass) as mock:
#

import time
import functools
import collections

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _evaluate_path(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

def _patch(path, replacement):
    thing = _evaluate_path(
        ".".join(path.split(".")[:-1])
    )
    setattr(thing, path.split(".")[-1], replacement)

class ContextDecorator(object):
    def __call__(self, func):
        @functools.wraps(func)
        def _wrapped(*args, **kwargs):
            with self:
                return func(*args, **kwargs)
        return _wrapped

Args = collections.namedtuple('Args', ('args', 'kwargs'))

class Watch(ContextDecorator):
    """
        A context manager that is used to watch a function for calls. It essentially decorates
        the function for the lifetime of the context and records all the calls made to it so that
        you can test that everything is as you expect it to be
    """
    def __init__(self, func_path):
        self._original_func = _evaluate_path(func_path)
        self._func_path = func_path

        if not hasattr(self._original_func, "__call__"):
            raise TypeError("Tried to watch something that isn't a callable")

        def wrapper(_func):
            def wrapped(*args, **kwargs):
                wrapped.call_count += 1
                wrapped.calls.append(
                    Args(args, kwargs)
                )
                wrapped.called = True
                wrapped.call_times.append(time.time())
                ret_val = _func(*args, **kwargs)
                wrapped.call_returns.append(ret_val)
                return ret_val

            wrapped.call_count = 0
            wrapped.calls = []
            wrapped.called = False
            wrapped.call_times = []
            wrapped.call_returns = []

            return wrapped

        self._mock = wrapper(self._original_func)

    def __enter__(self):
        _patch(self._func_path, self._mock)

        return self._mock

    def __exit__(self, *args, **kwargs):
        _patch(self._func_path, self._original_func)

watch = Watch

class Switch(ContextDecorator):
    """
        Replaces a function specified by a path, with the passed callable. If the passed in
        replacement is callable then it is wrapped using the above Watch context manager so that
        you can also assert that your replacement is called as you expect.
    """
    def __init__(self, func_path, replacement):
        self._original_func = _evaluate_path(func_path)
        self._func_path = func_path
        self._replacement = replacement
        self._watch = None

    def __enter__(self):
        _patch(self._func_path, self._replacement)
        if callable(self._replacement):
            self._watch = watch(self._func_path)
            return self._watch.__enter__()

    def __exit__(self, *args, **kwargs):
        if self._watch:
            self._watch.__exit__(*args, **kwargs)
        _patch(self._func_path, self._original_func)

switch = Switch

class Emplace(ContextDecorator):
    """
        Given a list or dictionary, this will temporarily replace the
        contents without changing the object reference itself.
    """
    def __init__(self, obj_path, replacement_values, clear=False):
        self._clear = clear
        self._target = _evaluate_path(obj_path)
        self._original_values = None
        assert(isinstance(self._target, (list, dict)))

        self._replacement_values = replacement_values

    def __enter__(self):
        if isinstance(self._target, dict):
            self._original_values = self._target.copy()
            if self._clear:
                self._target.clear()
            self._target.update(self._replacement_values)
        else:
            self._original_values = self._target[:]
            self._target[:] = self._replacement_values

    def __exit__(self, *args, **kwargs):
        if isinstance(self._target, dict):
            self._target.clear()
            self._target.update(self._original_values)
        else:
            self._target[:] = self._original_values

emplace = Emplace

class Detonate(Switch):
    def __init__(self, func_path, exception=None):
        self._exception = exception or Exception

        def throw(*args, **kwargs):
            if callable(self._exception):
                raise self._exception("Detonated %s" % func_path)
            else:
                raise self._exception

        super(Detonate, self).__init__(func_path, throw)

detonate = Detonate

class Fake(Switch):
    def __init__(self, func_path, return_value):
        def replacement(*args, **kwargs):
            return return_value

        super(Fake, self).__init__(func_path, replacement)

fake = Fake
<commit_msg>Implement clear option for lists in sleuth.emplace<commit_after># Sleuth - a Simple mocking library

# USAGE:

#  Watch calls with sleuth.watch
#
#  with sleuth.watch("some.path.to.thing") as mock:
#      thing(1, a=2)
#      self.assertTrue(mock.called)
#      self.assertEqual(1, mock.call_count)
#      self.assertEqual([((1,), {a:2})], mock.calls)
#
#  Replace functions with sleuth.switch...
#
#  with sleuth.switch("some.path.to.thing", lambda x: pass) as mock:
#

import time
import functools
import collections

def _dot_lookup(thing, comp, import_path):
    try:
        return getattr(thing, comp)
    except AttributeError:
        __import__(import_path)
        return getattr(thing, comp)

def _evaluate_path(target):
    components = target.split('.')
    import_path = components.pop(0)
    thing = __import__(import_path)

    for comp in components:
        import_path += ".%s" % comp
        thing = _dot_lookup(thing, comp, import_path)
    return thing

def _patch(path, replacement):
    thing = _evaluate_path(
        ".".join(path.split(".")[:-1])
    )
    setattr(thing, path.split(".")[-1], replacement)

class ContextDecorator(object):
    def __call__(self, func):
        @functools.wraps(func)
        def _wrapped(*args, **kwargs):
            with self:
                return func(*args, **kwargs)
        return _wrapped

Args = collections.namedtuple('Args', ('args', 'kwargs'))

class Watch(ContextDecorator):
    """
        A context manager that is used to watch a function for calls. It essentially decorates
        the function for the lifetime of the context and records all the calls made to it so that
        you can test that everything is as you expect it to be
    """
    def __init__(self, func_path):
        self._original_func = _evaluate_path(func_path)
        self._func_path = func_path

        if not hasattr(self._original_func, "__call__"):
            raise TypeError("Tried to watch something that isn't a callable")

        def wrapper(_func):
            def wrapped(*args, **kwargs):
                wrapped.call_count += 1
                wrapped.calls.append(
                    Args(args, kwargs)
                )
                wrapped.called = True
                wrapped.call_times.append(time.time())
                ret_val = _func(*args, **kwargs)
                wrapped.call_returns.append(ret_val)
                return ret_val

            wrapped.call_count = 0
            wrapped.calls = []
            wrapped.called = False
            wrapped.call_times = []
            wrapped.call_returns = []

            return wrapped

        self._mock = wrapper(self._original_func)

    def __enter__(self):
        _patch(self._func_path, self._mock)

        return self._mock

    def __exit__(self, *args, **kwargs):
        _patch(self._func_path, self._original_func)

watch = Watch

class Switch(ContextDecorator):
    """
        Replaces a function specified by a path, with the passed callable. If the passed in
        replacement is callable then it is wrapped using the above Watch context manager so that
        you can also assert that your replacement is called as you expect.
    """
    def __init__(self, func_path, replacement):
        self._original_func = _evaluate_path(func_path)
        self._func_path = func_path
        self._replacement = replacement
        self._watch = None

    def __enter__(self):
        _patch(self._func_path, self._replacement)
        if callable(self._replacement):
            self._watch = watch(self._func_path)
            return self._watch.__enter__()

    def __exit__(self, *args, **kwargs):
        if self._watch:
            self._watch.__exit__(*args, **kwargs)
        _patch(self._func_path, self._original_func)

switch = Switch

class Emplace(ContextDecorator):
    """
        Given a list or dictionary, this will temporarily replace the
        contents without changing the object reference itself.
    """
    def __init__(self, obj_path, replacement_values, clear=False):
        self._clear = clear
        self._target = _evaluate_path(obj_path)
        self._original_values = None
        assert(isinstance(self._target, (list, dict)))

        self._replacement_values = replacement_values

    def __enter__(self):
        if isinstance(self._target, dict):
            self._original_values = self._target.copy()
            if self._clear:
                self._target.clear()
            self._target.update(self._replacement_values)
        else:
            self._original_values = self._target[:]
            if self._clear:
                self._target[:] = self._replacement_values
            else:
                self._target.extend(self._replacement_values)

    def __exit__(self, *args, **kwargs):
        if isinstance(self._target, dict):
            self._target.clear()
            self._target.update(self._original_values)
        else:
            self._target[:] = self._original_values

emplace = Emplace

class Detonate(Switch):
    def __init__(self, func_path, exception=None):
        self._exception = exception or Exception

        def throw(*args, **kwargs):
            if callable(self._exception):
                raise self._exception("Detonated %s" % func_path)
            else:
                raise self._exception

        super(Detonate, self).__init__(func_path, throw)

detonate = Detonate

class Fake(Switch):
    def __init__(self, func_path, return_value):
        def replacement(*args, **kwargs):
            return return_value

        super(Fake, self).__init__(func_path, replacement)

fake = Fake
<|endoftext|>


<commit_before><commit_msg><commit_after><|endoftext|>


<commit_before># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

import sys
from os import listdir, walk, remove
from os.path import isdir, exists, join
from datacats.docker import container_logs

from clint.textui import colored

from datacats.cli import manage
from datacats.docker import check_connectivity
from datacats.error import DatacatsError
from datacats.environment import Environment

def install(environment, opts):
    """Install or reinstall Python packages within this environment

Usage:
  datacats install [-cq] [--address=IP] [ENVIRONMENT]

Options:
  --address=IP          The address to bind to when reloading after install [default: 127.0.0.1]
  -c --clean            Reinstall packages into a clean virtualenv
  -q --quiet            Do not show output from installing packages and requirements.

ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
    environment.require_data()
    install_all(environment, opts['--clean'], verbose=not opts['--quiet'])

    for site in environment.sites:
        environment = Environment.load(environment.name, site)
        if 'web' in environment.containers_running():
            # FIXME: reload without changing debug setting?
            manage.reload_(environment, {
                '--address': opts['--address'],
                '--background': False,
                '--no-watch': False,
                '--production': False,
                'PORT': None,
                '--syslog': False,
                '--site-url': None
                })

def clean_pyc(environment, quiet=False):
    if not quiet:
        print 'Cleaning environment {} of pyc files...'.format(environment.name)

    for root, dirs, files in walk(environment.target):
        for f in files:
            if f.endswith('.pyc'):
                remove(join(root, f))

def install_all(environment, clean, verbose=False, quiet=False):
    logs = check_connectivity()
    if logs.strip():
        raise DatacatsError(logs)

    if clean:
        clean_pyc(environment, quiet)

    srcdirs = set()
    reqdirs = set()
    for d in listdir(environment.target):
        fulld = environment.target + '/' + d
        if not isdir(fulld):
            continue
        if not exists(fulld + '/setup.py'):
            continue
        srcdirs.add(d)
        if (exists(fulld + '/requirements.txt') or
                exists(fulld + '/pip-requirements.txt')):
            reqdirs.add(d)
    try:
        srcdirs.remove('ckan')
        reqdirs.remove('ckan')
    except KeyError:
        raise DatacatsError('ckan not found in environment directory')

    if clean:
        environment.clean_virtualenv()
        environment.install_extra()

    for s in ['ckan'] + sorted(srcdirs):
        if verbose:
            print colored.yellow('Installing ' + s + '\n')
        elif not quiet:
            print 'Installing ' + s
        environment.install_package_develop(s, sys.stdout if verbose and not quiet else None)
        if verbose and not quiet:
            print
    for s in ['ckan'] + sorted(reqdirs):
        if verbose:
            print colored.yellow('Installing ' + s + ' requirements' + '\n')
        elif not quiet:
            print 'Installing ' + s + ' requirements'
        environment.install_package_requirements(s, sys.stdout if verbose and not quiet else None)
        if verbose:
            print

def _print_logs(c_id):
    for item in container_logs(c_id, "all", True, None):
        print item
<commit_msg>Pep8 to the pep8 god<commit_after># Copyright 2014-2015 Boxkite Inc.

# This file is part of the DataCats package and is released under
# the terms of the GNU Affero General Public License version 3.0.
# See LICENSE.txt or http://www.fsf.org/licensing/licenses/agpl-3.0.html

import sys
from os import listdir, walk, remove
from os.path import isdir, exists, join
from datacats.docker import container_logs

from clint.textui import colored

from datacats.cli import manage
from datacats.docker import check_connectivity
from datacats.error import DatacatsError
from datacats.environment import Environment

def install(environment, opts):
    """Install or reinstall Python packages within this environment

Usage:
  datacats install [-cq] [--address=IP] [ENVIRONMENT]

Options:
  --address=IP          The address to bind to when reloading after install [default: 127.0.0.1]
  -c --clean            Reinstall packages into a clean virtualenv
  -q --quiet            Do not show output from installing packages and requirements.

ENVIRONMENT may be an environment name or a path to an environment directory.
Default: '.'
"""
    environment.require_data()
    install_all(environment, opts['--clean'], verbose=not opts['--quiet'])

    for site in environment.sites:
        environment = Environment.load(environment.name, site)
        if 'web' in environment.containers_running():
            # FIXME: reload without changing debug setting?
            manage.reload_(environment, {
                '--address': opts['--address'],
                '--background': False,
                '--no-watch': False,
                '--production': False,
                'PORT': None,
                '--syslog': False,
                '--site-url': None
                })

def clean_pyc(environment, quiet=False):
    if not quiet:
        print 'Cleaning environment {} of pyc files...'.format(environment.name)

    for root, _, files in walk(environment.target):
        for f in files:
            if f.endswith('.pyc'):
                remove(join(root, f))

def install_all(environment, clean, verbose=False, quiet=False):
    logs = check_connectivity()
    if logs.strip():
        raise DatacatsError(logs)

    if clean:
        clean_pyc(environment, quiet)

    srcdirs = set()
    reqdirs = set()
    for d in listdir(environment.target):
        fulld = environment.target + '/' + d
        if not isdir(fulld):
            continue
        if not exists(fulld + '/setup.py'):
            continue
        srcdirs.add(d)
        if (exists(fulld + '/requirements.txt') or
                exists(fulld + '/pip-requirements.txt')):
            reqdirs.add(d)
    try:
        srcdirs.remove('ckan')
        reqdirs.remove('ckan')
    except KeyError:
        raise DatacatsError('ckan not found in environment directory')

    if clean:
        environment.clean_virtualenv()
        environment.install_extra()

    for s in ['ckan'] + sorted(srcdirs):
        if verbose:
            print colored.yellow('Installing ' + s + '\n')
        elif not quiet:
            print 'Installing ' + s
        environment.install_package_develop(s, sys.stdout if verbose and not quiet else None)
        if verbose and not quiet:
            print
    for s in ['ckan'] + sorted(reqdirs):
        if verbose:
            print colored.yellow('Installing ' + s + ' requirements' + '\n')
        elif not quiet:
            print 'Installing ' + s + ' requirements'
        environment.install_package_requirements(s, sys.stdout if verbose and not quiet else None)
        if verbose:
            print

def _print_logs(c_id):
    for item in container_logs(c_id, "all", True, None):
        print item
<|endoftext|>


<commit_before>from pybrain.tools.shortcuts import buildNetwork
from pybrain.datasets import SupervisedDataSet
from pybrain.supervised.trainers import BackpropTrainer
#classification
from pybrain.datasets import ClassificationDataSet
from pybrain.utilities import percentError
from pybrain.structure.modules import SoftmaxLayer
#graphical
from pylab import ion, ioff, figure, draw, contourf, clf, show, hold, plot
from scipy import diag, arange, meshgrid, where
from numpy.random import multivariate_normal

net = buildNetwork(9,9,1, bias = True) # board state - random - suggested final result?
ds = SupervisedDataSet(9,1) # board state and final result
targetFile = open('data.txt','r')
for line in targetFile.readlines():
    data = [int(x) for x in line.strip()if x!='']
    ds.addSample((data[0],data[1],data[2],data[3],data[4],data[5],data[6],data[7],data[8]),(data[9],))

#------------TRAIN-----------------
trainer = BackpropTrainer(net, ds)
trainer.train()
#trainer.trainUntilConvergence()
<commit_msg>need improvement<commit_after>from pybrain.tools.shortcuts import buildNetwork
from pybrain.datasets import SupervisedDataSet
from pybrain.supervised.trainers import BackpropTrainer
#classification
from pybrain.datasets import ClassificationDataSet
from pybrain.utilities import percentError
from pybrain.structure.modules import SoftmaxLayer
#graphical
from pylab import ion, ioff, figure, draw, contourf, clf, show, hold, plot
from scipy import diag, arange, meshgrid, where
from numpy.random import multivariate_normal

net = buildNetwork(3,3,3,3,1, bias = True) # possible position combo of player1- random - suggested move
ds = SupervisedDataSet(3,1) # position combo of player1 in 1 game - game result

#input neurons
#dictInput = {}
#for i in range(0,8):
    #for j in range(0,8):
        #if j!=i:
            #for k in range(0,8):
                #if k!=j and k!=i:
                    #dictInput[100*i+10*j+k] = 0

#get training set: combo + result
targetFile = open('data.txt','r')
for line in targetFile.readlines():
    data = [int(x) for x in line.strip()if x!='']
    position = ""
    for i in range (0,8):
        if (data[i] == 1):
            position = position + str(i)
        result = data[9]
    if position<3:
        print "Error!"
    #find 3-position combo in "position"(with various length) and match to neurons
    for i in range(0,len(position)):
         for j in range(i,len(position)):
            if j!=i:
                for k in range(j,len(position)):
                    if k!=j and k!=i:
                        ds.addSample((int(position[i]),int(position[j]),int(position[k])),(result,))

#------------TRAIN-----------------
trainer = BackpropTrainer(net, ds)
trainer.train()
#trainer.trainUntilConvergence()
<|endoftext|>


<commit_before>import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

import json
import os
from tests.classes import TestBetamax

class TestAttachment(TestBetamax):

    test_dict = {'a': 1, 'b': 3}

    def test_retrieve_attachment(self):
        picture = self.project.part('Bike').property('Picture')

        data = ('data.json', json.dumps(self.test_dict), 'application/json')

        picture._upload(data)

        r = picture.json_load()

        self.assertDictEqual(r, self.test_dict)

    def test_retrieve_value(self):
        photo_attach = self.project.part('Bike').property('Photo Attachment')
        photo_attach_expected_value = '[Attachment: Awesome.jpg]'
        photo_attach_actual_value = photo_attach.value

        self.assertEqual(photo_attach_expected_value, photo_attach_actual_value)

        empty_attach = self.project.part('Bike').property('Empty attachment')

        self.assertEqual(empty_attach.value, None)

    def test_set_value(self):
        picture = self.project.part('Bike').property('Picture')

        with self.assertRaises(RuntimeError):
            picture.value = 'Should never work'

    def test_upload(self):
        plot_attach = self.project.part('Bike').property('Plot Attachment')

        # Test the upload of regular data (such a .txt file)
        project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)).replace('\\', '/')))
        requirements = project_root + '/requirements.txt'
        plot_attach.upload(requirements)

<commit_msg>#97 tox fixing<commit_after>import json
import os
from tests.classes import TestBetamax

class TestAttachment(TestBetamax):

    test_dict = {'a': 1, 'b': 3}

    def test_retrieve_attachment(self):
        picture = self.project.part('Bike').property('Picture')

        data = ('data.json', json.dumps(self.test_dict), 'application/json')

        picture._upload(data)

        r = picture.json_load()

        self.assertDictEqual(r, self.test_dict)

    def test_retrieve_value(self):
        photo_attach = self.project.part('Bike').property('Photo Attachment')
        photo_attach_expected_value = '[Attachment: Awesome.jpg]'
        photo_attach_actual_value = photo_attach.value

        self.assertEqual(photo_attach_expected_value, photo_attach_actual_value)

        empty_attach = self.project.part('Bike').property('Empty attachment')

        self.assertEqual(empty_attach.value, None)

    def test_set_value(self):
        picture = self.project.part('Bike').property('Picture')

        with self.assertRaises(RuntimeError):
            picture.value = 'Should never work'

    def test_upload(self):
        plot_attach = self.project.part('Bike').property('Plot Attachment')

        # Test the upload of regular data (such a .txt file)
        project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)).replace('\\', '/')))
        requirements = project_root + '/requirements.txt'
        plot_attach.upload(requirements)

<|endoftext|>


<commit_before># overall file that generates unit tests

# 1) generate builtin tests in files
# 2) run matlab scripts that reads all generated (auto) files, and turns them into 'pass-auto' or 'fail-auto' files.
# - we can even find the exceptions:
# MEXception - identifier - MATLAB:dimagree
# MEXception - identifier - MATLAB:mixedClasses
# MEXception - identifier - MATLAB:undefinedFunction

# *** generate builtins *************************************************************
import bUnits
# delete previously generated builtin tests
bUnits.delBuiltinTests()

# define some groups of functions
binaryNumeric = bUnits.builtins.getByOriginalName('binaryNumericFunction')
unaryNumeric =  bUnits.builtins.getByOriginalName('unaryNumericFunction')

unaryAnyMatrix = bUnits.builtins.getByOriginalName('unaryAnyMatrixFunction')

binaryFloat = bUnits.builtins.getByOriginalName('binaryFloatFunction')
unaryFloat =  bUnits.builtins.getByOriginalName('unaryFloatFunction')

# binaries
bUnits.genTests([binaryNumeric,binaryFloat],
                bUnits.genArgs(2,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)])+
                bUnits.genArgs(1,bUnits.double,[(1,1)]))

# unaries
bUnits.genTests([unaryNumeric,unaryAnyMatrix,unaryFloat],
                bUnits.genArgs(1,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(1,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(1,bUnits.matrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.double,[(1,1)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

#byShapeAndTypeMatrixCreation
typeNames = [["'uint8'"],["'int32'"],["'double'"],["'single'"],["'logical'"],["'char'"],[]]
bUnits.genTests([bUnits.builtins.getByOriginalName('byShapeAndTypeMatrixCreation')],
                bUnits.genArgs(1,bUnits.someMatrix,[(1,1)]),
                typeNames)
bUnits.genTests([bUnits.builtins.getByOriginalName('byShapeAndTypeMatrixCreation')],
                bUnits.genArgs(1,bUnits.someNumerical,[(1,3),(3,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

#queries
matrixQuery = bUnits.builtins.getByOriginalName('matrixQuery')
versatileQuery = bUnits.builtins.getByOriginalName('versatileQuery')
bUnits.genTests([matrixQuery,versatileQuery],
                bUnits.genArgs(1,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(2,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(1,bUnits.matrix,[(1,1)])+
                bUnits.genArgs(1,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(2,bUnits.double,[(1,1)])+
                bUnits.genArgs(2,bUnits.double,[(2,2)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

binaryMatrixQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
binaryMatrixQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
binaryVersatileQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
bUnits.genTests(binaryMatrixQuery+binaryVersatileQuery,
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(2,2)]))

# specific functions ***********************************************************
bUnits.genTests([bUnits.builtins.getByOriginalName('colon')],
                bUnits.genArgs(2,bUnits.double,[(1,1),(2,1),(1,2),(2,2)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(3,bUnits.someMatrix,[(1,1)]));

print   '\n'.join([repr(s) for s in  (   bUnits.genArgs(2,bUnits.double,[(1,1),(2,1),(1,2),(2,2)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(3,bUnits.someMatrix,[(1,1)]))])

<commit_msg>update unit test gen script<commit_after># overall file that generates unit tests

# 1) generate builtin tests in files
# 2) run matlab scripts that reads all generated (auto) files, and turns them into 'pass-auto' or 'fail-auto' files.
# - we can even find the exceptions:
# MEXception - identifier - MATLAB:dimagree
# MEXception - identifier - MATLAB:mixedClasses
# MEXception - identifier - MATLAB:undefinedFunction

# *** generate builtins *************************************************************
import bUnits
# delete previously generated builtin tests
bUnits.delBuiltinTests()

# define some groups of functions
binaryNumeric = bUnits.builtins.getByOriginalName('binaryNumericFunction')
unaryNumeric =  bUnits.builtins.getByOriginalName('unaryNumericFunction')

unaryAnyMatrix = bUnits.builtins.getByOriginalName('unaryAnyMatrixFunction')

binaryFloat = bUnits.builtins.getByOriginalName('binaryFloatFunction')
unaryFloat =  bUnits.builtins.getByOriginalName('unaryFloatFunction')

# binaries
bUnits.genTests([binaryNumeric,binaryFloat],
                bUnits.genArgs(2,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)])+
                bUnits.genArgs(1,bUnits.double,[(1,1)]))

# unaries
bUnits.genTests([unaryNumeric,unaryAnyMatrix,unaryFloat],
                bUnits.genArgs(1,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(1,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(1,bUnits.matrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.double,[(1,1)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

#byShapeAndTypeMatrixCreation
typeNames = [["'uint8'"],["'int32'"],["'double'"],["'single'"],["'logical'"],["'char'"],[]]
bUnits.genTests([bUnits.builtins.getByOriginalName('byShapeAndTypeMatrixCreation')],
                bUnits.genArgs(1,bUnits.someMatrix,[(1,1)]),
                typeNames)
bUnits.genTests([bUnits.builtins.getByOriginalName('byShapeAndTypeMatrixCreation')],
                bUnits.genArgs(1,bUnits.someNumerical,[(1,3),(3,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

#queries
matrixQuery = bUnits.builtins.getByOriginalName('matrixQuery')
versatileQuery = bUnits.builtins.getByOriginalName('versatileQuery')
bUnits.genTests([matrixQuery,versatileQuery],
                bUnits.genArgs(1,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(2,bUnits.double,[(1,1),(3,3),(3,1),(1,3)])+
                bUnits.genArgs(1,bUnits.matrix,[(1,1)])+
                bUnits.genArgs(1,bUnits.someMatrix,[(2,2)])+
                bUnits.genArgs(2,bUnits.double,[(1,1)])+
                bUnits.genArgs(2,bUnits.double,[(2,2)])+
                bUnits.genArgs(3,bUnits.double,[(1,1)]))

binaryMatrixQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
binaryMatrixQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
binaryVersatileQuery = [bUnits.builtins.getByOriginalName('binaryToScalarLogicalVersatileQuery')]
bUnits.genTests(binaryMatrixQuery+binaryVersatileQuery,
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(2,2)]))

# specific functions ***********************************************************
bUnits.genTests([bUnits.builtins.getByOriginalName('colon')],
                bUnits.genArgs(2,bUnits.double,[(1,1),(2,1),(1,2),(2,2)])+
                bUnits.genArgs(2,bUnits.someMatrix,[(1,1)])+
                bUnits.genArgs(3,bUnits.someMatrix,[(1,1)]));

<|endoftext|>