<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
# vim: expandtab:tabstop=4:shiftwidth=4
'''
Custom filters for use in openshift-ansible
'''

from ansible import errors
from operator import itemgetter
import OpenSSL.crypto
import os.path
import pdb
import re
import json

class FilterModule(object):
    ''' Custom ansible filters '''

    @staticmethod
    def oo_pdb(arg):
        ''' This pops you into a pdb instance where arg is the data passed in
            from the filter.
            Ex: "{{ hostvars | oo_pdb }}"
        '''
        pdb.set_trace()
        return arg

    @staticmethod
    def get_attr(data, attribute=None):
        ''' This looks up dictionary attributes of the form a.b.c and returns
            the value.
            Ex: data = {'a': {'b': {'c': 5}}}
                attribute = "a.b.c"
                returns 5
        '''
        if not attribute:
            raise errors.AnsibleFilterError("|failed expects attribute to be set")

        ptr = data
        for attr in attribute.split('.'):
            ptr = ptr[attr]

        return ptr

    @staticmethod
    def oo_flatten(data):
        ''' This filter plugin will flatten a list of lists
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to flatten a List")

        return [item for sublist in data for item in sublist]

    @staticmethod
    def oo_collect(data, attribute=None, filters=None):
        ''' This takes a list of dict and collects all attributes specified into a
            list. If filter is specified then we will include all items that
            match _ALL_ of filters.  If a dict entry is missing the key in a
            filter it will be excluded from the match.
            Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return
                         {'a':2, 'z': 'z'},        # True, return
                         {'a':3, 'z': 'z'},        # True, return
                         {'a':4, 'z': 'b'},        # FAILED, obj['z'] != obj['z']
                       ]
                attribute = 'a'
                filters   = {'z': 'z'}
                returns [1, 2, 3]
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a List")

        if not attribute:
            raise errors.AnsibleFilterError("|failed expects attribute to be set")

        if filters is not None:
            if not issubclass(type(filters), dict):
                raise errors.AnsibleFilterError("|failed expects filter to be a"
                                                " dict")
            retval = [FilterModule.get_attr(d, attribute) for d in data if (
                all([d.get(key, None) == filters[key] for key in filters]))]
        else:
            retval = [FilterModule.get_attr(d, attribute) for d in data]

        return retval

    @staticmethod
    def oo_select_keys_from_list(data, keys):
        ''' This returns a list, which contains the value portions for the keys
            Ex: data = { 'a':1, 'b':2, 'c':3 }
                keys = ['a', 'c']
                returns [1, 3]
        '''

        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a list")

        if not issubclass(type(keys), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        # Gather up the values for the list of keys passed in
        retval = [FilterModule.oo_select_keys(item, keys) for item in data]

        return FilterModule.oo_flatten(retval)

    @staticmethod
    def oo_select_keys(data, keys):
        ''' This returns a list, which contains the value portions for the keys
            Ex: data = { 'a':1, 'b':2, 'c':3 }
                keys = ['a', 'c']
                returns [1, 3]
        '''

        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects to filter on a dict")

        if not issubclass(type(keys), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        # Gather up the values for the list of keys passed in
        retval = [data[key] for key in keys if data.has_key(key)]

        return retval

    @staticmethod
    def oo_prepend_strings_in_list(data, prepend):
        ''' This takes a list of strings and prepends a string to each item in the
            list
            Ex: data = ['cart', 'tree']
                prepend = 'apple-'
                returns ['apple-cart', 'apple-tree']
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")
        if not all(isinstance(x, basestring) for x in data):
            raise errors.AnsibleFilterError("|failed expects first param is a list"
                                            " of strings")
        retval = [prepend + s for s in data]
        return retval

    @staticmethod
    def oo_combine_key_value(data, joiner='='):
        '''Take a list of dict in the form of { 'key': 'value'} and
           arrange them as a list of strings ['key=value']
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        rval = []
        for item in data:
            rval.append("%s%s%s" % (item['key'], joiner, item['value']))

        return rval

    @staticmethod
    def oo_combine_dict(data, in_joiner='=', out_joiner=' '):
        '''Take a dict in the form of { 'key': 'value', 'key': 'value' } and
           arrange them as a string 'key=value key=value'
        '''
        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects first param is a dict")

        return out_joiner.join([in_joiner.join([k, v]) for k, v in data.items()])

    @staticmethod
    def oo_ami_selector(data, image_name):
        ''' This takes a list of amis and an image name and attempts to return
            the latest ami.
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        if not data:
            return None
        else:
            if image_name is None or not image_name.endswith('_*'):
                ami = sorted(data, key=itemgetter('name'), reverse=True)[0]
                return ami['ami_id']
            else:
                ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data]
                ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0]
                return ami['ami_id']

    @staticmethod
    def oo_ec2_volume_definition(data, host_type, docker_ephemeral=False):
        ''' This takes a dictionary of volume definitions and returns a valid ec2
            volume definition based on the host_type and the values in the
            dictionary.
            The dictionary should look similar to this:
                { 'master':
                    { 'root':
                        { 'volume_size': 10, 'device_type': 'gp2',
                          'iops': 500
                        }
                    },
                  'node':
                    { 'root':
                        { 'volume_size': 10, 'device_type': 'io1',
                          'iops': 1000
                        },
                      'docker':
                        { 'volume_size': 40, 'device_type': 'gp2',
                          'iops': 500, 'ephemeral': 'true'
                        }
                    }
                }
        '''
        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects first param is a dict")
        if host_type not in ['master', 'node', 'etcd']:
            raise errors.AnsibleFilterError("|failed expects etcd, master or node"
                                            " as the host type")

        root_vol = data[host_type]['root']
        root_vol['device_name'] = '/dev/sda1'
        root_vol['delete_on_termination'] = True
        if root_vol['device_type'] != 'io1':
            root_vol.pop('iops', None)
        if host_type == 'node':
            docker_vol = data[host_type]['docker']
            docker_vol['device_name'] = '/dev/xvdb'
            docker_vol['delete_on_termination'] = True
            if docker_vol['device_type'] != 'io1':
                docker_vol.pop('iops', None)
            if docker_ephemeral:
                docker_vol.pop('device_type', None)
                docker_vol.pop('delete_on_termination', None)
                docker_vol['ephemeral'] = 'ephemeral0'
            return [root_vol, docker_vol]
        elif host_type == 'etcd':
            etcd_vol = data[host_type]['etcd']
            etcd_vol['device_name'] = '/dev/xvdb'
            etcd_vol['delete_on_termination'] = True
            if etcd_vol['device_type'] != 'io1':
                etcd_vol.pop('iops', None)
            return [root_vol, etcd_vol]
        return [root_vol]

    @staticmethod
    def oo_split(string, separator=','):
        ''' This splits the input string into a list
        '''
        return string.split(separator)

    @staticmethod
    def oo_haproxy_backend_masters(hosts):
        ''' This takes an array of dicts and returns an array of dicts
            to be used as a backend for the haproxy role
        '''
        servers = []
        for idx, host_info in enumerate(hosts):
            server = dict(name="master%s" % idx)
            server_ip = host_info['openshift']['common']['ip']
            server_port = host_info['openshift']['master']['api_port']
            server['address'] = "%s:%s" % (server_ip, server_port)
            server['opts'] = 'check'
            servers.append(server)
        return servers

    @staticmethod
    def oo_filter_list(data, filter_attr=None):
        ''' This returns a list, which contains all items where filter_attr
            evaluates to true
            Ex: data = [ { a: 1, b: True },
                         { a: 3, b: False },
                         { a: 5, b: True } ]
                filter_attr = 'b'
                returns [ { a: 1, b: True },
                          { a: 5, b: True } ]
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a list")

        if not issubclass(type(filter_attr), str):
            raise errors.AnsibleFilterError("|failed expects filter_attr is a str")

        # Gather up the values for the list of keys passed in
        return [x for x in data if x[filter_attr]]

    @staticmethod
    def oo_parse_heat_stack_outputs(data):
        ''' Formats the HEAT stack output into a usable form

            The goal is to transform something like this:

            +---------------+-------------------------------------------------+
            | Property      | Value                                           |
            +---------------+-------------------------------------------------+
            | capabilities  | [] |                                            |
            | creation_time | 2015-06-26T12:26:26Z |                          |
            | description   | OpenShift cluster |                             |
            | …             | …                                               |
            | outputs       | [                                               |
            |               |   {                                             |
            |               |     "output_value": "value_A"                   |
            |               |     "description": "This is the value of Key_A" |
            |               |     "output_key": "Key_A"                       |
            |               |   },                                            |
            |               |   {                                             |
            |               |     "output_value": [                           |
            |               |       "value_B1",                               |
            |               |       "value_B2"                                |
            |               |     ],                                          |
            |               |     "description": "This is the value of Key_B" |
            |               |     "output_key": "Key_B"                       |
            |               |   },                                            |
            |               | ]                                               |
            | parameters    | {                                               |
            | …             | …                                               |
            +---------------+-------------------------------------------------+

            into something like this:

            {
              "Key_A": "value_A",
              "Key_B": [
                "value_B1",
                "value_B2"
              ]
            }
        '''

        # Extract the “outputs” JSON snippet from the pretty-printed array
        in_outputs = False
        outputs = ''

        line_regex = re.compile(r'\|\s*(.*?)\s*\|\s*(.*?)\s*\|')
        for line in data['stdout_lines']:
            match = line_regex.match(line)
            if match:
                if match.group(1) == 'outputs':
                    in_outputs = True
                elif match.group(1) != '':
                    in_outputs = False
                if in_outputs:
                    outputs += match.group(2)

        outputs = json.loads(outputs)

        # Revamp the “outputs” to put it in the form of a “Key: value” map
        revamped_outputs = {}
        for output in outputs:
            revamped_outputs[output['output_key']] = output['output_value']

        return revamped_outputs

    @staticmethod
    # pylint: disable=too-many-branches
    def oo_parse_certificate_names(certificates, data_dir, internal_hostnames):
        ''' Parses names from list of certificate hashes.

            Ex: certificates = [{ "certfile": "/etc/origin/master/custom1.crt",
                                  "keyfile": "/etc/origin/master/custom1.key" },
                                { "certfile": "custom2.crt",
                                  "keyfile": "custom2.key" }]

                returns [{ "certfile": "/etc/origin/master/custom1.crt",
                           "keyfile": "/etc/origin/master/custom1.key",
                           "names": [ "public-master-host.com",
                                      "other-master-host.com" ] },
                         { "certfile": "/etc/origin/master/custom2.crt",
                           "keyfile": "/etc/origin/master/custom2.key",
                           "names": [ "some-hostname.com" ] }]
        '''
        if not issubclass(type(certificates), list):
            raise errors.AnsibleFilterError("|failed expects certificates is a list")

        if not issubclass(type(data_dir), unicode):
            raise errors.AnsibleFilterError("|failed expects data_dir is unicode")

        if not issubclass(type(internal_hostnames), list):
            raise errors.AnsibleFilterError("|failed expects internal_hostnames is list")

        for certificate in certificates:
            if 'names' in certificate.keys():
                continue
            else:
                certificate['names'] = []

            if not os.path.isfile(certificate['certfile']) and not os.path.isfile(certificate['keyfile']):
                raise errors.AnsibleFilterError("|certificate and/or key does not exist '%s', '%s'" %
                                                (certificate['certfile'], certificate['keyfile']))

            try:
                st_cert = open(certificate['certfile'], 'rt').read()
                cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, st_cert)
                certificate['names'].append(str(cert.get_subject().commonName.decode()))
                for i in range(cert.get_extension_count()):
                    if cert.get_extension(i).get_short_name() == 'subjectAltName':
                        for name in str(cert.get_extension(i)).replace('DNS:', '').split(', '):
                            certificate['names'].append(name)
            except:
                raise errors.AnsibleFilterError(("|failed to parse certificate '%s', " % certificate['certfile'] +
                                                 "please specify certificate names in host inventory"))

            certificate['names'] = [name for name in certificate['names'] if name not in internal_hostnames]
            certificate['names'] = list(set(certificate['names']))
            if not certificate['names']:
                raise errors.AnsibleFilterError(("|failed to parse certificate '%s' or " % certificate['certfile'] +
                                                 "detected a collision with internal hostname, please specify " +
                                                 "certificate names in host inventory"))
        return certificates

    def filters(self):
        ''' returns a mapping of filters to methods '''
        return {
            "oo_select_keys": self.oo_select_keys,
            "oo_select_keys_from_list": self.oo_select_keys_from_list,
            "oo_collect": self.oo_collect,
            "oo_flatten": self.oo_flatten,
            "oo_pdb": self.oo_pdb,
            "oo_prepend_strings_in_list": self.oo_prepend_strings_in_list,
            "oo_ami_selector": self.oo_ami_selector,
            "oo_ec2_volume_definition": self.oo_ec2_volume_definition,
            "oo_combine_key_value": self.oo_combine_key_value,
            "oo_combine_dict": self.oo_combine_dict,
            "oo_split": self.oo_split,
            "oo_filter_list": self.oo_filter_list,
            "oo_parse_heat_stack_outputs": self.oo_parse_heat_stack_outputs,
            "oo_parse_certificate_names": self.oo_parse_certificate_names,
            "oo_haproxy_backend_masters": self.oo_haproxy_backend_masters
        }
<commit_msg>Fix file check conditional.<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
# vim: expandtab:tabstop=4:shiftwidth=4
'''
Custom filters for use in openshift-ansible
'''

from ansible import errors
from operator import itemgetter
import OpenSSL.crypto
import os.path
import pdb
import re
import json

class FilterModule(object):
    ''' Custom ansible filters '''

    @staticmethod
    def oo_pdb(arg):
        ''' This pops you into a pdb instance where arg is the data passed in
            from the filter.
            Ex: "{{ hostvars | oo_pdb }}"
        '''
        pdb.set_trace()
        return arg

    @staticmethod
    def get_attr(data, attribute=None):
        ''' This looks up dictionary attributes of the form a.b.c and returns
            the value.
            Ex: data = {'a': {'b': {'c': 5}}}
                attribute = "a.b.c"
                returns 5
        '''
        if not attribute:
            raise errors.AnsibleFilterError("|failed expects attribute to be set")

        ptr = data
        for attr in attribute.split('.'):
            ptr = ptr[attr]

        return ptr

    @staticmethod
    def oo_flatten(data):
        ''' This filter plugin will flatten a list of lists
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to flatten a List")

        return [item for sublist in data for item in sublist]

    @staticmethod
    def oo_collect(data, attribute=None, filters=None):
        ''' This takes a list of dict and collects all attributes specified into a
            list. If filter is specified then we will include all items that
            match _ALL_ of filters.  If a dict entry is missing the key in a
            filter it will be excluded from the match.
            Ex: data = [ {'a':1, 'b':5, 'z': 'z'}, # True, return
                         {'a':2, 'z': 'z'},        # True, return
                         {'a':3, 'z': 'z'},        # True, return
                         {'a':4, 'z': 'b'},        # FAILED, obj['z'] != obj['z']
                       ]
                attribute = 'a'
                filters   = {'z': 'z'}
                returns [1, 2, 3]
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a List")

        if not attribute:
            raise errors.AnsibleFilterError("|failed expects attribute to be set")

        if filters is not None:
            if not issubclass(type(filters), dict):
                raise errors.AnsibleFilterError("|failed expects filter to be a"
                                                " dict")
            retval = [FilterModule.get_attr(d, attribute) for d in data if (
                all([d.get(key, None) == filters[key] for key in filters]))]
        else:
            retval = [FilterModule.get_attr(d, attribute) for d in data]

        return retval

    @staticmethod
    def oo_select_keys_from_list(data, keys):
        ''' This returns a list, which contains the value portions for the keys
            Ex: data = { 'a':1, 'b':2, 'c':3 }
                keys = ['a', 'c']
                returns [1, 3]
        '''

        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a list")

        if not issubclass(type(keys), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        # Gather up the values for the list of keys passed in
        retval = [FilterModule.oo_select_keys(item, keys) for item in data]

        return FilterModule.oo_flatten(retval)

    @staticmethod
    def oo_select_keys(data, keys):
        ''' This returns a list, which contains the value portions for the keys
            Ex: data = { 'a':1, 'b':2, 'c':3 }
                keys = ['a', 'c']
                returns [1, 3]
        '''

        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects to filter on a dict")

        if not issubclass(type(keys), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        # Gather up the values for the list of keys passed in
        retval = [data[key] for key in keys if data.has_key(key)]

        return retval

    @staticmethod
    def oo_prepend_strings_in_list(data, prepend):
        ''' This takes a list of strings and prepends a string to each item in the
            list
            Ex: data = ['cart', 'tree']
                prepend = 'apple-'
                returns ['apple-cart', 'apple-tree']
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")
        if not all(isinstance(x, basestring) for x in data):
            raise errors.AnsibleFilterError("|failed expects first param is a list"
                                            " of strings")
        retval = [prepend + s for s in data]
        return retval

    @staticmethod
    def oo_combine_key_value(data, joiner='='):
        '''Take a list of dict in the form of { 'key': 'value'} and
           arrange them as a list of strings ['key=value']
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        rval = []
        for item in data:
            rval.append("%s%s%s" % (item['key'], joiner, item['value']))

        return rval

    @staticmethod
    def oo_combine_dict(data, in_joiner='=', out_joiner=' '):
        '''Take a dict in the form of { 'key': 'value', 'key': 'value' } and
           arrange them as a string 'key=value key=value'
        '''
        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects first param is a dict")

        return out_joiner.join([in_joiner.join([k, v]) for k, v in data.items()])

    @staticmethod
    def oo_ami_selector(data, image_name):
        ''' This takes a list of amis and an image name and attempts to return
            the latest ami.
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects first param is a list")

        if not data:
            return None
        else:
            if image_name is None or not image_name.endswith('_*'):
                ami = sorted(data, key=itemgetter('name'), reverse=True)[0]
                return ami['ami_id']
            else:
                ami_info = [(ami, ami['name'].split('_')[-1]) for ami in data]
                ami = sorted(ami_info, key=itemgetter(1), reverse=True)[0][0]
                return ami['ami_id']

    @staticmethod
    def oo_ec2_volume_definition(data, host_type, docker_ephemeral=False):
        ''' This takes a dictionary of volume definitions and returns a valid ec2
            volume definition based on the host_type and the values in the
            dictionary.
            The dictionary should look similar to this:
                { 'master':
                    { 'root':
                        { 'volume_size': 10, 'device_type': 'gp2',
                          'iops': 500
                        }
                    },
                  'node':
                    { 'root':
                        { 'volume_size': 10, 'device_type': 'io1',
                          'iops': 1000
                        },
                      'docker':
                        { 'volume_size': 40, 'device_type': 'gp2',
                          'iops': 500, 'ephemeral': 'true'
                        }
                    }
                }
        '''
        if not issubclass(type(data), dict):
            raise errors.AnsibleFilterError("|failed expects first param is a dict")
        if host_type not in ['master', 'node', 'etcd']:
            raise errors.AnsibleFilterError("|failed expects etcd, master or node"
                                            " as the host type")

        root_vol = data[host_type]['root']
        root_vol['device_name'] = '/dev/sda1'
        root_vol['delete_on_termination'] = True
        if root_vol['device_type'] != 'io1':
            root_vol.pop('iops', None)
        if host_type == 'node':
            docker_vol = data[host_type]['docker']
            docker_vol['device_name'] = '/dev/xvdb'
            docker_vol['delete_on_termination'] = True
            if docker_vol['device_type'] != 'io1':
                docker_vol.pop('iops', None)
            if docker_ephemeral:
                docker_vol.pop('device_type', None)
                docker_vol.pop('delete_on_termination', None)
                docker_vol['ephemeral'] = 'ephemeral0'
            return [root_vol, docker_vol]
        elif host_type == 'etcd':
            etcd_vol = data[host_type]['etcd']
            etcd_vol['device_name'] = '/dev/xvdb'
            etcd_vol['delete_on_termination'] = True
            if etcd_vol['device_type'] != 'io1':
                etcd_vol.pop('iops', None)
            return [root_vol, etcd_vol]
        return [root_vol]

    @staticmethod
    def oo_split(string, separator=','):
        ''' This splits the input string into a list
        '''
        return string.split(separator)

    @staticmethod
    def oo_haproxy_backend_masters(hosts):
        ''' This takes an array of dicts and returns an array of dicts
            to be used as a backend for the haproxy role
        '''
        servers = []
        for idx, host_info in enumerate(hosts):
            server = dict(name="master%s" % idx)
            server_ip = host_info['openshift']['common']['ip']
            server_port = host_info['openshift']['master']['api_port']
            server['address'] = "%s:%s" % (server_ip, server_port)
            server['opts'] = 'check'
            servers.append(server)
        return servers

    @staticmethod
    def oo_filter_list(data, filter_attr=None):
        ''' This returns a list, which contains all items where filter_attr
            evaluates to true
            Ex: data = [ { a: 1, b: True },
                         { a: 3, b: False },
                         { a: 5, b: True } ]
                filter_attr = 'b'
                returns [ { a: 1, b: True },
                          { a: 5, b: True } ]
        '''
        if not issubclass(type(data), list):
            raise errors.AnsibleFilterError("|failed expects to filter on a list")

        if not issubclass(type(filter_attr), str):
            raise errors.AnsibleFilterError("|failed expects filter_attr is a str")

        # Gather up the values for the list of keys passed in
        return [x for x in data if x[filter_attr]]

    @staticmethod
    def oo_parse_heat_stack_outputs(data):
        ''' Formats the HEAT stack output into a usable form

            The goal is to transform something like this:

            +---------------+-------------------------------------------------+
            | Property      | Value                                           |
            +---------------+-------------------------------------------------+
            | capabilities  | [] |                                            |
            | creation_time | 2015-06-26T12:26:26Z |                          |
            | description   | OpenShift cluster |                             |
            | …             | …                                               |
            | outputs       | [                                               |
            |               |   {                                             |
            |               |     "output_value": "value_A"                   |
            |               |     "description": "This is the value of Key_A" |
            |               |     "output_key": "Key_A"                       |
            |               |   },                                            |
            |               |   {                                             |
            |               |     "output_value": [                           |
            |               |       "value_B1",                               |
            |               |       "value_B2"                                |
            |               |     ],                                          |
            |               |     "description": "This is the value of Key_B" |
            |               |     "output_key": "Key_B"                       |
            |               |   },                                            |
            |               | ]                                               |
            | parameters    | {                                               |
            | …             | …                                               |
            +---------------+-------------------------------------------------+

            into something like this:

            {
              "Key_A": "value_A",
              "Key_B": [
                "value_B1",
                "value_B2"
              ]
            }
        '''

        # Extract the “outputs” JSON snippet from the pretty-printed array
        in_outputs = False
        outputs = ''

        line_regex = re.compile(r'\|\s*(.*?)\s*\|\s*(.*?)\s*\|')
        for line in data['stdout_lines']:
            match = line_regex.match(line)
            if match:
                if match.group(1) == 'outputs':
                    in_outputs = True
                elif match.group(1) != '':
                    in_outputs = False
                if in_outputs:
                    outputs += match.group(2)

        outputs = json.loads(outputs)

        # Revamp the “outputs” to put it in the form of a “Key: value” map
        revamped_outputs = {}
        for output in outputs:
            revamped_outputs[output['output_key']] = output['output_value']

        return revamped_outputs

    @staticmethod
    # pylint: disable=too-many-branches
    def oo_parse_certificate_names(certificates, data_dir, internal_hostnames):
        ''' Parses names from list of certificate hashes.

            Ex: certificates = [{ "certfile": "/etc/origin/master/custom1.crt",
                                  "keyfile": "/etc/origin/master/custom1.key" },
                                { "certfile": "custom2.crt",
                                  "keyfile": "custom2.key" }]

                returns [{ "certfile": "/etc/origin/master/custom1.crt",
                           "keyfile": "/etc/origin/master/custom1.key",
                           "names": [ "public-master-host.com",
                                      "other-master-host.com" ] },
                         { "certfile": "/etc/origin/master/custom2.crt",
                           "keyfile": "/etc/origin/master/custom2.key",
                           "names": [ "some-hostname.com" ] }]
        '''
        if not issubclass(type(certificates), list):
            raise errors.AnsibleFilterError("|failed expects certificates is a list")

        if not issubclass(type(data_dir), unicode):
            raise errors.AnsibleFilterError("|failed expects data_dir is unicode")

        if not issubclass(type(internal_hostnames), list):
            raise errors.AnsibleFilterError("|failed expects internal_hostnames is list")

        for certificate in certificates:
            if 'names' in certificate.keys():
                continue
            else:
                certificate['names'] = []

            if not os.path.isfile(certificate['certfile']) or not os.path.isfile(certificate['keyfile']):
                raise errors.AnsibleFilterError("|certificate and/or key does not exist '%s', '%s'" %
                                                (certificate['certfile'], certificate['keyfile']))

            try:
                st_cert = open(certificate['certfile'], 'rt').read()
                cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, st_cert)
                certificate['names'].append(str(cert.get_subject().commonName.decode()))
                for i in range(cert.get_extension_count()):
                    if cert.get_extension(i).get_short_name() == 'subjectAltName':
                        for name in str(cert.get_extension(i)).replace('DNS:', '').split(', '):
                            certificate['names'].append(name)
            except:
                raise errors.AnsibleFilterError(("|failed to parse certificate '%s', " % certificate['certfile'] +
                                                 "please specify certificate names in host inventory"))

            certificate['names'] = [name for name in certificate['names'] if name not in internal_hostnames]
            certificate['names'] = list(set(certificate['names']))
            if not certificate['names']:
                raise errors.AnsibleFilterError(("|failed to parse certificate '%s' or " % certificate['certfile'] +
                                                 "detected a collision with internal hostname, please specify " +
                                                 "certificate names in host inventory"))
        return certificates

    def filters(self):
        ''' returns a mapping of filters to methods '''
        return {
            "oo_select_keys": self.oo_select_keys,
            "oo_select_keys_from_list": self.oo_select_keys_from_list,
            "oo_collect": self.oo_collect,
            "oo_flatten": self.oo_flatten,
            "oo_pdb": self.oo_pdb,
            "oo_prepend_strings_in_list": self.oo_prepend_strings_in_list,
            "oo_ami_selector": self.oo_ami_selector,
            "oo_ec2_volume_definition": self.oo_ec2_volume_definition,
            "oo_combine_key_value": self.oo_combine_key_value,
            "oo_combine_dict": self.oo_combine_dict,
            "oo_split": self.oo_split,
            "oo_filter_list": self.oo_filter_list,
            "oo_parse_heat_stack_outputs": self.oo_parse_heat_stack_outputs,
            "oo_parse_certificate_names": self.oo_parse_certificate_names,
            "oo_haproxy_backend_masters": self.oo_haproxy_backend_masters
        }
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
import argparse
import logging
import os
import pprint
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile
import matplotlib.pyplot as plt

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper

def main(args):
  # FIXME: Clean me up!
  #resultTypes = [ r.name for r in list(FinalResultType)] # Get list of ResultTypes as strings
  #defaultTypes = [ r.name for r in list(FinalResultType) if r in [FinalResultType.FULLY_EXPLORED, FinalResultType.BUG_FOUND]]

  #failureTypes = { FinalResultType.UNKNOWN, FinalResultType.TIMED_OUT, FinalResultType.OUT_OF_MEMORY }

  usefulAnswerResultTypes = { FinalResultType.FULLY_EXPLORED, FinalResultType.BUG_FOUND}

  parser = argparse.ArgumentParser()
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument("-v", "--verbose", action='store_true', help='Show detailed information about mismatch')
  parser.add_argument('label_mapping', type=argparse.FileType('r'), help='correctness mapping YAML file')
  parser.add_argument('result_ymls', nargs=2, help='Input YAML files')
  parser.add_argument('max_time', type=int, help='Maximum time in seconds, results timings will be clamped to this value')
  parser.add_argument('--ipython', action='store_true')
  parser.add_argument('--point-size', type=float, default=30.0, dest='point_size')
  parser.add_argument('--click-shows-raw-data', dest='click_shows_raw_data', action='store_true', default=False)
  parser.add_argument('--only-show', dest='only_show', default='incorrect', choices=['correct', 'incorrect', 'unknown'])
  parser.add_argument('-c', '--only-allow-consistent', dest='only_allow_consistent',action='store_true', default=False)
  parser.add_argument('--axis-name-map',dest='axis_name_map', default=None, type=str)

  #group = parser.add_mutually_exclusive_group()
  #group.add_argument('-r', '--result-types-to-plot', nargs='+', dest='result_types_to_plot',
  #  choices=resultTypes, default=defaultTypes,
  #  help='Result types to plot (at least one of the pair must be of this type). Default: %(default)s')
  #group.add_argument('--all', default=False, action='store_true', help='Plot all result types')

  pargs = parser.parse_args(args)
  print(pargs)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  if len(pargs.result_ymls) != 2:
    logger.error('Need two YAML files')

  # Load mapping
  correctnessMapping = yaml.load(pargs.label_mapping, Loader=Loader)
  validateMappingFile(correctnessMapping)

  axisMapping = None
  # Load the legend mapping if it exists
  if pargs.axis_name_map != None:
    if not os.path.exists(pargs.axis_name_map):
      logging.error('"{}" does not exist'.format(pargs.axis_name_map))
      return 1
    with open(pargs.axis_name_map, 'r') as openFile:
      axisMapping = yaml.load(openFile, Loader=Loader)
      if not isinstance(axisMapping, dict):
        logging.error('axis mapping should be a dictionary mapping file paths to axis name')
        return 1
  ## Create set of allowed result types
  #if pargs.all:
  #  allowedResultTypes = set(FinalResultType)
  #else:
  #  allowedResultTypes = set()
  #  for rType in pargs.result_types_to_plot:
  #    allowedResultTypes.add(FinalResultType[rType])
  #logging.info('Plotting points of type {}'.format(allowedResultTypes))

  # Check that each yml file exists
  # resultListName (i.e. filename) -> list of results for that file
  data = { }
  resultListNames = [ ]
  for f in pargs.result_ymls:
    if not os.path.exists(f):
      logging.error('YAML file {} does not exist'.format(f))
      return 1

    # Compute result set name
    resultListName = f
    resultListNames.append(resultListName)
    if resultListName in data:
      logging.error('Can\'t use {} as label name because it is already used'.format(resultListName))
      return 1

    data[resultListName] = None # Will be filled with loaded YAML data

  # Now load YAML
  length = 0
  for f in pargs.result_ymls:
    logging.info('Loading YAML file {}'.format(f))
    with open(f, 'r') as openFile:
      results = yaml.load(openFile, Loader=Loader)
    logging.info('Loading complete')
    assert isinstance(results, list)
    resultListName = f
    data[resultListName] = results
    length = len(results)

  # Check the lengths are the same
  for name, rList in data.items():
    if len(rList) != length:
      logging.error('There is a length mismatch for {}, expected {} entries but was'.format(name, length, len(rList)))
      return 1

  programToResultSetsMap = { }
  for resultListName in resultListNames:
    for r in data[resultListName]:
      programName = r['program']

      # Only add programs that we want
      if not programName in correctnessMapping:
        logging.error('{} is missing from correctness mapping'.format(programName))
        return 1
      expectedCorrect = correctnessMapping[programName]['expected_correct']
      assert expectedCorrect == None or isinstance(expectedCorrect, bool)

      skipBenchmark = False
      if pargs.only_show == 'incorrect':
        if expectedCorrect != False:
          skipBenchmark = True
      elif pargs.only_show == 'correct':
        if expectedCorrect != True:
          skipBenchmark = True
      elif pargs.only_show == 'unknown':
        if expectedCorrect != None:
          skipBenchmark = True
      else:
        assert False

      if skipBenchmark:
        logging.info('Filtering out {} ({}) because we are only showing benchmarks labelled as {}'.format(programName, expectedCorrect, pargs.only_show))
        continue
      try:
        existingDict = programToResultSetsMap[programName]
        existingDict[resultListName] = r
      except KeyError:
        programToResultSetsMap[programName] = { resultListName:r }

  # Check there are the same number of results for each program
  allowConsistentMismatchCount = 0
  disregardedResultTypeCount = 0
  clampCount = 0
  failedAnalysisGivenMaxTimeCount = 0
  xData = [ ]
  yData = [ ]
  annotationLabels = [ ]
  for programName, resultListNameToRawResultMap  in programToResultSetsMap.items():
    if len(resultListNameToRawResultMap) != len(resultListNames):
      logging.error('For program {} there we only {} result lists but expected {}'.format(
        programName, len(resultListNameToRawResultMap), len(resultListNames)))
      logging.error(pprint.pformat(resultListNameToRawResultMap))
      return 1

    firstResult = resultListNameToRawResultMap[resultListNames[0]]
    secondResult = resultListNameToRawResultMap[resultListNames[1]]
    firstType = classifyResult(firstResult)
    secondType = classifyResult(secondResult)

    if pargs.only_allow_consistent:
      # For this program check that classifications are consistent
      # we take the first programList name as the expected
      if firstType != secondType:
        allowConsistentMismatchCount += 1
        logging.warning('Found mismatch for program {}:'.format(programName))
        for resultListName in resultListNames:
          logging.warning('{}: {}'.format(resultListName, classifyResult(resultListNameToRawResultMap[resultListName])))
        if pargs.verbose:
          logging.warning('\n{}'.format(pprint.pformat(resultListNameToRawResultMap)))
        logging.warning('Disregarding result\n')
        continue

    #if not firstType in allowedResultTypes and not secondType in allowedResultTypes:
    #  disregardedResultTypeCount += 1
    #  logging.warning('Disregarding {} in {} due to neither result types ({} and {}) not being one of the allow result types'.format(
    #    programName,
    #    resultListNames[0],
    #    firstType,
    #    secondType))
    #  continue

    # Clamp timings
    didClamp = False
    for resultListName in resultListNames:
      r = resultListNameToRawResultMap[resultListName]
      if r['total_time'] > pargs.max_time:
        logging.debug('Clamping {} for {}'.format(programName, resultListName))
        r['total_time'] = pargs.max_time
        didClamp = True
      if not classifyResult(r) in usefulAnswerResultTypes:
        logging.info('Clamping time for program {} for {} with result type {}'.format(
          r['program'], resultListName, classifyResult(r)))
        r['total_time'] = pargs.max_time
        didClamp = True

    if didClamp:
      clampCount += 1

    # Add data point for plotting
    xData.append(firstResult['total_time'])
    yData.append(secondResult['total_time'])
    annotationLabels.append(programName)

  # Finally do plotting
  assert len(xData) == len(yData) == len(annotationLabels)
  extend = 100
  tickFreq = 100
  if pargs.only_allow_consistent:
    logging.info('# of mismatches when only allowing consistent results: {}'.format(allowConsistentMismatchCount))
  logging.info('# of result pairs clamped: {}'.format(clampCount))
  logging.info('# of points plotted: {} out of {}'.format(len(xData), len(programToResultSetsMap)))
  fig, ax = plt.subplots()
  splot = ax.scatter(xData, yData, picker=5, s=pargs.point_size)

  xAxisLabel=""
  yAxisLabel=""
  if pargs.axis_name_map:
    xAxisLabel = axisMapping[resultListNames[0]]
    yAxisLabel = axisMapping[resultListNames[1]]
  else:
    xAxisLabel=resultListNames[0]
    yAxisLabel=resultListNames[1]
  xAxisLabel += " execution time (s)"
  yAxisLabel += " execution time (s)"
  ax.set_xlabel(xAxisLabel)
  ax.set_xlim(0,pargs.max_time + extend)
  # +1 is just so the pargs.max_time is included because range()'s end is not inclusive
  ax.set_xticks(range(0, pargs.max_time + 1, tickFreq))
  ax.set_ylabel(yAxisLabel)
  ax.set_ylim(0,pargs.max_time + extend)
  ax.set_yticks(range(0, pargs.max_time + 1, tickFreq))
  ax.grid(False)

  # Add annotations that become visible when clicked
  DataPointReporter(splot, xData, yData, annotationLabels, programToResultSetsMap, pargs.click_shows_raw_data)

  # Identity line
  ax.plot([ 0 , pargs.max_time + extend], [0, pargs.max_time + extend], linewidth=1.0, color='black')
  if pargs.ipython:
    from IPython import embed
    embed()
    # Call fig.show() to see the figure
  else:
    plt.show()

  return 0

class DataPointReporter:
  def __init__(self, scatter, xData, yData, annotationLabels, programToResultSetsMap, clickShowsRawData):
    self.scatter = scatter
    self.cid = scatter.figure.canvas.mpl_connect('pick_event', self)
    self.annotationLabels = annotationLabels
    self.programToResultSetsMap = programToResultSetsMap
    self.clickShowsRawData = clickShowsRawData
    # Add annotations, by hide them by default
    self.annotationObjects = [ ]
    self.lastClickedAnnotationObj = None
    for index, text in enumerate(annotationLabels):
      text = text.replace('/','/\n')
      annotation = scatter.axes.annotate(text, (xData[index], yData[index]))
      annotation.set_visible(False)
      annotation.set_horizontalalignment('center')
      self.annotationObjects.append(annotation)

  def __call__(self, event):
    programName = self.annotationLabels[event.ind[0]]
    logging.info('*****')
    logging.info('{}'.format(programName))
    for resultListName, rawResultData in self.programToResultSetsMap[programName].items():
      if self.clickShowsRawData:
        logging.info('{}: {}\n{}'.format(resultListName, classifyResult(rawResultData), pprint.pformat(rawResultData)))
      else:
        logging.info('{}: {} ({} ± {} secs)'.format(
          resultListName,
          classifyResult(rawResultData),
          rawResultData['total_time'],
          rawResultData['total_time_stddev'] if 'total_time_stddev' in rawResultData else 'UNKNOWN'
          ))
    logging.info('*****')

    theAnnotation = self.annotationObjects[event.ind[0]]
    if self.lastClickedAnnotationObj != None:
      self.lastClickedAnnotationObj.set_visible(False)

    if self.lastClickedAnnotationObj != theAnnotation:
      theAnnotation.set_visible(True)
    else:
      # If the user click the point again we hide it
      theAnnotation.set_visible(False)

    self.lastClickedAnnotationObj = theAnnotation
    # Force redraw
    self.scatter.figure.canvas.draw()

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<commit_msg>Teach plot_scatter_pair_by_label.py to print how many times x>y, y>x, x==y without timeout and how many dual timeouts.<commit_after>#!/usr/bin/env python
# vim: set sw=2 ts=2 softtabstop=2 expandtab:
import argparse
import logging
import os
import pprint
import sys
import yaml
from br_util import FinalResultType, classifyResult, validateMappingFile
import matplotlib.pyplot as plt

try:
  # Try to use libyaml which is faster
  from yaml import CLoader as Loader, CDumper as Dumper
except ImportError:
  # fall back on python implementation
  from yaml import Loader, Dumper

def main(args):
  # FIXME: Clean me up!
  #resultTypes = [ r.name for r in list(FinalResultType)] # Get list of ResultTypes as strings
  #defaultTypes = [ r.name for r in list(FinalResultType) if r in [FinalResultType.FULLY_EXPLORED, FinalResultType.BUG_FOUND]]

  #failureTypes = { FinalResultType.UNKNOWN, FinalResultType.TIMED_OUT, FinalResultType.OUT_OF_MEMORY }

  usefulAnswerResultTypes = { FinalResultType.FULLY_EXPLORED, FinalResultType.BUG_FOUND}

  parser = argparse.ArgumentParser()
  parser.add_argument("-l","--log-level",type=str, default="info", dest="log_level", choices=['debug','info','warning','error'])
  parser.add_argument("-v", "--verbose", action='store_true', help='Show detailed information about mismatch')
  parser.add_argument('label_mapping', type=argparse.FileType('r'), help='correctness mapping YAML file')
  parser.add_argument('result_ymls', nargs=2, help='Input YAML files')
  parser.add_argument('max_time', type=int, help='Maximum time in seconds, results timings will be clamped to this value')
  parser.add_argument('--ipython', action='store_true')
  parser.add_argument('--point-size', type=float, default=30.0, dest='point_size')
  parser.add_argument('--click-shows-raw-data', dest='click_shows_raw_data', action='store_true', default=False)
  parser.add_argument('--only-show', dest='only_show', default='incorrect', choices=['correct', 'incorrect', 'unknown'])
  parser.add_argument('-c', '--only-allow-consistent', dest='only_allow_consistent',action='store_true', default=False)
  parser.add_argument('--axis-name-map',dest='axis_name_map', default=None, type=str)

  #group = parser.add_mutually_exclusive_group()
  #group.add_argument('-r', '--result-types-to-plot', nargs='+', dest='result_types_to_plot',
  #  choices=resultTypes, default=defaultTypes,
  #  help='Result types to plot (at least one of the pair must be of this type). Default: %(default)s')
  #group.add_argument('--all', default=False, action='store_true', help='Plot all result types')

  pargs = parser.parse_args(args)
  print(pargs)

  logLevel = getattr(logging, pargs.log_level.upper(),None)
  logging.basicConfig(level=logLevel)

  if len(pargs.result_ymls) != 2:
    logger.error('Need two YAML files')

  # Load mapping
  correctnessMapping = yaml.load(pargs.label_mapping, Loader=Loader)
  validateMappingFile(correctnessMapping)

  axisMapping = None
  # Load the legend mapping if it exists
  if pargs.axis_name_map != None:
    if not os.path.exists(pargs.axis_name_map):
      logging.error('"{}" does not exist'.format(pargs.axis_name_map))
      return 1
    with open(pargs.axis_name_map, 'r') as openFile:
      axisMapping = yaml.load(openFile, Loader=Loader)
      if not isinstance(axisMapping, dict):
        logging.error('axis mapping should be a dictionary mapping file paths to axis name')
        return 1
  ## Create set of allowed result types
  #if pargs.all:
  #  allowedResultTypes = set(FinalResultType)
  #else:
  #  allowedResultTypes = set()
  #  for rType in pargs.result_types_to_plot:
  #    allowedResultTypes.add(FinalResultType[rType])
  #logging.info('Plotting points of type {}'.format(allowedResultTypes))

  # Check that each yml file exists
  # resultListName (i.e. filename) -> list of results for that file
  data = { }
  resultListNames = [ ]
  for f in pargs.result_ymls:
    if not os.path.exists(f):
      logging.error('YAML file {} does not exist'.format(f))
      return 1

    # Compute result set name
    resultListName = f
    resultListNames.append(resultListName)
    if resultListName in data:
      logging.error('Can\'t use {} as label name because it is already used'.format(resultListName))
      return 1

    data[resultListName] = None # Will be filled with loaded YAML data

  # Now load YAML
  length = 0
  for f in pargs.result_ymls:
    logging.info('Loading YAML file {}'.format(f))
    with open(f, 'r') as openFile:
      results = yaml.load(openFile, Loader=Loader)
    logging.info('Loading complete')
    assert isinstance(results, list)
    resultListName = f
    data[resultListName] = results
    length = len(results)

  # Check the lengths are the same
  for name, rList in data.items():
    if len(rList) != length:
      logging.error('There is a length mismatch for {}, expected {} entries but was'.format(name, length, len(rList)))
      return 1

  programToResultSetsMap = { }
  for resultListName in resultListNames:
    for r in data[resultListName]:
      programName = r['program']

      # Only add programs that we want
      if not programName in correctnessMapping:
        logging.error('{} is missing from correctness mapping'.format(programName))
        return 1
      expectedCorrect = correctnessMapping[programName]['expected_correct']
      assert expectedCorrect == None or isinstance(expectedCorrect, bool)

      skipBenchmark = False
      if pargs.only_show == 'incorrect':
        if expectedCorrect != False:
          skipBenchmark = True
      elif pargs.only_show == 'correct':
        if expectedCorrect != True:
          skipBenchmark = True
      elif pargs.only_show == 'unknown':
        if expectedCorrect != None:
          skipBenchmark = True
      else:
        assert False

      if skipBenchmark:
        logging.info('Filtering out {} ({}) because we are only showing benchmarks labelled as {}'.format(programName, expectedCorrect, pargs.only_show))
        continue
      try:
        existingDict = programToResultSetsMap[programName]
        existingDict[resultListName] = r
      except KeyError:
        programToResultSetsMap[programName] = { resultListName:r }

  # Check there are the same number of results for each program
  allowConsistentMismatchCount = 0
  disregardedResultTypeCount = 0
  clampCount = 0
  failedAnalysisGivenMaxTimeCount = 0
  xData = [ ]
  yData = [ ]
  countXLtYExceptDualTimeout=0
  countYLtXExceptDualTimeout=0
  countDualTimeout=0
  countXEqYExceptDualTimeout=0
  annotationLabels = [ ]
  for programName, resultListNameToRawResultMap  in programToResultSetsMap.items():
    if len(resultListNameToRawResultMap) != len(resultListNames):
      logging.error('For program {} there we only {} result lists but expected {}'.format(
        programName, len(resultListNameToRawResultMap), len(resultListNames)))
      logging.error(pprint.pformat(resultListNameToRawResultMap))
      return 1

    firstResult = resultListNameToRawResultMap[resultListNames[0]]
    secondResult = resultListNameToRawResultMap[resultListNames[1]]
    firstType = classifyResult(firstResult)
    secondType = classifyResult(secondResult)

    if pargs.only_allow_consistent:
      # For this program check that classifications are consistent
      # we take the first programList name as the expected
      if firstType != secondType:
        allowConsistentMismatchCount += 1
        logging.warning('Found mismatch for program {}:'.format(programName))
        for resultListName in resultListNames:
          logging.warning('{}: {}'.format(resultListName, classifyResult(resultListNameToRawResultMap[resultListName])))
        if pargs.verbose:
          logging.warning('\n{}'.format(pprint.pformat(resultListNameToRawResultMap)))
        logging.warning('Disregarding result\n')
        continue

    #if not firstType in allowedResultTypes and not secondType in allowedResultTypes:
    #  disregardedResultTypeCount += 1
    #  logging.warning('Disregarding {} in {} due to neither result types ({} and {}) not being one of the allow result types'.format(
    #    programName,
    #    resultListNames[0],
    #    firstType,
    #    secondType))
    #  continue

    # Clamp timings
    didClamp = False
    for resultListName in resultListNames:
      r = resultListNameToRawResultMap[resultListName]
      if r['total_time'] > pargs.max_time:
        logging.debug('Clamping {} for {}'.format(programName, resultListName))
        r['total_time'] = pargs.max_time
        didClamp = True
      if not classifyResult(r) in usefulAnswerResultTypes:
        logging.info('Clamping time for program {} for {} with result type {}'.format(
          r['program'], resultListName, classifyResult(r)))
        r['total_time'] = pargs.max_time
        didClamp = True

    if didClamp:
      clampCount += 1

    # Add data point for plotting
    xPoint = firstResult['total_time']
    yPoint = secondResult['total_time']

    # Update counts
    if xPoint == yPoint and xPoint == pargs.max_time:
      countDualTimeout += 1
    else:
      if xPoint < yPoint:
        countXLtYExceptDualTimeout +=1
        #print(programName)
        #print(pprint.pformat(resultListNameToRawResultMap))
        #return 1
      elif yPoint < xPoint:
        countYLtXExceptDualTimeout +=1
      else:
        assert xPoint == yPoint
        countXEqYExceptDualTimeout +=1

    xData.append(xPoint)
    yData.append(yPoint)
    annotationLabels.append(programName)

  # Finally do plotting
  assert len(xData) == len(yData) == len(annotationLabels)
  assert countXLtYExceptDualTimeout + countYLtXExceptDualTimeout + countXEqYExceptDualTimeout + countDualTimeout == len(xData)
  extend = 100
  tickFreq = 100
  xAxisLabel=""
  yAxisLabel=""
  if pargs.axis_name_map:
    xAxisLabel = axisMapping[resultListNames[0]]
    yAxisLabel = axisMapping[resultListNames[1]]
  else:
    xAxisLabel=resultListNames[0]
    yAxisLabel=resultListNames[1]

  if pargs.only_allow_consistent:
    logging.info('# of mismatches when only allowing consistent results: {}'.format(allowConsistentMismatchCount))
  logging.info('# of result pairs clamped: {}'.format(clampCount))
  logging.info('# of points plotted: {} out of {}'.format(len(xData), len(programToResultSetsMap)))
  logging.info('# of {x} < {y} (x < y): {}'.format(countXLtYExceptDualTimeout, x=xAxisLabel, y=yAxisLabel))
  logging.info('# of {y} < {x} (y < x): {}'.format(countYLtXExceptDualTimeout, x=xAxisLabel, y=yAxisLabel))
  logging.info('# of {x} == {y} (x == y) (ignoring timeouts): {}'.format(countXEqYExceptDualTimeout, x=xAxisLabel, y=yAxisLabel))
  logging.info('# of dual timeouts: {}'.format(countDualTimeout))
  fig, ax = plt.subplots()
  splot = ax.scatter(xData, yData, picker=5, s=pargs.point_size)

  xAxisLabel += " execution time (s)"
  yAxisLabel += " execution time (s)"
  ax.set_xlabel(xAxisLabel)
  ax.set_xlim(0,pargs.max_time + extend)
  # +1 is just so the pargs.max_time is included because range()'s end is not inclusive
  ax.set_xticks(range(0, pargs.max_time + 1, tickFreq))
  ax.set_ylabel(yAxisLabel)
  ax.set_ylim(0,pargs.max_time + extend)
  ax.set_yticks(range(0, pargs.max_time + 1, tickFreq))
  ax.grid(False)

  # Add annotations that become visible when clicked
  DataPointReporter(splot, xData, yData, annotationLabels, programToResultSetsMap, pargs.click_shows_raw_data)

  # Identity line
  ax.plot([ 0 , pargs.max_time + extend], [0, pargs.max_time + extend], linewidth=1.0, color='black')
  if pargs.ipython:
    from IPython import embed
    embed()
    # Call fig.show() to see the figure
  else:
    plt.show()

  return 0

class DataPointReporter:
  def __init__(self, scatter, xData, yData, annotationLabels, programToResultSetsMap, clickShowsRawData):
    self.scatter = scatter
    self.cid = scatter.figure.canvas.mpl_connect('pick_event', self)
    self.annotationLabels = annotationLabels
    self.programToResultSetsMap = programToResultSetsMap
    self.clickShowsRawData = clickShowsRawData
    # Add annotations, by hide them by default
    self.annotationObjects = [ ]
    self.lastClickedAnnotationObj = None
    for index, text in enumerate(annotationLabels):
      text = text.replace('/','/\n')
      annotation = scatter.axes.annotate(text, (xData[index], yData[index]))
      annotation.set_visible(False)
      annotation.set_horizontalalignment('center')
      self.annotationObjects.append(annotation)

  def __call__(self, event):
    programName = self.annotationLabels[event.ind[0]]
    logging.info('*****')
    logging.info('{}'.format(programName))
    for resultListName, rawResultData in self.programToResultSetsMap[programName].items():
      if self.clickShowsRawData:
        logging.info('{}: {}\n{}'.format(resultListName, classifyResult(rawResultData), pprint.pformat(rawResultData)))
      else:
        logging.info('{}: {} ({} ± {} secs)'.format(
          resultListName,
          classifyResult(rawResultData),
          rawResultData['total_time'],
          rawResultData['total_time_stddev'] if 'total_time_stddev' in rawResultData else 'UNKNOWN'
          ))
    logging.info('*****')

    theAnnotation = self.annotationObjects[event.ind[0]]
    if self.lastClickedAnnotationObj != None:
      self.lastClickedAnnotationObj.set_visible(False)

    if self.lastClickedAnnotationObj != theAnnotation:
      theAnnotation.set_visible(True)
    else:
      # If the user click the point again we hide it
      theAnnotation.set_visible(False)

    self.lastClickedAnnotationObj = theAnnotation
    # Force redraw
    self.scatter.figure.canvas.draw()

if __name__ == '__main__':
  sys.exit(main(sys.argv[1:]))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Rename a single audio file."""

import os

import ansicolor
import shutil

import phrydy
from phrydy.utils import as_string
from tmep import Functions
from tmep import Template

from .meta import Meta
import six

if six.PY2:
    import sys
    reload(sys)
    sys.setdefaultencoding('utf8')

counter = 0

class DefaultFormats(object):

    default = '$artist_initial/' + \
              '%shorten{$artistsafe_sort}/' + \
              '%shorten{$album_clean}%ifdef{year_safe,_${year_safe}}/' + \
              '${disctrack}_%shorten{$title}'

    compilation = '_compilations/' + \
                  '$album_initial/' + \
                  '%shorten{$album_clean}' + \
                  '%ifdef{year_safe,_${year_safe}}/' + \
                  '${disctrack}_%shorten{$title}'

    soundtrack = '_soundtrack/' + \
                 '$album_initial/' + \
                 '%shorten{$album_clean}' + \
                 '%ifdef{year_safe,_${year_safe}}/' + \
                 '${disctrack}_${artist}_%shorten{$title}'

    classical = '$composer_initial/$composer_safe/' + \
                '%shorten{$album_classical,48}' + \
                '_[%shorten{$performer_classical,32}]/' + \
                '${disctrack}_%shorten{$title_classical,64}_' + \
                '%shorten{$acoustid_id,8}'

class Formats(object):

    default = u''
    compilation = u''
    soundtrack = u''

    def __init__(self, args):
        defaults = DefaultFormats()

        if args.format:
            defaults.default = args.format

        if args.compilation:
            defaults.compilation = args.compilation

        if args.soundtrack:
            defaults.soundtrack = args.soundtrack

        if args.classical:
            self.default = defaults.classical
            self.compilation = defaults.classical
            self.soundtrack = defaults.classical
        else:
            self.default = defaults.default
            self.compilation = defaults.compilation
            self.soundtrack = defaults.soundtrack

class Rename(object):
    """Rename one file"""

    old_path = ''
    """The absolute path of the old file."""

    old_file = ''
    """The input path of the old file."""

    new_path = ''
    """The absolute path of the new file."""

    new_file = ''
    """The path inside the target directory of the new file."""

    extension = ''
    """The extension"""

    meta = ''
    """The meta object :class:`audiorename.meta.Meta`"""

    args = ''

    target_dir = ''
    """The target directory"""

    cwd = os.getcwd()
    """The path of the current working directory"""

    def __init__(self, old_file=False, args=False, job=False):
        self.skip = False

        if args:
            self.args = args

        if job:
            self.job = job

        if old_file:
            self.old_file = old_file

            self.old_path = os.path.realpath(self.old_file)
            self.extension = self.old_file.split('.')[-1]
            try:
                self.meta = Meta(self.old_path, self.args)

            except phrydy.mediafile.UnreadableFileError:
                self.skip = True

    def generateFilename(self):
        formats = Formats(self.args)
        if self.meta.soundtrack:
            format_string = formats.soundtrack
        elif self.meta.comp:
            format_string = formats.compilation
        else:
            format_string = formats.default

        meta_dict = self.meta.export_dict()

        t = Template(as_string(format_string))
        f = Functions(meta_dict)
        new = t.substitute(meta_dict, f.functions())
        new = self.postTemplate(new)
        new = f.tmpl_deldupchars(new + '.' + self.extension.lower())
        self.new_file = new
        self.new_path = os.path.join(self.job.target, new)

    def postTemplate(self, text):
        if isinstance(text, str) or isinstance(text, unicode):
            if self.args.shell_friendly:
                text = Functions.tmpl_asciify(text)
                text = Functions.tmpl_delchars(text, '().,!"\'’')
                text = Functions.tmpl_replchars(text, '-', ' ')
            # asciify generates new characters which must be sanitzed, e. g.:
            # ¿ -> ?
            text = Functions.tmpl_delchars(text, ':*?"<>|\~&{}')
        return text

    def createDir(self, path):
        path = os.path.dirname(path)
        import errno
        try:
            os.makedirs(path)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

    def processMessage(self, action=u'Rename', error=False, indent=12,
                       old_path=False, new_path=False, output=u'print'):
        action_processed = action + u':'
        message = action_processed.ljust(indent)
        message = u'[' + message + u']'

        if action == u'Rename' or action == u'Copy':
            message = ansicolor.yellow(message, reverse=True)
        elif action == u'Renamed':
            message = ansicolor.green(message, reverse=True)
        elif action == u'Dry run':
            message = ansicolor.white(message, reverse=True)
        elif error:
            message = ansicolor.red(message, reverse=True)
        else:
            message = ansicolor.white(message, reverse=True)

        if not old_path and hasattr(self, 'old_path'):
            output_old = self.old_path
        else:
            output_old = old_path

        if not new_path and hasattr(self, 'new_path'):
            output_new = self.new_path
        else:
            output_new = new_path

        if not self.args.verbose:
            if output_old and hasattr(self, 'cwd') and len(self.cwd) > 1:
                output_old = output_old.replace(self.cwd, '')
            if output_new and len(self.job.target) > 1:
                output_new = output_new.replace(self.job.target, '')

        line1 = message + u' ' + output_old + '\n'
        if output_new:
            line2 = u'-> '.rjust(indent + 3) + ansicolor.yellow(output_new)
        else:
            line2 = u''

        out = line1 + line2

        if output == u'print':
            print(out)
        else:
            return out

    def dryRun(self):
        self.generateFilename()
        self.processMessage(action=u'Dry run')

    def mbTrackListing(self):
        m, s = divmod(self.meta.length, 60)
        mmss = '{:d}:{:02d}'.format(int(m), int(s))
        output = '{:d}. {:s}: {:s} ({:s})'.format(counter, self.meta.album,
                                                  self.meta.title, mmss)
        output = output.replace('Op.', 'op.')
        output = output.replace('- ', '')
        print(output)

    def fetch_work(self):
        self.meta.fetch_work()
        self.meta.save()
        self.processMessage(action=u'Get work')

    def action(self, copy=False):
        """Rename or copy to new path

        :param bool copy: Copy file

        :return: None
        """
        self.generateFilename()
        if not os.path.exists(self.new_path):
            self.createDir(self.new_path)
            if copy:
                self.processMessage(action=u'Copy')
                shutil.copy2(self.old_path, self.new_path)
            else:
                self.processMessage(action=u'Rename')
                shutil.move(self.old_path, self.new_path)
        elif self.new_path == self.old_path:
            self.processMessage(action=u'Renamed', error=False)
        else:
            self.processMessage(action=u'Exists', error=True)
            if self.args.delete_existing:
                os.remove(self.old_path)
                print('Delete existing file: ' + self.old_path)

    def execute(self):
        global counter
        counter += 1
        skip = self.args.skip_if_empty
        if not self.meta:
            self.processMessage(action=u'Broken file', error=True)
        elif skip and (not hasattr(self.meta, skip) or not
                       getattr(self.meta, skip)):
            self.processMessage(action=u'No field', error=True)
        else:
            if self.job.action == u'dry_run':
                self.dryRun()
            elif self.job.action == u'mb_track_listing':
                self.mbTrackListing()
            elif self.job.action == u'copy':
                self.action(copy=True)
            elif self.args.work:
                self.fetch_work()
            else:
                self.action()

def do_rename(path, args=None, job=None):
    if args.unittest:
        print(os.path.abspath(path))
    else:
        audio = Rename(path, args, job)
        audio.execute()
<commit_msg>Add new todo<commit_after># -*- coding: utf-8 -*-

"""Rename a single audio file."""

import os

import ansicolor
import shutil

import phrydy
from phrydy.utils import as_string
from tmep import Functions
from tmep import Template

from .meta import Meta
import six

if six.PY2:
    import sys
    reload(sys)
    sys.setdefaultencoding('utf8')

counter = 0

class DefaultFormats(object):

    default = '$artist_initial/' + \
              '%shorten{$artistsafe_sort}/' + \
              '%shorten{$album_clean}%ifdef{year_safe,_${year_safe}}/' + \
              '${disctrack}_%shorten{$title}'

    compilation = '_compilations/' + \
                  '$album_initial/' + \
                  '%shorten{$album_clean}' + \
                  '%ifdef{year_safe,_${year_safe}}/' + \
                  '${disctrack}_%shorten{$title}'

    soundtrack = '_soundtrack/' + \
                 '$album_initial/' + \
                 '%shorten{$album_clean}' + \
                 '%ifdef{year_safe,_${year_safe}}/' + \
                 '${disctrack}_${artist}_%shorten{$title}'

    classical = '$composer_initial/$composer_safe/' + \
                '%shorten{$album_classical,48}' + \
                '_[%shorten{$performer_classical,32}]/' + \
                '${disctrack}_%shorten{$title_classical,64}_' + \
                '%shorten{$acoustid_id,8}'

class Formats(object):

    default = u''
    compilation = u''
    soundtrack = u''

    def __init__(self, args):
        defaults = DefaultFormats()

        if args.format:
            defaults.default = args.format

        if args.compilation:
            defaults.compilation = args.compilation

        if args.soundtrack:
            defaults.soundtrack = args.soundtrack

        if args.classical:
            self.default = defaults.classical
            self.compilation = defaults.classical
            self.soundtrack = defaults.classical
        else:
            self.default = defaults.default
            self.compilation = defaults.compilation
            self.soundtrack = defaults.soundtrack

class Rename(object):
    """Rename one file"""

    old_path = ''
    """The absolute path of the old file."""

    old_file = ''
    """The input path of the old file."""

    new_path = ''
    """The absolute path of the new file."""

    new_file = ''
    """The path inside the target directory of the new file."""

    extension = ''
    """The extension"""

    meta = ''
    """The meta object :class:`audiorename.meta.Meta`"""

    args = ''

    target_dir = ''
    """The target directory"""

    cwd = os.getcwd()
    """The path of the current working directory"""

    def __init__(self, old_file=False, args=False, job=False):
        self.skip = False

        if args:
            self.args = args

        if job:
            self.job = job

        if old_file:
            self.old_file = old_file

            self.old_path = os.path.realpath(self.old_file)
            self.extension = self.old_file.split('.')[-1]
            try:
                self.meta = Meta(self.old_path, self.args)

            except phrydy.mediafile.UnreadableFileError:
                self.skip = True

    def generateFilename(self):
        formats = Formats(self.args)
        if self.meta.soundtrack:
            format_string = formats.soundtrack
        elif self.meta.comp:
            format_string = formats.compilation
        else:
            format_string = formats.default

        meta_dict = self.meta.export_dict()

        t = Template(as_string(format_string))
        f = Functions(meta_dict)
        new = t.substitute(meta_dict, f.functions())
        new = self.postTemplate(new)
        new = f.tmpl_deldupchars(new + '.' + self.extension.lower())
        self.new_file = new
        self.new_path = os.path.join(self.job.target, new)

    def postTemplate(self, text):
        if isinstance(text, str) or isinstance(text, unicode):
            if self.args.shell_friendly:
                text = Functions.tmpl_asciify(text)
                text = Functions.tmpl_delchars(text, '().,!"\'’')
                text = Functions.tmpl_replchars(text, '-', ' ')
            # asciify generates new characters which must be sanitzed, e. g.:
            # ¿ -> ?
            text = Functions.tmpl_delchars(text, ':*?"<>|\~&{}')
        return text

    def createDir(self, path):
        path = os.path.dirname(path)
        import errno
        try:
            os.makedirs(path)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise

    def processMessage(self, action=u'Rename', error=False, indent=12,
                       old_path=False, new_path=False, output=u'print'):
        action_processed = action + u':'
        message = action_processed.ljust(indent)
        message = u'[' + message + u']'

        if action == u'Rename' or action == u'Copy':
            message = ansicolor.yellow(message, reverse=True)
        elif action == u'Renamed':
            message = ansicolor.green(message, reverse=True)
        elif action == u'Dry run':
            message = ansicolor.white(message, reverse=True)
        elif error:
            message = ansicolor.red(message, reverse=True)
        else:
            message = ansicolor.white(message, reverse=True)

        if not old_path and hasattr(self, 'old_path'):
            output_old = self.old_path
        else:
            output_old = old_path

        if not new_path and hasattr(self, 'new_path'):
            output_new = self.new_path
        else:
            output_new = new_path

        if not self.args.verbose:
            if output_old and hasattr(self, 'cwd') and len(self.cwd) > 1:
                output_old = output_old.replace(self.cwd, '')
            if output_new and len(self.job.target) > 1:
                output_new = output_new.replace(self.job.target, '')

        line1 = message + u' ' + output_old + '\n'
        if output_new:
            line2 = u'-> '.rjust(indent + 3) + ansicolor.yellow(output_new)
        else:
            line2 = u''

        out = line1 + line2

        if output == u'print':
            print(out)
        else:
            return out

    def dryRun(self):
        self.generateFilename()
        self.processMessage(action=u'Dry run')

    def mbTrackListing(self):
        m, s = divmod(self.meta.length, 60)
        mmss = '{:d}:{:02d}'.format(int(m), int(s))
        output = '{:d}. {:s}: {:s} ({:s})'.format(counter, self.meta.album,
                                                  self.meta.title, mmss)
        output = output.replace('Op.', 'op.')
        output = output.replace('- ', '')
        print(output)

    def fetch_work(self):
        self.meta.fetch_work()
        self.meta.save()
        self.processMessage(action=u'Get work')

    def action(self, copy=False):
        """Rename or copy to new path

        :param bool copy: Copy file

        :return: None
        """
        self.generateFilename()
        if not os.path.exists(self.new_path):
            self.createDir(self.new_path)
            if copy:
                self.processMessage(action=u'Copy')
                shutil.copy2(self.old_path, self.new_path)
            else:
                self.processMessage(action=u'Rename')
                shutil.move(self.old_path, self.new_path)
        elif self.new_path == self.old_path:
            self.processMessage(action=u'Renamed', error=False)
        else:
            self.processMessage(action=u'Exists', error=True)
            if self.args.delete_existing:
                os.remove(self.old_path)
                print('Delete existing file: ' + self.old_path)

    def execute(self):
        """
        .. todo:: Rethink `fetch work`
        """
        global counter
        counter += 1
        skip = self.args.skip_if_empty
        if not self.meta:
            self.processMessage(action=u'Broken file', error=True)
        elif skip and (not hasattr(self.meta, skip) or not
                       getattr(self.meta, skip)):
            self.processMessage(action=u'No field', error=True)
        else:
            if self.job.action == u'dry_run':
                self.dryRun()
            elif self.job.action == u'mb_track_listing':
                self.mbTrackListing()
            elif self.job.action == u'copy':
                self.action(copy=True)
            elif self.args.work:
                self.fetch_work()
            else:
                self.action()

def do_rename(path, args=None, job=None):
    if args.unittest:
        print(os.path.abspath(path))
    else:
        audio = Rename(path, args, job)
        audio.execute()
<|endoftext|>


<commit_before>#encoding=latin1

"Reimplement some of the watsup tests using pywinauto"
import sys
from time import sleep
import os
import os.path

from application import Application, WindowNotFoundError

def do_test_1():
    "1st Watsup Test"
    app = Application().start_(r"c:\windows\Notepad")
    notepadWindow = app.Notepad

    notepadWindow.Edit1.SetText(u"Hello, gain!", 0, -1)
    sleep(0.8)
    notepadWindow.Edit.SetText("\r\nYou still there?")

    sleep(0.2)
    notepadWindow.Edit.SetText("\r\nGoing Bye Bye now!!")

    sleep(1)
    notepadWindow.MenuSelect("File->Exit")

    app.Notepad.No.Click()

def do_test_2():
    "2nd Watsup Test"
    filename = 'atestfile.txt'

    if os.path.exists(filename):
        os.remove(filename)

    app = Application()
    try:
        app.connect_(title ='Simple Form')
    except WindowNotFoundError:
        app.start_(r'examples\simple.exe')

    form = app.SimpleForm

    form.Edit.SetText(filename)
    sleep(.6)

    print 'clicking button to create file'
    form.CreateFile.Click()

    # now check that the file is there
    if os.path.exists(filename):
        print 'file %s is present' % filename
    else:
        print "file %s isn't there" % filename

    form.MenuSelect("File->Exit")

def do_test_3():
    "3rd Watsup Test"
    app = Application()
    try:
        app.connect_(title ='Performance Form 2')
    except WindowNotFoundError:
        app.start_(r'examples\perform2.exe')

    app.PerformanceForm1.Clickme.Click()
    waited = 0
    while not app.PerformacneForm1.Edit1._.Texts and not waited >= 1:
        print 'waiting'
        sleep(.1)
        waited += .1

    print `app.PerformacneForm1.Edit1._.Texts`

def Main():
    "run the specified test"
    for test_num in sys.argv[1:]:
        globals()['do_test_%s'%test_num] ()

if __name__ == "__main__":
    Main()
<commit_msg>Re-organized some files<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3

import os, sys
sys.path.append(os.path.dirname(__file__))

import matplotlib.pyplot as pp
import numpy as np
import support
import tensorflow as tf

def assess(f):
    input_size = 5
    layer_count = 1
    layer_size = 20
    cell_type = 'lstm'

    train_batch_count = 10000
    predict_batch_count = 100
    imagine_batch_count = 10
    report_each = 100
    batch_size = 10
    start_learning_rate = 0.03
    learning_rate_decay = 1.0

    decay_fn = decay(start_learning_rate, learning_rate_decay)
    model_fn = model(input_size, layer_count, layer_size, cell_type)
    batch_fn = batch(f, input_size, 1, batch_size)

    graph = tf.Graph()
    with graph.as_default():
        x = tf.placeholder(tf.float32, [None, input_size, 1])
        y = tf.placeholder(tf.float32, [None, 1, 1])
        r = tf.Variable(0.0, trainable=False)

        y_hat, l = model_fn(x, y)

        trainees = tf.trainable_variables()
        gradient = tf.gradients(l, trainees)

        optimizer = tf.train.AdamOptimizer(r)
        train = optimizer.apply_gradients(zip(gradient, trainees))

        initialize = tf.initialize_all_variables()

    with tf.Session(graph=graph) as session:
        initialize.run()
        cursor = 0

        print('%10s %10s %10s' % ('Step', 'Rate', 'Loss'))
        for i in range(train_batch_count):
            x_observed, y_observed, cursor = batch_fn(cursor)
            r_current = decay_fn(i)
            l_current, _ = session.run([l, train], {
                r: r_current,
                x: x_observed,
                y: y_observed,
            })
            if (i + 1) % report_each == 0:
                print('%10s %10.2e %10.2f' % (i + 1, r_current, l_current))

        Y_observed = np.zeros([predict_batch_count * batch_size, 1])
        Y_predicted = np.zeros([predict_batch_count * batch_size, 1])
        for i in range(predict_batch_count):
            j, k = i * batch_size, (i + 1) * batch_size
            x_observed, y_observed, cursor = batch_fn(cursor)
            Y_observed[j:k] = np.reshape(y_observed, [batch_size, 1])
            y_predicted = session.run(y_hat, {x: x_observed})
            Y_predicted[j:k] = np.reshape(y_predicted, [batch_size, 1])
        compare(Y_observed, Y_predicted)

        x_observed = np.reshape(Y_observed[-input_size:], [1, input_size, 1])
        Y_observed = np.zeros([imagine_batch_count * batch_size, 1])
        Y_imagined = np.zeros([imagine_batch_count * batch_size, 1])
        for i in range(imagine_batch_count):
            j, k = i * batch_size, (i + 1) * batch_size
            _, y_observed, cursor = batch_fn(cursor)
            Y_observed[j:k] = np.reshape(y_observed, [batch_size, 1])
            for l in range(batch_size):
                y_predicted = session.run(y_hat, {x: x_observed})
                Y_imagined[j + l] = y_predicted[-1]
                x_observed[0, 0:(input_size - 1), 0] = x_observed[0, 1:, 0]
                x_observed[0, -1, 0 ] = y_predicted[-1]
        compare(Y_observed, Y_imagined, name='Imagined')

    pp.show()

def batch(f, input_size, output_size, batch_size):
    def compute(cursor):
        data = f(np.arange(cursor, cursor + input_size + batch_size - 1 + output_size))
        x = np.zeros([batch_size, input_size, 1], dtype=np.float32)
        y = np.zeros([batch_size, output_size, 1], dtype=np.float32)
        for i in range(batch_size):
            j = i + input_size
            k = j + output_size
            x[i, :, 0] = data[i:j]
            y[i, :, 0] = data[j:k]
        return x, y, cursor + batch_size

    return compute

def compare(y, y_hat, name='Predicted'):
    support.figure(height=6)
    pp.plot(y)
    pp.plot(y_hat)
    pp.legend(['Observed', name])

def decay(start, rate):
    def compute(i):
        return start * (rate ** i)

    return compute

def model(input_size, layer_count, layer_size, cell_type):
    def compute(x, y):
        if cell_type == 'rnn':
            cell = tf.nn.rnn_cell.BasicRNNCell(layer_size)
        elif cell_type == 'gru':
            cell = tf.nn.rnn_cell.GRUCell(layer_size)
        elif cell_type == 'lstm':
            cell = tf.nn.rnn_cell.BasicLSTMCell(layer_size, state_is_tuple=True)
        cell = tf.nn.rnn_cell.MultiRNNCell([cell] * layer_count, state_is_tuple=True)
        x = [tf.squeeze(x, squeeze_dims=[1]) for x in tf.split(1, input_size, x)]
        h, _ = tf.nn.rnn(cell, x, dtype=tf.float32)
        return regress(h[-1], y)

    def regress(x, y):
        w = tf.Variable(tf.truncated_normal([layer_size, 1]))
        b = tf.Variable(tf.zeros([1]))
        y_hat = tf.squeeze(tf.matmul(x, w) + b, squeeze_dims=[1])
        loss = tf.reduce_sum(tf.square(tf.sub(y_hat, y)))
        return y_hat, loss

    return compute

def target(x):
    return np.sin(0.1 * x)

assess(target)
<commit_msg>Rename a variable<commit_after>#!/usr/bin/env python3

import os, sys
sys.path.append(os.path.dirname(__file__))

import matplotlib.pyplot as pp
import numpy as np
import support
import tensorflow as tf

def assess(f):
    input_size = 5
    layer_count = 1
    layer_size = 20
    cell_type = 'lstm'

    batch_size = 10
    train_batch_count = 10000
    train_report_period = 100
    predict_batch_count = 100
    imagine_batch_count = 10
    start_learning_rate = 0.03
    learning_rate_decay = 1.0

    decay_fn = decay(start_learning_rate, learning_rate_decay)
    model_fn = model(input_size, layer_count, layer_size, cell_type)
    batch_fn = batch(f, input_size, 1, batch_size)

    graph = tf.Graph()
    with graph.as_default():
        x = tf.placeholder(tf.float32, [None, input_size, 1])
        y = tf.placeholder(tf.float32, [None, 1, 1])
        r = tf.Variable(0.0, trainable=False)

        y_hat, l = model_fn(x, y)

        trainees = tf.trainable_variables()
        gradient = tf.gradients(l, trainees)

        optimizer = tf.train.AdamOptimizer(r)
        train = optimizer.apply_gradients(zip(gradient, trainees))

        initialize = tf.initialize_all_variables()

    with tf.Session(graph=graph) as session:
        initialize.run()
        cursor = 0

        print('%10s %10s %10s' % ('Step', 'Rate', 'Loss'))
        for i in range(train_batch_count):
            x_observed, y_observed, cursor = batch_fn(cursor)
            r_current = decay_fn(i)
            l_current, _ = session.run([l, train], {
                r: r_current,
                x: x_observed,
                y: y_observed,
            })
            if (i + 1) % train_report_period == 0:
                print('%10s %10.2e %10.2f' % (i + 1, r_current, l_current))

        Y_observed = np.zeros([predict_batch_count * batch_size, 1])
        Y_predicted = np.zeros([predict_batch_count * batch_size, 1])
        for i in range(predict_batch_count):
            j, k = i * batch_size, (i + 1) * batch_size
            x_observed, y_observed, cursor = batch_fn(cursor)
            Y_observed[j:k] = np.reshape(y_observed, [batch_size, 1])
            y_predicted = session.run(y_hat, {x: x_observed})
            Y_predicted[j:k] = np.reshape(y_predicted, [batch_size, 1])
        compare(Y_observed, Y_predicted)

        x_observed = np.reshape(Y_observed[-input_size:], [1, input_size, 1])
        Y_observed = np.zeros([imagine_batch_count * batch_size, 1])
        Y_imagined = np.zeros([imagine_batch_count * batch_size, 1])
        for i in range(imagine_batch_count):
            j, k = i * batch_size, (i + 1) * batch_size
            _, y_observed, cursor = batch_fn(cursor)
            Y_observed[j:k] = np.reshape(y_observed, [batch_size, 1])
            for l in range(batch_size):
                y_predicted = session.run(y_hat, {x: x_observed})
                Y_imagined[j + l] = y_predicted[-1]
                x_observed[0, 0:(input_size - 1), 0] = x_observed[0, 1:, 0]
                x_observed[0, -1, 0 ] = y_predicted[-1]
        compare(Y_observed, Y_imagined, name='Imagined')

    pp.show()

def batch(f, input_size, output_size, batch_size):
    def compute(cursor):
        data = f(np.arange(cursor, cursor + input_size + batch_size - 1 + output_size))
        x = np.zeros([batch_size, input_size, 1], dtype=np.float32)
        y = np.zeros([batch_size, output_size, 1], dtype=np.float32)
        for i in range(batch_size):
            j = i + input_size
            k = j + output_size
            x[i, :, 0] = data[i:j]
            y[i, :, 0] = data[j:k]
        return x, y, cursor + batch_size

    return compute

def compare(y, y_hat, name='Predicted'):
    support.figure(height=6)
    pp.plot(y)
    pp.plot(y_hat)
    pp.legend(['Observed', name])

def decay(start, rate):
    def compute(i):
        return start * (rate ** i)

    return compute

def model(input_size, layer_count, layer_size, cell_type):
    def compute(x, y):
        if cell_type == 'rnn':
            cell = tf.nn.rnn_cell.BasicRNNCell(layer_size)
        elif cell_type == 'gru':
            cell = tf.nn.rnn_cell.GRUCell(layer_size)
        elif cell_type == 'lstm':
            cell = tf.nn.rnn_cell.BasicLSTMCell(layer_size, state_is_tuple=True)
        cell = tf.nn.rnn_cell.MultiRNNCell([cell] * layer_count, state_is_tuple=True)
        x = [tf.squeeze(x, squeeze_dims=[1]) for x in tf.split(1, input_size, x)]
        h, _ = tf.nn.rnn(cell, x, dtype=tf.float32)
        return regress(h[-1], y)

    def regress(x, y):
        w = tf.Variable(tf.truncated_normal([layer_size, 1]))
        b = tf.Variable(tf.zeros([1]))
        y_hat = tf.squeeze(tf.matmul(x, w) + b, squeeze_dims=[1])
        loss = tf.reduce_sum(tf.square(tf.sub(y_hat, y)))
        return y_hat, loss

    return compute

def target(x):
    return np.sin(0.1 * x)

assess(target)
<|endoftext|>


<commit_before>import json
import time

from anchore_engine.db import db_services, session_scope
from anchore_engine.subsys import logger
import anchore_engine.subsys.metrics
import anchore_engine.configuration.localconfig

service_statuses = {}

def set_status(service_record, up=True, available=True, busy=False, message="all good", detail={}, update_db=False):
    global service_statuses
    hostid = service_record['hostid']
    servicename = service_record['servicename']
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    if service not in service_statuses:
        service_statuses[service] = {}

    service_statuses[service]['up']= up
    service_statuses[service]['available'] = available
    service_statuses[service]['busy'] = busy
    service_statuses[service]['message'] = message
    service_statuses[service]['detail'] = detail

    if update_db:
        update_status(service_record)

def update_status(service_record):
    global service_statuses
    hostid = service_record['hostid']
    servicename = service_record['servicename']
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    #timer = time.time()
    with session_scope() as dbsession:
        my_service_record = db_services.get(hostid, servicename, session=dbsession)
        my_service_record['heartbeat'] = time.time()
        if my_service_record:
            if service_statuses[service]['up'] and service_statuses[service]['available']:
                my_service_record['status'] = True
            else:
                my_service_record['status'] = False

            my_service_record['short_description'] = json.dumps(service_statuses[service])
            db_services.update_record(my_service_record, session=dbsession)

    #anchore_engine.subsys.metrics.summary_observe('db_rw_probe', time.time() - timer)

    return(True)

def get_status(service_record):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    if service in service_statuses:
        ret = service_statuses[service]
    else:
        raise Exception("no service status set for service: " + str(service))
    return(ret)

def initialize_status(service_record, up=True, available=True, busy=False, message="all good", detail={}):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])
    if service not in service_statuses:
        set_status(service_record, up=up, available=available, busy=busy, message=message, detail=detail)
    return(True)

def has_status(service_record):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])
    ret = False
    if service in service_statuses:
        ret = True

    return(ret)

def handle_service_heartbeat(*args, **kwargs):
    cycle_timer = kwargs['mythread']['cycle_timer']
    localconfig = anchore_engine.configuration.localconfig.get_config()
    try:
        servicename = args[0]
    except Exception as err:
        raise Exception("BUG: need to provide service name as first argument to function: " + str(args))

    while(True):
        logger.debug("storing service status: " + str(servicename))
        try:
            service_record = {'hostid': localconfig['host_id'], 'servicename': servicename}
            update_status(service_record)
            logger.debug("service status stored: next in "+str(cycle_timer))
        except Exception as err:
            logger.error(str(err))

        time.sleep(cycle_timer)

    return(True)

<commit_msg>latest merge from master<commit_after>import json
import time

from anchore_engine.db import db_services, session_scope
from anchore_engine.subsys import logger
import anchore_engine.subsys.metrics
import anchore_engine.configuration.localconfig
import anchore_engine.version

service_statuses = {}

def set_status(service_record, up=True, available=True, busy=False, message="all good", detail={}, update_db=False):
    global service_statuses
    hostid = service_record['hostid']
    servicename = service_record['servicename']
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    if service not in service_statuses:
        service_statuses[service] = {}

    code_version = anchore_engine.version.version
    db_version = anchore_engine.version.db_version

    service_statuses[service]['up']= up
    service_statuses[service]['available'] = available
    service_statuses[service]['busy'] = busy
    service_statuses[service]['message'] = message
    service_statuses[service]['detail'] = detail
    service_statuses[service]['version'] = code_version
    service_statuses[service]['db_version'] = db_version

    if update_db:
        update_status(service_record)

def update_status(service_record):
    global service_statuses
    hostid = service_record['hostid']
    servicename = service_record['servicename']
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    #timer = time.time()
    with session_scope() as dbsession:
        my_service_record = db_services.get(hostid, servicename, session=dbsession)
        my_service_record['heartbeat'] = time.time()
        if my_service_record:
            if service_statuses[service]['up'] and service_statuses[service]['available']:
                my_service_record['status'] = True
            else:
                my_service_record['status'] = False

            my_service_record['short_description'] = json.dumps(service_statuses[service])
            db_services.update_record(my_service_record, session=dbsession)

    #anchore_engine.subsys.metrics.summary_observe('db_rw_probe', time.time() - timer)

    return(True)

def get_status(service_record):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])

    if service in service_statuses:
        ret = service_statuses[service]
    else:
        raise Exception("no service status set for service: " + str(service))
    return(ret)

def initialize_status(service_record, up=True, available=True, busy=False, message="all good", detail={}):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])
    if service not in service_statuses:
        set_status(service_record, up=up, available=available, busy=busy, message=message, detail=detail)
    return(True)

def has_status(service_record):
    global service_statuses
    service = '__'.join([service_record['hostid'], service_record['servicename']])
    ret = False
    if service in service_statuses:
        ret = True

    return(ret)

def handle_service_heartbeat(*args, **kwargs):
    cycle_timer = kwargs['mythread']['cycle_timer']
    localconfig = anchore_engine.configuration.localconfig.get_config()
    try:
        servicename = args[0]
    except Exception as err:
        raise Exception("BUG: need to provide service name as first argument to function: " + str(args))

    while(True):
        logger.debug("storing service status: " + str(servicename))
        try:
            service_record = {'hostid': localconfig['host_id'], 'servicename': servicename}
            update_status(service_record)
            logger.debug("service status stored: next in "+str(cycle_timer))
        except Exception as err:
            logger.error(str(err))

        time.sleep(cycle_timer)

    return(True)

<|endoftext|>