<commit_before># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import time

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

CS_HYPERVISORS = [
    "KVM", "kvm",
    "VMware", "vmware",
    "BareMetal", "baremetal",
    "XenServer", "xenserver",
    "LXC", "lxc",
    "HyperV", "hyperv",
    "UCS", "ucs",
    "OVM", "ovm",
    "Simulator", "simulator",
    ]

def cs_argument_spec():
    return dict(
        api_key = dict(default=None),
        api_secret = dict(default=None, no_log=True),
        api_url = dict(default=None),
        api_http_method = dict(choices=['get', 'post'], default='get'),
        api_timeout = dict(type='int', default=10),
        api_region = dict(default='cloudstack'),
    )

def cs_required_together():
    return [['api_key', 'api_secret', 'api_url']]

class AnsibleCloudStack(object):

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.result = {
            'changed': False,
        }

        # Common returns, will be merged with self.returns
        # search_for_key: replace_with_key
        self.common_returns = {
            'id':           'id',
            'name':         'name',
            'created':      'created',
            'zonename':     'zone',
            'state':        'state',
            'project':      'project',
            'account':      'account',
            'domain':       'domain',
            'displaytext':  'display_text',
            'displayname':  'display_name',
            'description':  'description',
        }

        # Init returns dict for use in subclasses
        self.returns = {}
        # these values will be casted to int
        self.returns_to_int = {}
        # these keys will be compared case sensitive in self.has_changed()
        self.case_sensitive_keys = [
            'id',
        ]

        self.module = module
        self._connect()

        self.domain = None
        self.account = None
        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None
        self.tags = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('secret_key')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')
        api_timeout = self.module.params.get('api_timeout')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                timeout=api_timeout,
                method=api_http_method
                )
        else:
            api_region = self.module.params.get('api_region', 'cloudstack')
            self.cs = CloudStack(**read_config(api_region))

    def get_or_fallback(self, key=None, fallback_key=None):
        value = self.module.params.get(key)
        if not value:
            value = self.module.params.get(fallback_key)
        return value

    # TODO: for backward compatibility only, remove if not used anymore
    def _has_changed(self, want_dict, current_dict, only_keys=None):
        return self.has_changed(want_dict=want_dict, current_dict=current_dict, only_keys=only_keys)

    def has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue

            # Skip None values
            if value is None:
                continue

            if key in current_dict:
                if self.case_sensitive_keys and key in self.case_sensitive_keys:
                    if str(value) != str(current_dict[key]):
                        return True
                # Test for diff in case insensitive way
                elif str(value).lower() != str(current_dict[key]).lower():
                    return True
            else:
                return True
        return False

    def _get_by_key(self, key=None, my_dict=None):
        if my_dict is None:
            my_dict = {}
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None
        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        projects = self.cs.listProjects(**args)
        if projects:
            for p in projects['project']:
                if project.lower() in [ p['name'].lower(), p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        args['zoneid'] = self.get_zone(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm.lower() in [ v['name'].lower(), v['displayname'].lower(), v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone.lower() in [ z['name'].lower(), z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def get_account(self, key=None):
        if self.account:
            return self._get_by_key(key, self.account)

        account = self.module.params.get('account')
        if not account:
            return None

        domain = self.module.params.get('domain')
        if not domain:
            self.module.fail_json(msg="Account must be specified with Domain")

        args = {}
        args['name'] = account
        args['domainid'] = self.get_domain(key='id')
        args['listall'] = True
        accounts = self.cs.listAccounts(**args)
        if accounts:
            self.account = accounts['account'][0]
            return self._get_by_key(key, self.account)
        self.module.fail_json(msg="Account '%s' not found" % account)

    def get_domain(self, key=None):
        if self.domain:
            return self._get_by_key(key, self.domain)

        domain = self.module.params.get('domain')
        if not domain:
            return None

        args = {}
        args['listall'] = True
        domains = self.cs.listDomains(**args)
        if domains:
            for d in domains['domain']:
                if d['path'].lower() in [ domain.lower(), "root/" + domain.lower(), "root" + domain.lower() ]:
                    self.domain = d
                    return self._get_by_key(key, self.domain)
        self.module.fail_json(msg="Domain '%s' not found" % domain)

    def get_tags(self, resource=None):
        if not self.tags:
            args = {}
            args['projectid'] = self.get_project(key='id')
            args['account'] = self.get_account(key='name')
            args['domainid'] = self.get_domain(key='id')
            args['resourceid'] = resource['id']
            response = self.cs.listTags(**args)
            self.tags = response.get('tag', [])

        existing_tags = []
        if self.tags:
            for tag in self.tags:
                existing_tags.append({'key': tag['key'], 'value': tag['value']})
        return existing_tags

    def _process_tags(self, resource, resource_type, tags, operation="create"):
        if tags:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags
                if operation == "create":
                    response = self.cs.createTags(**args)
                else:
                    response = self.cs.deleteTags(**args)
                self.poll_job(response)

    def _tags_that_should_exist_or_be_updated(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in tags if tag not in existing_tags]

    def _tags_that_should_not_exist(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in existing_tags if tag not in tags]

    def ensure_tags(self, resource, resource_type=None):
        if not resource_type or not resource:
            self.module.fail_json(msg="Error: Missing resource or resource_type for tags.")

        if 'tags' in resource:
            tags = self.module.params.get('tags')
            if tags is not None:
                self._process_tags(resource, resource_type, self._tags_that_should_not_exist(resource, tags), operation="delete")
                self._process_tags(resource, resource_type, self._tags_that_should_exist_or_be_updated(resource, tags))
                self.tags = None
                resource['tags'] = self.get_tags(resource)
        return resource

    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)

    # TODO: for backward compatibility only, remove if not used anymore
    def _poll_job(self, job=None, key=None):
        return self.poll_job(job=job, key=key)

    def poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job

    def get_result(self, resource):
        if resource:
            returns = self.common_returns.copy()
            returns.update(self.returns)
            for search_key, return_key in returns.iteritems():
                if search_key in resource:
                    self.result[return_key] = resource[search_key]

            # Bad bad API does not always return int when it should.
            for search_key, return_key in self.returns_to_int.iteritems():
                if search_key in resource:
                    self.result[return_key] = int(resource[search_key])

            # Special handling for tags
            if 'tags' in resource:
                self.result['tags'] = []
                for tag in resource['tags']:
                    result_tag          = {}
                    result_tag['key']   = tag['key']
                    result_tag['value'] = tag['value']
                    self.result['tags'].append(result_tag)
        return self.result
<commit_msg>cloudstack: fix bug, api_secret always None <commit_after># -*- coding: utf-8 -*-
#
# (c) 2015, René Moser <mail@renemoser.net>
#
# This code is part of Ansible, but is an independent component.
# This particular file snippet, and this file snippet only, is BSD licensed.
# Modules you write using this snippet, which is embedded dynamically by Ansible
# still belong to the author of the module, and may assign their own license
# to the complete work.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright notice,
#      this list of conditions and the following disclaimer in the documentation
#      and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import time

try:
    from cs import CloudStack, CloudStackException, read_config
    has_lib_cs = True
except ImportError:
    has_lib_cs = False

CS_HYPERVISORS = [
    "KVM", "kvm",
    "VMware", "vmware",
    "BareMetal", "baremetal",
    "XenServer", "xenserver",
    "LXC", "lxc",
    "HyperV", "hyperv",
    "UCS", "ucs",
    "OVM", "ovm",
    "Simulator", "simulator",
    ]

def cs_argument_spec():
    return dict(
        api_key = dict(default=None),
        api_secret = dict(default=None, no_log=True),
        api_url = dict(default=None),
        api_http_method = dict(choices=['get', 'post'], default='get'),
        api_timeout = dict(type='int', default=10),
        api_region = dict(default='cloudstack'),
    )

def cs_required_together():
    return [['api_key', 'api_secret', 'api_url']]

class AnsibleCloudStack(object):

    def __init__(self, module):
        if not has_lib_cs:
            module.fail_json(msg="python library cs required: pip install cs")

        self.result = {
            'changed': False,
        }

        # Common returns, will be merged with self.returns
        # search_for_key: replace_with_key
        self.common_returns = {
            'id':           'id',
            'name':         'name',
            'created':      'created',
            'zonename':     'zone',
            'state':        'state',
            'project':      'project',
            'account':      'account',
            'domain':       'domain',
            'displaytext':  'display_text',
            'displayname':  'display_name',
            'description':  'description',
        }

        # Init returns dict for use in subclasses
        self.returns = {}
        # these values will be casted to int
        self.returns_to_int = {}
        # these keys will be compared case sensitive in self.has_changed()
        self.case_sensitive_keys = [
            'id',
        ]

        self.module = module
        self._connect()

        self.domain = None
        self.account = None
        self.project = None
        self.ip_address = None
        self.zone = None
        self.vm = None
        self.os_type = None
        self.hypervisor = None
        self.capabilities = None
        self.tags = None

    def _connect(self):
        api_key = self.module.params.get('api_key')
        api_secret = self.module.params.get('api_secret')
        api_url = self.module.params.get('api_url')
        api_http_method = self.module.params.get('api_http_method')
        api_timeout = self.module.params.get('api_timeout')

        if api_key and api_secret and api_url:
            self.cs = CloudStack(
                endpoint=api_url,
                key=api_key,
                secret=api_secret,
                timeout=api_timeout,
                method=api_http_method
                )
        else:
            api_region = self.module.params.get('api_region', 'cloudstack')
            self.cs = CloudStack(**read_config(api_region))

    def get_or_fallback(self, key=None, fallback_key=None):
        value = self.module.params.get(key)
        if not value:
            value = self.module.params.get(fallback_key)
        return value

    # TODO: for backward compatibility only, remove if not used anymore
    def _has_changed(self, want_dict, current_dict, only_keys=None):
        return self.has_changed(want_dict=want_dict, current_dict=current_dict, only_keys=only_keys)

    def has_changed(self, want_dict, current_dict, only_keys=None):
        for key, value in want_dict.iteritems():

            # Optionally limit by a list of keys
            if only_keys and key not in only_keys:
                continue

            # Skip None values
            if value is None:
                continue

            if key in current_dict:
                if self.case_sensitive_keys and key in self.case_sensitive_keys:
                    if str(value) != str(current_dict[key]):
                        return True
                # Test for diff in case insensitive way
                elif str(value).lower() != str(current_dict[key]).lower():
                    return True
            else:
                return True
        return False

    def _get_by_key(self, key=None, my_dict=None):
        if my_dict is None:
            my_dict = {}
        if key:
            if key in my_dict:
                return my_dict[key]
            self.module.fail_json(msg="Something went wrong: %s not found" % key)
        return my_dict

    def get_project(self, key=None):
        if self.project:
            return self._get_by_key(key, self.project)

        project = self.module.params.get('project')
        if not project:
            return None
        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        projects = self.cs.listProjects(**args)
        if projects:
            for p in projects['project']:
                if project.lower() in [ p['name'].lower(), p['id'] ]:
                    self.project = p
                    return self._get_by_key(key, self.project)
        self.module.fail_json(msg="project '%s' not found" % project)

    def get_ip_address(self, key=None):
        if self.ip_address:
            return self._get_by_key(key, self.ip_address)

        ip_address = self.module.params.get('ip_address')
        if not ip_address:
            self.module.fail_json(msg="IP address param 'ip_address' is required")

        args = {}
        args['ipaddress'] = ip_address
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        ip_addresses = self.cs.listPublicIpAddresses(**args)

        if not ip_addresses:
            self.module.fail_json(msg="IP address '%s' not found" % args['ipaddress'])

        self.ip_address = ip_addresses['publicipaddress'][0]
        return self._get_by_key(key, self.ip_address)

    def get_vm(self, key=None):
        if self.vm:
            return self._get_by_key(key, self.vm)

        vm = self.module.params.get('vm')
        if not vm:
            self.module.fail_json(msg="Virtual machine param 'vm' is required")

        args = {}
        args['account'] = self.get_account(key='name')
        args['domainid'] = self.get_domain(key='id')
        args['projectid'] = self.get_project(key='id')
        args['zoneid'] = self.get_zone(key='id')
        vms = self.cs.listVirtualMachines(**args)
        if vms:
            for v in vms['virtualmachine']:
                if vm.lower() in [ v['name'].lower(), v['displayname'].lower(), v['id'] ]:
                    self.vm = v
                    return self._get_by_key(key, self.vm)
        self.module.fail_json(msg="Virtual machine '%s' not found" % vm)

    def get_zone(self, key=None):
        if self.zone:
            return self._get_by_key(key, self.zone)

        zone = self.module.params.get('zone')
        zones = self.cs.listZones()

        # use the first zone if no zone param given
        if not zone:
            self.zone = zones['zone'][0]
            return self._get_by_key(key, self.zone)

        if zones:
            for z in zones['zone']:
                if zone.lower() in [ z['name'].lower(), z['id'] ]:
                    self.zone = z
                    return self._get_by_key(key, self.zone)
        self.module.fail_json(msg="zone '%s' not found" % zone)

    def get_os_type(self, key=None):
        if self.os_type:
            return self._get_by_key(key, self.zone)

        os_type = self.module.params.get('os_type')
        if not os_type:
            return None

        os_types = self.cs.listOsTypes()
        if os_types:
            for o in os_types['ostype']:
                if os_type in [ o['description'], o['id'] ]:
                    self.os_type = o
                    return self._get_by_key(key, self.os_type)
        self.module.fail_json(msg="OS type '%s' not found" % os_type)

    def get_hypervisor(self):
        if self.hypervisor:
            return self.hypervisor

        hypervisor = self.module.params.get('hypervisor')
        hypervisors = self.cs.listHypervisors()

        # use the first hypervisor if no hypervisor param given
        if not hypervisor:
            self.hypervisor = hypervisors['hypervisor'][0]['name']
            return self.hypervisor

        for h in hypervisors['hypervisor']:
            if hypervisor.lower() == h['name'].lower():
                self.hypervisor = h['name']
                return self.hypervisor
        self.module.fail_json(msg="Hypervisor '%s' not found" % hypervisor)

    def get_account(self, key=None):
        if self.account:
            return self._get_by_key(key, self.account)

        account = self.module.params.get('account')
        if not account:
            return None

        domain = self.module.params.get('domain')
        if not domain:
            self.module.fail_json(msg="Account must be specified with Domain")

        args = {}
        args['name'] = account
        args['domainid'] = self.get_domain(key='id')
        args['listall'] = True
        accounts = self.cs.listAccounts(**args)
        if accounts:
            self.account = accounts['account'][0]
            return self._get_by_key(key, self.account)
        self.module.fail_json(msg="Account '%s' not found" % account)

    def get_domain(self, key=None):
        if self.domain:
            return self._get_by_key(key, self.domain)

        domain = self.module.params.get('domain')
        if not domain:
            return None

        args = {}
        args['listall'] = True
        domains = self.cs.listDomains(**args)
        if domains:
            for d in domains['domain']:
                if d['path'].lower() in [ domain.lower(), "root/" + domain.lower(), "root" + domain.lower() ]:
                    self.domain = d
                    return self._get_by_key(key, self.domain)
        self.module.fail_json(msg="Domain '%s' not found" % domain)

    def get_tags(self, resource=None):
        if not self.tags:
            args = {}
            args['projectid'] = self.get_project(key='id')
            args['account'] = self.get_account(key='name')
            args['domainid'] = self.get_domain(key='id')
            args['resourceid'] = resource['id']
            response = self.cs.listTags(**args)
            self.tags = response.get('tag', [])

        existing_tags = []
        if self.tags:
            for tag in self.tags:
                existing_tags.append({'key': tag['key'], 'value': tag['value']})
        return existing_tags

    def _process_tags(self, resource, resource_type, tags, operation="create"):
        if tags:
            self.result['changed'] = True
            if not self.module.check_mode:
                args = {}
                args['resourceids']  = resource['id']
                args['resourcetype'] = resource_type
                args['tags']         = tags
                if operation == "create":
                    response = self.cs.createTags(**args)
                else:
                    response = self.cs.deleteTags(**args)
                self.poll_job(response)

    def _tags_that_should_exist_or_be_updated(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in tags if tag not in existing_tags]

    def _tags_that_should_not_exist(self, resource, tags):
        existing_tags = self.get_tags(resource)
        return [tag for tag in existing_tags if tag not in tags]

    def ensure_tags(self, resource, resource_type=None):
        if not resource_type or not resource:
            self.module.fail_json(msg="Error: Missing resource or resource_type for tags.")

        if 'tags' in resource:
            tags = self.module.params.get('tags')
            if tags is not None:
                self._process_tags(resource, resource_type, self._tags_that_should_not_exist(resource, tags), operation="delete")
                self._process_tags(resource, resource_type, self._tags_that_should_exist_or_be_updated(resource, tags))
                self.tags = None
                resource['tags'] = self.get_tags(resource)
        return resource

    def get_capabilities(self, key=None):
        if self.capabilities:
            return self._get_by_key(key, self.capabilities)
        capabilities = self.cs.listCapabilities()
        self.capabilities = capabilities['capability']
        return self._get_by_key(key, self.capabilities)

    # TODO: for backward compatibility only, remove if not used anymore
    def _poll_job(self, job=None, key=None):
        return self.poll_job(job=job, key=key)

    def poll_job(self, job=None, key=None):
        if 'jobid' in job:
            while True:
                res = self.cs.queryAsyncJobResult(jobid=job['jobid'])
                if res['jobstatus'] != 0 and 'jobresult' in res:
                    if 'errortext' in res['jobresult']:
                        self.module.fail_json(msg="Failed: '%s'" % res['jobresult']['errortext'])
                    if key and key in res['jobresult']:
                        job = res['jobresult'][key]
                    break
                time.sleep(2)
        return job

    def get_result(self, resource):
        if resource:
            returns = self.common_returns.copy()
            returns.update(self.returns)
            for search_key, return_key in returns.iteritems():
                if search_key in resource:
                    self.result[return_key] = resource[search_key]

            # Bad bad API does not always return int when it should.
            for search_key, return_key in self.returns_to_int.iteritems():
                if search_key in resource:
                    self.result[return_key] = int(resource[search_key])

            # Special handling for tags
            if 'tags' in resource:
                self.result['tags'] = []
                for tag in resource['tags']:
                    result_tag          = {}
                    result_tag['key']   = tag['key']
                    result_tag['value'] = tag['value']
                    self.result['tags'].append(result_tag)
        return self.result
<|endoftext|>


<commit_before>'''
 Scrape out the Soil Nitrate data from Google Drive
'''
import sys
import pyiem.cscap_utils as util
import psycopg2
import datetime

YEAR = sys.argv[1]

config = util.get_config()

pgconn = psycopg2.connect(database='sustainablecorn',
                          host=config['database']['host'])
pcursor = pgconn.cursor()

# Get me a client, stat
spr_client = util.get_spreadsheet_client(config)
drive_client = util.get_driveclient(config)

res = drive_client.files(
        ).list(q="title contains 'Soil Nitrate Data'").execute()

DOMAIN = ['SOIL15', 'SOIL22', 'SOIL16', 'SOIL22', 'SOIL25', 'SOIL95',
          'SOIL94', 'SOIL93', 'SOIL92', 'SOIL91', 'SOIL90', 'SOIL89',
          'SOIL88', 'SOIL87', 'SOIL86', 'SOIL85', 'SOIL84', 'SOIL99']

# Load up what data we have for this year
current = {}
pcursor.execute("""SELECT site, plotid, varname, depth, subsample, sampledate
 from soil_data WHERE year = %s and varname in %s""", (YEAR,
                                                       tuple(DOMAIN)))
for row in pcursor:
    key = "%s|%s|%s|%s|%s|%s" % row
    current[key] = True

for item in res['items']:
    if item['mimeType'] != 'application/vnd.google-apps.spreadsheet':
        continue
    spreadsheet = util.Spreadsheet(spr_client, item['id'])
    spreadsheet.get_worksheets()
    worksheet = spreadsheet.worksheets.get(YEAR)
    if worksheet is None:
        # print("Missing Year: %s from %s" % (YEAR, spreadsheet.title))
        continue
    worksheet.get_cell_feed()
    siteid = item['title'].split()[0]
    # print 'Processing %s Soil Nitrate Year %s' % (siteid, YEAR),
    if worksheet.get_cell_value(1, 1) != 'plotid':
        print(('harvest_soil_nitrate: %s[%s] cell(1,1)="%s", skipping'
               ) % (siteid, YEAR, worksheet.get_cell_value(1, 1)))
        continue
    startcol = 3
    if worksheet.get_cell_value(1, 2) == 'depth':
        depthcol = 2
    elif worksheet.get_cell_value(1, 3) == 'depth':
        depthcol = 3
        startcol = 4
    if worksheet.get_cell_value(1, 2) == 'location':
        locationcol = 2
    else:
        locationcol = None

    for row in range(3, worksheet.rows+1):
        plotid = worksheet.get_cell_value(row, 1)
        depth = worksheet.get_cell_value(row, depthcol)
        if plotid is None or depth is None:
            continue
        subsample = "1"
        if locationcol is not None:
            subsample = worksheet.get_cell_value(row, locationcol)

        for col in range(startcol, worksheet.cols+1):
            if worksheet.get_cell_value(1, col) is None:
                print(("h_soil_nitrate site: %s year: %s col: %s is null"
                       ) % (siteid, YEAR, col))
                continue
            colheading = worksheet.get_cell_value(1, col).strip()
            if not colheading.startswith('SOIL'):
                print(('Invalid colheading: %s site: %s year: %s'
                       ) % (colheading, siteid, YEAR))
                continue
            # Attempt to tease out the sampledate
            tokens = colheading.split()
            varname = tokens[0]
            datetest = tokens[1]
            if len(datetest.split("/")) == 3:
                date = datetime.datetime.strptime(datetest, '%m/%d/%Y')
            else:
                if row == 3:
                    print(("h_soil_nitrate %s[%s] unknown sample date %s"
                           ) % (siteid, YEAR, repr(colheading)))
                date = None
            val = worksheet.get_cell_value(row, col, numeric=True)
            if varname not in DOMAIN:
                print(("harvest_soil_nitrate %s[%s] found additional var: %s"
                       ) % (siteid, YEAR, varname))
                DOMAIN.append(varname)
            try:
                pcursor.execute("""
                    INSERT into soil_data(site, plotid, varname, year,
                    depth, value, subsample, sampledate)
                    values (%s, %s, %s, %s, %s, %s, %s, %s)
                    """, (siteid, plotid, varname, YEAR, depth, val,
                          subsample, date))
            except Exception, exp:
                print(('site: %s year: %s HARVEST_SOIL_NITRATE TRACEBACK'
                       ) % (siteid, YEAR))
                print exp
                print '%s %s %s %s %s %s' % (siteid, plotid, varname, depth,
                                             date, val)
                sys.exit()
            key = ("%s|%s|%s|%s|%s|%s"
                   ) % (siteid, plotid, varname, depth, subsample,
                        date if date is None else date.strftime("%Y-%m-%d"))
            if key in current:
                del(current[key])

for key in current:
    (siteid, plotid, varname, depth, subsample, date) = key.split("|")
    if date != 'None':
        datesql = " and sampledate = '%s' " % (date, )
    else:
        datesql = " and sampledate is null "
    print(('h_soil_nitrate rm %s %s %s %s %s %s %s'
           ) % (YEAR, siteid, plotid, varname, depth, subsample,
                date))
    pcursor.execute("""
        DELETE from soil_data where site = %s and
        plotid = %s and varname = %s and year = %s and depth = %s and
        subsample = %s """ + datesql + """
    """, (siteid, plotid, varname, YEAR, depth, subsample))

    # print "...done"
pcursor.close()
pgconn.commit()
pgconn.close()
<commit_msg>[cscap] add more vars to soil nitrate deletion<commit_after>'''
 Scrape out the Soil Nitrate data from Google Drive
'''
import sys
import pyiem.cscap_utils as util
import psycopg2
import datetime

YEAR = sys.argv[1]

config = util.get_config()

pgconn = psycopg2.connect(database='sustainablecorn',
                          host=config['database']['host'])
pcursor = pgconn.cursor()

# Get me a client, stat
spr_client = util.get_spreadsheet_client(config)
drive_client = util.get_driveclient(config)

res = drive_client.files(
        ).list(q="title contains 'Soil Nitrate Data'").execute()

DOMAIN = ['SOIL15', 'SOIL22', 'SOIL16', 'SOIL22', 'SOIL25', 'SOIL95',
          'SOIL94', 'SOIL93', 'SOIL92', 'SOIL91', 'SOIL90', 'SOIL89',
          'SOIL88', 'SOIL87', 'SOIL86', 'SOIL85', 'SOIL84', 'SOIL99',
          'SOIL23', 'SOIL76', 'SOIL77', 'SOIL78', 'SOIL79', 'SOIL80',
          'SOIL81', 'SOIL82', 'SOIL83']

# Load up what data we have for this year
current = {}
pcursor.execute("""SELECT site, plotid, varname, depth, subsample, sampledate
 from soil_data WHERE year = %s and varname in %s""", (YEAR,
                                                       tuple(DOMAIN)))
for row in pcursor:
    key = "%s|%s|%s|%s|%s|%s" % row
    current[key] = True

for item in res['items']:
    if item['mimeType'] != 'application/vnd.google-apps.spreadsheet':
        continue
    spreadsheet = util.Spreadsheet(spr_client, item['id'])
    spreadsheet.get_worksheets()
    worksheet = spreadsheet.worksheets.get(YEAR)
    if worksheet is None:
        # print("Missing Year: %s from %s" % (YEAR, spreadsheet.title))
        continue
    worksheet.get_cell_feed()
    siteid = item['title'].split()[0]
    # print 'Processing %s Soil Nitrate Year %s' % (siteid, YEAR),
    if worksheet.get_cell_value(1, 1) != 'plotid':
        print(('harvest_soil_nitrate: %s[%s] cell(1,1)="%s", skipping'
               ) % (siteid, YEAR, worksheet.get_cell_value(1, 1)))
        continue
    startcol = 3
    if worksheet.get_cell_value(1, 2) == 'depth':
        depthcol = 2
    elif worksheet.get_cell_value(1, 3) == 'depth':
        depthcol = 3
        startcol = 4
    if worksheet.get_cell_value(1, 2) == 'location':
        locationcol = 2
    else:
        locationcol = None

    for row in range(3, worksheet.rows+1):
        plotid = worksheet.get_cell_value(row, 1)
        depth = worksheet.get_cell_value(row, depthcol)
        if plotid is None or depth is None:
            continue
        subsample = "1"
        if locationcol is not None:
            subsample = worksheet.get_cell_value(row, locationcol)

        for col in range(startcol, worksheet.cols+1):
            if worksheet.get_cell_value(1, col) is None:
                print(("h_soil_nitrate site: %s year: %s col: %s is null"
                       ) % (siteid, YEAR, col))
                continue
            colheading = worksheet.get_cell_value(1, col).strip()
            if not colheading.startswith('SOIL'):
                print(('Invalid colheading: %s site: %s year: %s'
                       ) % (colheading, siteid, YEAR))
                continue
            # Attempt to tease out the sampledate
            tokens = colheading.split()
            varname = tokens[0]
            datetest = tokens[1]
            if len(datetest.split("/")) == 3:
                date = datetime.datetime.strptime(datetest, '%m/%d/%Y')
            else:
                if row == 3:
                    print(("h_soil_nitrate %s[%s] unknown sample date %s"
                           ) % (siteid, YEAR, repr(colheading)))
                date = None
            val = worksheet.get_cell_value(row, col, numeric=True)
            if varname not in DOMAIN:
                print(("harvest_soil_nitrate %s[%s] found additional var: %s"
                       ) % (siteid, YEAR, varname))
                DOMAIN.append(varname)
            try:
                pcursor.execute("""
                    INSERT into soil_data(site, plotid, varname, year,
                    depth, value, subsample, sampledate)
                    values (%s, %s, %s, %s, %s, %s, %s, %s)
                    """, (siteid, plotid, varname, YEAR, depth, val,
                          subsample, date))
            except Exception, exp:
                print(('site: %s year: %s HARVEST_SOIL_NITRATE TRACEBACK'
                       ) % (siteid, YEAR))
                print exp
                print '%s %s %s %s %s %s' % (siteid, plotid, varname, depth,
                                             date, val)
                sys.exit()
            key = ("%s|%s|%s|%s|%s|%s"
                   ) % (siteid, plotid, varname, depth, subsample,
                        date if date is None else date.strftime("%Y-%m-%d"))
            if key in current:
                del(current[key])

for key in current:
    (siteid, plotid, varname, depth, subsample, date) = key.split("|")
    if date != 'None':
        datesql = " and sampledate = '%s' " % (date, )
    else:
        datesql = " and sampledate is null "
    print(('h_soil_nitrate rm %s %s %s %s %s %s %s'
           ) % (YEAR, siteid, plotid, varname, depth, subsample,
                date))
    pcursor.execute("""
        DELETE from soil_data where site = %s and
        plotid = %s and varname = %s and year = %s and depth = %s and
        subsample = %s """ + datesql + """
    """, (siteid, plotid, varname, YEAR, depth, subsample))

    # print "...done"
pcursor.close()
pgconn.commit()
pgconn.close()
<|endoftext|>


<commit_before>from Framework.BancoDeDados import BancoDeDados
from Database.Models.Usuario import Usuario as ModelUsuario

class Usuario(object):

	def pegarUsuarios(self, condicao, valores, inicio=0, quantidade=0):
		usuarios = []
		for usuario in BancoDeDados().consultarMultiplos("SELECT * FROM usuario WHERE %s" % (condicao), valores):
			usuarios.append(ModelUsuario(usuario))
		return usuarios

	def pegarUsuario(self, condicao, valores):
		return ModelUsuario(BancoDeDados().consultarUnico("SELECT * FROM usuario WHERE %s" % (condicao), valores))

	def inserirUsuario(self, usuario):
		BancoDeDados().executar("INSERT INTO usuario (matricula, nome, cpf, perfil, senha) VALUES (%s,%s,%s,%s,%s) RETURNING id", (usuario.matricula, usuario.nome, usuario.cpf, usuario.perfil, usuario.senha))
		usuario.id = BancoDeDados().pegarUltimoIDInserido()
		return usuario

	def removerUsuario(self, usuario):
		BancoDeDados().executar("DELETE FROM usuario WHERE id = %s", (str(usuario.id)))

	def alterarUsuario(self, usuario):
		SQL = "UPDATE usuario SET matricula = %s, nome = %s, cpf = %s, perfil = %s, senha = %s WHERE id = %s"
		BancoDeDados().executar(SQL, (usuario.matricula, usuario.nome, usuario.cpf, usuario.perfil, usuario.senha, usuario.id))
<commit_msg>corrigido bug de usuario nao encontrado<commit_after>from Framework.BancoDeDados import BancoDeDados
from Database.Models.Usuario import Usuario as ModelUsuario

class Usuario(object):

	def pegarUsuarios(self, condicao, valores, inicio=0, quantidade=0):
		usuarios = []
		for usuario in BancoDeDados().consultarMultiplos("SELECT * FROM usuario WHERE %s" % (condicao), valores):
			usuarios.append(ModelUsuario(usuario))
		return usuarios

	def pegarUsuario(self, condicao, valores):
		usuario = BancoDeDados().consultarUnico("SELECT * FROM usuario WHERE %s" % (condicao), valores)
		return (usuario is None) ? None : ModelUsuario(usuario)

	def inserirUsuario(self, usuario):
		BancoDeDados().executar("INSERT INTO usuario (matricula, nome, cpf, perfil, senha) VALUES (%s,%s,%s,%s,%s) RETURNING id", (usuario.matricula, usuario.nome, usuario.cpf, usuario.perfil, usuario.senha))
		usuario.id = BancoDeDados().pegarUltimoIDInserido()
		return usuario

	def removerUsuario(self, usuario):
		BancoDeDados().executar("DELETE FROM usuario WHERE id = %s", (str(usuario.id)))

	def alterarUsuario(self, usuario):
		SQL = "UPDATE usuario SET matricula = %s, nome = %s, cpf = %s, perfil = %s, senha = %s WHERE id = %s"
		BancoDeDados().executar(SQL, (usuario.matricula, usuario.nome, usuario.cpf, usuario.perfil, usuario.senha, usuario.id))
<|endoftext|>


<commit_before>#
#  Copyright 2018 Red Hat | Ansible
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import, division, print_function

import json
import os

from ansible.module_utils.k8s.common import OpenShiftAnsibleModuleMixin, DateTimeEncoder, remove_secret_data, to_snake
from ansible.module_utils.k8s.helper import AUTH_ARG_SPEC

try:
    from openshift.helper.kubernetes import KubernetesObjectHelper
    from openshift.helper.exceptions import KubernetesException
    HAS_K8S_MODULE_HELPER = True
except ImportError as exc:
    HAS_K8S_MODULE_HELPER = False

try:
    import yaml
    HAS_YAML = True
except ImportError:
    HAS_YAML = False

class KubernetesLookup(object):

    def __init__(self):

        if not HAS_K8S_MODULE_HELPER:
            raise Exception(
                "Requires the OpenShift Python client. Try `pip install openshift`"
            )

        if not HAS_YAML:
            raise Exception(
                "Requires PyYAML. Try `pip install PyYAML`"
            )

        self.kind = None
        self.name = None
        self.namespace = None
        self.api_version = None
        self.label_selector = None
        self.field_selector = None
        self.include_uninitialized = None
        self.resource_definition = None
        self.helper = None
        self.connection = {}

    def run(self, terms, variables=None, **kwargs):
        self.kind = kwargs.get('kind')
        self.name = kwargs.get('resource_name')
        self.namespace = kwargs.get('namespace')
        self.api_version = kwargs.get('api_version', 'v1')
        self.label_selector = kwargs.get('label_selector')
        self.field_selector = kwargs.get('field_selector')
        self.include_uninitialized = kwargs.get('include_uninitialized', False)

        resource_definition = kwargs.get('resource_definition')
        src = kwargs.get('src')
        if src:
            resource_definition = self.load_resource_definition(src)
        if resource_definition:
            self.params_from_resource_definition(resource_definition)

        if not self.kind:
            raise Exception(
                "Error: no Kind specified. Use the 'kind' parameter, or provide an object YAML configuration "
                "using the 'resource_definition' parameter."
            )

        self.kind = to_snake(self.kind)
        self.helper = self.get_helper(self.api_version, self.kind)

        for arg in AUTH_ARG_SPEC:
            self.connection[arg] = kwargs.get(arg)

        try:
            self.helper.set_client_config(**self.connection)
        except Exception as exc:
            raise Exception(
                "Client authentication failed: {0}".format(exc.message)
            )

        if self.name:
            return self.get_object()

        return self.list_objects()

    def get_helper(self, api_version, kind):
        try:
            helper = KubernetesObjectHelper(api_version=api_version, kind=kind, debug=False)
            helper.get_model(api_version, kind)
            return helper
        except KubernetesException as exc:
            raise Exception("Error initializing helper: {0}".format(exc.message))

    def load_resource_definition(self, src):
        """ Load the requested src path """
        path = os.path.normpath(src)
        if not os.path.exists(path):
            raise Exception("Error accessing {0}. Does the file exist?".format(path))
        try:
            result = yaml.safe_load(open(path, 'r'))
        except (IOError, yaml.YAMLError) as exc:
            raise Exception("Error loading resource_definition: {0}".format(exc))
        return result

    def params_from_resource_definition(self, defn):
        if defn.get('apiVersion'):
            self.api_version = defn['apiVersion']
        if defn.get('kind'):
            self.kind = defn['kind']
        if defn.get('metadata', {}).get('name'):
            self.name = defn['metadata']['name']
        if defn.get('metadata', {}).get('namespace'):
            self.namespace = defn['metadata']['namespace']

    def get_object(self):
        """ Fetch a named object """
        try:
            result = self.helper.get_object(self.name, self.namespace)
        except KubernetesException as exc:
            raise Exception('Failed to retrieve requested object: {0}'.format(exc.message))
        response = []
        if result is not None:
            # Convert Datetime objects to ISO format
            result_json = json.loads(json.dumps(result.to_dict(), cls=DateTimeEncoder))
            if self.kind == 'secret':
                remove_secret_data(result_json)
            response.append(result_json)

        return response

    def list_objects(self):
        """ Query for a set of objects """
        if self.namespace:
            method_name = 'list_namespaced_{0}'.format(self.kind)
            try:
                method = self.helper.lookup_method(method_name=method_name)
            except KubernetesException:
                raise Exception(
                    "Failed to find method {0} for API {1}".format(method_name, self.api_version)
                )
        else:
            method_name = 'list_{0}_for_all_namespaces'.format(self.kind)
            try:
                method = self.helper.lookup_method(method_name=method_name)
            except KubernetesException:
                method_name = 'list_{0}'.format(self.kind)
                try:
                    method = self.helper.lookup_method(method_name=method_name)
                except KubernetesException:
                    raise Exception(
                        "Failed to find method for API {0} and Kind {1}".format(self.api_version, self.kind)
                    )

        params = {}
        if self.field_selector:
            params['field_selector'] = self.field_selector
        if self.label_selector:
            params['label_selector'] = self.label_selector
        params['include_uninitialized'] = self.include_uninitialized

        if self.namespace:
            try:
                result = method(self.namespace, **params)
            except KubernetesException as exc:
                raise Exception(exc.message)
        else:
            try:
                result = method(**params)
            except KubernetesException as exc:
                raise Exception(exc.message)

        response = []
        if result is not None:
            # Convert Datetime objects to ISO format
            result_json = json.loads(json.dumps(result.to_dict(), cls=DateTimeEncoder))
            response = result_json.get('items', [])
            if self.kind == 'secret':
                for item in response:
                    remove_secret_data(item)
        return response

class OpenShiftLookup(OpenShiftAnsibleModuleMixin, KubernetesLookup):
    pass
<commit_msg>Auth method for the k8s lookup plugin now matches other modules (#37533)<commit_after>#
#  Copyright 2018 Red Hat | Ansible
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import, division, print_function

import json
import os

from ansible.module_utils.k8s.common import OpenShiftAnsibleModuleMixin, DateTimeEncoder, remove_secret_data, to_snake
from ansible.module_utils.k8s.helper import AUTH_ARG_SPEC

try:
    from openshift.helper.kubernetes import KubernetesObjectHelper
    from openshift.helper.exceptions import KubernetesException
    HAS_K8S_MODULE_HELPER = True
except ImportError as exc:
    HAS_K8S_MODULE_HELPER = False

try:
    import yaml
    HAS_YAML = True
except ImportError:
    HAS_YAML = False

class KubernetesLookup(object):

    def __init__(self):

        if not HAS_K8S_MODULE_HELPER:
            raise Exception(
                "Requires the OpenShift Python client. Try `pip install openshift`"
            )

        if not HAS_YAML:
            raise Exception(
                "Requires PyYAML. Try `pip install PyYAML`"
            )

        self.kind = None
        self.name = None
        self.namespace = None
        self.api_version = None
        self.label_selector = None
        self.field_selector = None
        self.include_uninitialized = None
        self.resource_definition = None
        self.helper = None
        self.connection = {}

    def run(self, terms, variables=None, **kwargs):
        self.kind = kwargs.get('kind')
        self.name = kwargs.get('resource_name')
        self.namespace = kwargs.get('namespace')
        self.api_version = kwargs.get('api_version', 'v1')
        self.label_selector = kwargs.get('label_selector')
        self.field_selector = kwargs.get('field_selector')
        self.include_uninitialized = kwargs.get('include_uninitialized', False)

        resource_definition = kwargs.get('resource_definition')
        src = kwargs.get('src')
        if src:
            resource_definition = self.load_resource_definition(src)
        if resource_definition:
            self.params_from_resource_definition(resource_definition)

        if not self.kind:
            raise Exception(
                "Error: no Kind specified. Use the 'kind' parameter, or provide an object YAML configuration "
                "using the 'resource_definition' parameter."
            )

        self.kind = to_snake(self.kind)
        self.helper = self.get_helper(self.api_version, self.kind)

        auth_args = ('host', 'api_key', 'kubeconfig', 'context', 'username', 'password',
                     'cert_file', 'key_file', 'ssl_ca_cert', 'verify_ssl')

        for arg in AUTH_ARG_SPEC:
            if arg in auth_args and kwargs.get(arg) is not None:
                self.connection[arg] = kwargs.get(arg)

        try:
            self.helper.set_client_config(**self.connection)
        except Exception as exc:
            raise Exception(
                "Client authentication failed: {0}".format(exc.message)
            )

        if self.name:
            return self.get_object()

        return self.list_objects()

    def get_helper(self, api_version, kind):
        try:
            helper = KubernetesObjectHelper(api_version=api_version, kind=kind, debug=False)
            helper.get_model(api_version, kind)
            return helper
        except KubernetesException as exc:
            raise Exception("Error initializing helper: {0}".format(exc.message))

    def load_resource_definition(self, src):
        """ Load the requested src path """
        path = os.path.normpath(src)
        if not os.path.exists(path):
            raise Exception("Error accessing {0}. Does the file exist?".format(path))
        try:
            result = yaml.safe_load(open(path, 'r'))
        except (IOError, yaml.YAMLError) as exc:
            raise Exception("Error loading resource_definition: {0}".format(exc))
        return result

    def params_from_resource_definition(self, defn):
        if defn.get('apiVersion'):
            self.api_version = defn['apiVersion']
        if defn.get('kind'):
            self.kind = defn['kind']
        if defn.get('metadata', {}).get('name'):
            self.name = defn['metadata']['name']
        if defn.get('metadata', {}).get('namespace'):
            self.namespace = defn['metadata']['namespace']

    def get_object(self):
        """ Fetch a named object """
        try:
            result = self.helper.get_object(self.name, self.namespace)
        except KubernetesException as exc:
            raise Exception('Failed to retrieve requested object: {0}'.format(exc.message))
        response = []
        if result is not None:
            # Convert Datetime objects to ISO format
            result_json = json.loads(json.dumps(result.to_dict(), cls=DateTimeEncoder))
            if self.kind == 'secret':
                remove_secret_data(result_json)
            response.append(result_json)

        return response

    def list_objects(self):
        """ Query for a set of objects """
        if self.namespace:
            method_name = 'list_namespaced_{0}'.format(self.kind)
            try:
                method = self.helper.lookup_method(method_name=method_name)
            except KubernetesException:
                raise Exception(
                    "Failed to find method {0} for API {1}".format(method_name, self.api_version)
                )
        else:
            method_name = 'list_{0}_for_all_namespaces'.format(self.kind)
            try:
                method = self.helper.lookup_method(method_name=method_name)
            except KubernetesException:
                method_name = 'list_{0}'.format(self.kind)
                try:
                    method = self.helper.lookup_method(method_name=method_name)
                except KubernetesException:
                    raise Exception(
                        "Failed to find method for API {0} and Kind {1}".format(self.api_version, self.kind)
                    )

        params = {}
        if self.field_selector:
            params['field_selector'] = self.field_selector
        if self.label_selector:
            params['label_selector'] = self.label_selector
        params['include_uninitialized'] = self.include_uninitialized

        if self.namespace:
            try:
                result = method(self.namespace, **params)
            except KubernetesException as exc:
                raise Exception(exc.message)
        else:
            try:
                result = method(**params)
            except KubernetesException as exc:
                raise Exception(exc.message)

        response = []
        if result is not None:
            # Convert Datetime objects to ISO format
            result_json = json.loads(json.dumps(result.to_dict(), cls=DateTimeEncoder))
            response = result_json.get('items', [])
            if self.kind == 'secret':
                for item in response:
                    remove_secret_data(item)
        return response

class OpenShiftLookup(OpenShiftAnsibleModuleMixin, KubernetesLookup):
    pass
<|endoftext|>


<commit_before>import Starfish
from Starfish import emulator
from Starfish.covariance import Sigma
import numpy as np
import math
import multiprocessing as mp

# Load the PCAGrid from the location specified in the config file
pca = emulator.PCAGrid.open()
PhiPhi = np.linalg.inv(emulator.skinny_kron(pca.eigenspectra, pca.M))

def Glnprior(x, s, r):
    return s * np.log(r) + (s - 1.) * np.log(x) - r*x - math.lgamma(s)

def lnprob(p):
    '''

    :param p: Gaussian Processes hyper-parameters
    :type p: 1D np.array

    Calculate the lnprob using Habib's posterior formula for the emulator.

    '''

    # We don't allow negative parameters.
    if np.any(p < 0.):
        return 1e99

    lambda_xi = p[0]
    hparams = p[1:].reshape((pca.m, -1))

    print("lambda_xi", lambda_xi)
    for row in hparams:
        print(row)
    print()

    # Calculate the prior for temp, logg, and Z
    priors = np.sum(Glnprior(hparams[:, 1], 2., 0.0075)) + np.sum(Glnprior(hparams[:, 2], 2., 0.75)) + np.sum(Glnprior(hparams[:, 3], 2., 0.75))

    h2params = hparams**2
    #Fold hparams into the new shape
    Sig_w = Sigma(pca.gparams, h2params)

    C = (1./lambda_xi) * PhiPhi + Sig_w

    sign, pref = np.linalg.slogdet(C)

    central = pca.w_hat.T.dot(np.linalg.solve(C, pca.w_hat))

    lnp = -0.5 * (pref + central + pca.M * pca.m * np.log(2. * np.pi)) + priors
    print(lnp)

    # Negate this since we are using the fmin algorithm
    return -lnp

def minimize():

    amp = 50.0
    lt = 200.
    ll = 1.25
    lZ = 1.25
    #
    p0 = np.hstack((np.array([1., ]),
    np.hstack([np.array([amp, lt, ll, lZ]) for i in range(pca.m)]) ))

    # # Set lambda_xi separately
    p0[0] = 0.3

    # p0 = np.load("eparams.npy")

    from scipy.optimize import fmin
    result = fmin(lnprob, p0)
    print(result)
    np.save("eparams.npy", result)

def sample():
    import emcee

    ndim = 1 + (1 + len(Starfish.parname)) * pca.m
    print(ndim)
    nwalkers = 4 * ndim # about the minimum per dimension we can get by with
    print(nwalkers)

    # Assemble p0 based off either a guess or the previous state of walkers

    p0 = []
    # p0 is a (nwalkers, ndim) array
    amp = [40.0, 100]
    lt = [150., 300]
    ll = [0.8, 1.65]
    lZ = [0.8, 1.65]

    p0.append(np.random.uniform(0.1, 1.0, nwalkers))
    block = [np.random.uniform(amp[0], amp[1], nwalkers),
            np.random.uniform(lt[0], lt[1], nwalkers),
            np.random.uniform(ll[0], ll[1], nwalkers),
            np.random.uniform(lZ[0], lZ[1], nwalkers)]
    for i in range(pca.m):
        p0 += block

    p0 = np.array(p0).T

    # p0 = np.load("eparams_walkers.npy")

    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, threads=mp.cpu_count())

    # burn in
    sampler.run_mcmc(p0, 20)
    sampler.reset()

    # actual run
    sampler.run_mcmc(p0, 100)

    np.save("eparams.npy", sampler.flatchain)

def main():

    # Set up starting parameters and see if lnprob evaluates.
    # p will have a length of 1 + (pca.m * (1 + len(Starfish.parname)))

    # minimize(p0)
    sample()

if __name__=="__main__":
    main()
<commit_msg>[skip ci] negate lnprob from fmin<commit_after>import Starfish
from Starfish import emulator
from Starfish.covariance import Sigma
import numpy as np
import math
import multiprocessing as mp

# Load the PCAGrid from the location specified in the config file
pca = emulator.PCAGrid.open()
PhiPhi = np.linalg.inv(emulator.skinny_kron(pca.eigenspectra, pca.M))

def Glnprior(x, s, r):
    return s * np.log(r) + (s - 1.) * np.log(x) - r*x - math.lgamma(s)

def lnprob(p):
    '''

    :param p: Gaussian Processes hyper-parameters
    :type p: 1D np.array

    Calculate the lnprob using Habib's posterior formula for the emulator.

    '''

    # We don't allow negative parameters.
    if np.any(p < 0.):
        return 1e99

    lambda_xi = p[0]
    hparams = p[1:].reshape((pca.m, -1))

    print("lambda_xi", lambda_xi)
    for row in hparams:
        print(row)
    print()

    # Calculate the prior for temp, logg, and Z
    priors = np.sum(Glnprior(hparams[:, 1], 2., 0.0075)) + np.sum(Glnprior(hparams[:, 2], 2., 0.75)) + np.sum(Glnprior(hparams[:, 3], 2., 0.75))

    h2params = hparams**2
    #Fold hparams into the new shape
    Sig_w = Sigma(pca.gparams, h2params)

    C = (1./lambda_xi) * PhiPhi + Sig_w

    sign, pref = np.linalg.slogdet(C)

    central = pca.w_hat.T.dot(np.linalg.solve(C, pca.w_hat))

    lnp = -0.5 * (pref + central + pca.M * pca.m * np.log(2. * np.pi)) + priors
    print(lnp)

    # Negate this when using the fmin algorithm
    return lnp

def minimize():

    amp = 50.0
    lt = 200.
    ll = 1.25
    lZ = 1.25
    #
    p0 = np.hstack((np.array([1., ]),
    np.hstack([np.array([amp, lt, ll, lZ]) for i in range(pca.m)]) ))

    # # Set lambda_xi separately
    p0[0] = 0.3

    # p0 = np.load("eparams.npy")

    from scipy.optimize import fmin
    result = fmin(lnprob, p0)
    print(result)
    np.save("eparams.npy", result)

def sample():
    import emcee

    ndim = 1 + (1 + len(Starfish.parname)) * pca.m
    print(ndim)
    nwalkers = 4 * ndim # about the minimum per dimension we can get by with
    print(nwalkers)

    # Assemble p0 based off either a guess or the previous state of walkers

    p0 = []
    # p0 is a (nwalkers, ndim) array
    amp = [40.0, 100]
    lt = [150., 300]
    ll = [0.8, 1.65]
    lZ = [0.8, 1.65]

    p0.append(np.random.uniform(0.1, 1.0, nwalkers))
    block = [np.random.uniform(amp[0], amp[1], nwalkers),
            np.random.uniform(lt[0], lt[1], nwalkers),
            np.random.uniform(ll[0], ll[1], nwalkers),
            np.random.uniform(lZ[0], lZ[1], nwalkers)]
    for i in range(pca.m):
        p0 += block

    p0 = np.array(p0).T

    # p0 = np.load("eparams_walkers.npy")

    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, threads=mp.cpu_count())

    # burn in
    sampler.run_mcmc(p0, 20)
    sampler.reset()

    # actual run
    sampler.run_mcmc(p0, 100)

    np.save("eparams.npy", sampler.flatchain)

def main():

    # Set up starting parameters and see if lnprob evaluates.
    # p will have a length of 1 + (pca.m * (1 + len(Starfish.parname)))

    # minimize(p0)
    sample()

if __name__=="__main__":
    main()
<|endoftext|>


<commit_before>import sys
from array import array
from functools import partial
from collections.abc import Mapping
from math import ceil
from datetime import datetime, timezone

from ncplib.constants import PACKET_HEADER_SIZE, PACKET_FIELD_HEADER_SIZE, PACKET_PARAM_HEADER_SIZE, PACKET_FOOTER_SIZE, PACKET_HEADER_HEADER, PACKET_FOOTER_HEADER, PacketFormat, ParamType
from ncplib.packets import RawParamValue

# Type checking.

def _require_type(name, value, type, type_desc=None):  # pragma: no cover
    if not isinstance(value, type):
        raise TypeError("{} '{}' should be of type {}".format(name, value, type_desc or type.__name__))

_require_bytes = partial(_require_type, type=(bytes, bytearray, memoryview), type_desc="bytes")

_require_int = partial(_require_type, type=int)

def _require_key(name, value):  # pragma: no cover
    _require_bytes(name, value)
    if len(value) != 4:
        raise ValueError("{} '{}' should be a four-byte value".format(name, value))

def _require_timestamp(name, value):  # pragma: no cover
    _require_type(name, value, datetime)
    if value.tzinfo is None:
        raise ValueError("{} '{}' should be timezone-aware".format(name, value))

# Param serialization.

def _serialize_param_value_raw(param_value):
    _require_bytes("Raw param value", param_value.value)
    _require_int("Raw param type ID", param_value.type_id)
    return param_value.type_id, param_value.value

def _serialize_param_value_int(param_value):
    return ParamType.i32.value, param_value.to_bytes(4, "little", signed=True)

def _serialize_param_value_str(param_value):
    return ParamType.string.value, param_value.encode("latin1", errors="ignore") + b"\x00"

def _serialize_param_value_bytes(param_value):
    return ParamType.raw.value, param_value

_PARAM_VALUE_ARRAY_TYPES = {
    "B": ParamType.u8array,
    "H": ParamType.u16array,
    "I": ParamType.u32array,
    "b": ParamType.i8array,
    "h": ParamType.i16array,
    "i": ParamType.i32array,
}

def _serialize_param_value_array(param_value):
    try:
        param_type = _PARAM_VALUE_ARRAY_TYPES[param_value.typecode]
    except KeyError:  # pragma: no cover
        raise TypeError("Unsupported param value type (array[{}])".format(param_value.typecode))
    # Convert the value to bytes.
    if sys.byteorder == "big":
        param_value = param_value[:].byteswap()
    # All done!
    return param_type.value, param_value.tobytes()

_PARAM_VALUE_SERIALIZERS = {
    RawParamValue: _serialize_param_value_raw,
    int: _serialize_param_value_int,
    str: _serialize_param_value_str,
    bytes: _serialize_param_value_bytes,
    bytearray: _serialize_param_value_bytes,
    memoryview: _serialize_param_value_bytes,
    array: _serialize_param_value_array,
}

def _serialize_param_value(param_value):
    try:
        param_value_serializer = _PARAM_VALUE_SERIALIZERS[type(param_value)]
    except KeyError:  # pragma: no cover
        raise TypeError("Unsupported param value type ({})".format(type(param_value).__name__))
    param_type_id, serialized_param_value = param_value_serializer(param_value)
    param_size = int(ceil((PACKET_PARAM_HEADER_SIZE + len(serialized_param_value)) / 4) * 4)  # Round up size to nearest 4 bytes.
    return param_size, param_type_id, serialized_param_value

# Encoders.

def _encode_uint(buf, value):
    buf[:] = value.to_bytes(len(buf), "little", signed=False)

def _encode_size(buf, value):
    _encode_uint(buf, int(value / 4))

def _encode_timestamp(buf, value):
    value = value.astimezone(timezone.utc)
    _encode_uint(buf[:4], int(value.timestamp()))
    _encode_uint(buf[4:8], value.microsecond * 1000)

def _encode_param(buf, param_name, param_size, param_type_id, serialized_param_value):
    _require_key("Param name", param_name)
    buf[:4] = param_name
    _encode_size(buf[4:7], param_size)
    _encode_uint(buf[7:8], param_type_id)
    buf[8:8+len(serialized_param_value)] = serialized_param_value

def _encode_field(buf, field_name, field_size, field_id, serialized_params):
    _require_key("Field name", field_name)
    buf[:4] = field_name
    _encode_size(buf[4:7], field_size)
    _encode_uint(buf[8:12], field_id)
    # Encode the params.
    field_write_position = PACKET_FIELD_HEADER_SIZE
    for param_name, param_size, param_type_id, serialized_param_value in serialized_params:
        _encode_param(buf[field_write_position:field_write_position+param_size], param_name, param_size, param_type_id, serialized_param_value)
        field_write_position += param_size

def encode_packet(packet_type, packet_id, packet_timestamp, packet_info, packet_fields):
    _require_key("Packet type", packet_type)
    _require_int("Packet ID", packet_id)
    _require_timestamp("Packet timestamp", packet_timestamp)
    _require_key("Packet info", packet_info)
    _require_type("Packet fields", packet_fields, Mapping)
    # Serialize the param values.
    packet_size = PACKET_HEADER_SIZE + PACKET_FOOTER_SIZE
    serialized_fields = []
    for field_name, params in packet_fields.items():
        field_size = PACKET_FIELD_HEADER_SIZE
        serialized_params = []
        for param_name, param_value in params.items():
            param_size, param_type_id, serialized_param_value = _serialize_param_value(param_value)
            serialized_params.append((param_name, param_size, param_type_id, serialized_param_value))
            field_size += param_size
        serialized_fields.append((field_name, field_size, serialized_params))
        packet_size += field_size
    # Create a buffer for the packet.
    packet_data = bytearray(packet_size)
    buf = memoryview(packet_data)
    # Encode the header.
    buf[:4] = PACKET_HEADER_HEADER
    buf[4:8] = packet_type
    _encode_size(buf[8:12], packet_size)
    _encode_uint(buf[12:16], packet_id)
    _encode_uint(buf[16:20], PacketFormat.standard.value)
    _encode_timestamp(buf[20:28], packet_timestamp)
    buf[28:32] = packet_info
    # Encode the fields.
    field_write_position = PACKET_HEADER_SIZE
    for field_id, serialized_field in enumerate(serialized_fields):
        field_name, field_size, serialized_params = serialized_field
        _encode_field(buf[field_write_position:field_write_position+field_size], field_name, field_size, field_id, serialized_params)
        field_write_position += field_size
    # Encode the footer.
    buf[-4:] = PACKET_FOOTER_HEADER
    # All done!
    buf.release()
    return packet_data
<commit_msg>Cleaning up decoder<commit_after>import sys
from array import array
from functools import partial
from collections.abc import Mapping
from math import ceil
from datetime import datetime, timezone

from ncplib.constants import PACKET_HEADER_SIZE, PACKET_FIELD_HEADER_SIZE, PACKET_PARAM_HEADER_SIZE, PACKET_FOOTER_SIZE, PACKET_HEADER_HEADER, PACKET_FOOTER_HEADER, PacketFormat, ParamType
from ncplib.packets import RawParamValue

# Type checking.

def _require_type(name, value, type, type_desc=None):  # pragma: no cover
    if not isinstance(value, type):
        raise TypeError("{} '{}' should be of type {}".format(name, value, type_desc or type.__name__))

_require_bytes = partial(_require_type, type=(bytes, bytearray, memoryview), type_desc="bytes")

_require_int = partial(_require_type, type=int)

def _require_key(name, value):  # pragma: no cover
    _require_bytes(name, value)
    if len(value) != 4:
        raise ValueError("{} '{}' should be a four-byte value".format(name, value))

def _require_timestamp(name, value):  # pragma: no cover
    _require_type(name, value, datetime)
    if value.tzinfo is None:
        raise ValueError("{} '{}' should be timezone-aware".format(name, value))

# Param serialization.

def _serialize_param_value_raw(param_value):
    _require_bytes("Raw param value", param_value.value)
    _require_int("Raw param type ID", param_value.type_id)
    return param_value.type_id, param_value.value

def _serialize_param_value_int(param_value):
    return ParamType.i32.value, param_value.to_bytes(4, "little", signed=True)

def _serialize_param_value_str(param_value):
    return ParamType.string.value, param_value.encode("latin1", errors="ignore") + b"\x00"

def _serialize_param_value_bytes(param_value):
    return ParamType.raw.value, param_value

_PARAM_VALUE_ARRAY_TYPES = {
    "B": ParamType.u8array,
    "H": ParamType.u16array,
    "I": ParamType.u32array,
    "b": ParamType.i8array,
    "h": ParamType.i16array,
    "i": ParamType.i32array,
}

def _serialize_param_value_array(param_value):
    try:
        param_type = _PARAM_VALUE_ARRAY_TYPES[param_value.typecode]
    except KeyError:  # pragma: no cover
        raise TypeError("Unsupported param value type (array[{}])".format(param_value.typecode))
    # Convert the value to bytes.
    if sys.byteorder == "big":
        param_value = param_value[:].byteswap()
    # All done!
    return param_type.value, param_value.tobytes()

_PARAM_VALUE_SERIALIZERS = {
    RawParamValue: _serialize_param_value_raw,
    int: _serialize_param_value_int,
    str: _serialize_param_value_str,
    bytes: _serialize_param_value_bytes,
    bytearray: _serialize_param_value_bytes,
    memoryview: _serialize_param_value_bytes,
    array: _serialize_param_value_array,
}

def _serialize_param_value(param_value):
    try:
        param_value_serializer = _PARAM_VALUE_SERIALIZERS[type(param_value)]
    except KeyError:  # pragma: no cover
        raise TypeError("Unsupported param value type ({})".format(type(param_value).__name__))
    param_type_id, serialized_param_value = param_value_serializer(param_value)
    param_size = int(ceil((PACKET_PARAM_HEADER_SIZE + len(serialized_param_value)) / 4) * 4)  # Round up size to nearest 4 bytes.
    return param_size, param_type_id, serialized_param_value

# Encoders.

def _encode_uint(buf, value):
    buf[:] = value.to_bytes(len(buf), "little", signed=False)

def _encode_size(buf, value):
    _encode_uint(buf, int(value / 4))

def _encode_timestamp(buf, value):
    value = value.astimezone(timezone.utc)
    _encode_uint(buf[:4], int(value.timestamp()))
    _encode_uint(buf[4:8], value.microsecond * 1000)

def _encode_param(buf, param_name, param_size, param_type_id, serialized_param_value):
    _require_key("Param name", param_name)
    buf[:4] = param_name
    _encode_size(buf[4:7], param_size)
    _encode_uint(buf[7:8], param_type_id)
    buf[8:8+len(serialized_param_value)] = serialized_param_value

def _encode_field(buf, field_name, field_size, field_id, serialized_params):
    _require_key("Field name", field_name)
    buf[:4] = field_name
    _encode_size(buf[4:7], field_size)
    _encode_uint(buf[8:12], field_id)
    # Encode the params.
    field_write_position = PACKET_FIELD_HEADER_SIZE
    for param_name, param_size, param_type_id, serialized_param_value in serialized_params:
        _encode_param(buf[field_write_position:field_write_position+param_size], param_name, param_size, param_type_id, serialized_param_value)
        field_write_position += param_size

def encode_packet(packet_type, packet_id, packet_timestamp, packet_info, packet_fields):
    _require_key("Packet type", packet_type)
    _require_int("Packet ID", packet_id)
    _require_timestamp("Packet timestamp", packet_timestamp)
    _require_key("Packet info", packet_info)
    _require_type("Packet fields", packet_fields, Mapping)
    # Serialize the param values.
    packet_size = PACKET_HEADER_SIZE + PACKET_FOOTER_SIZE
    serialized_fields = []
    for field_name, params in packet_fields.items():
        field_size = PACKET_FIELD_HEADER_SIZE
        serialized_params = []
        for param_name, param_value in params.items():
            param_size, param_type_id, serialized_param_value = _serialize_param_value(param_value)
            serialized_params.append((param_name, param_size, param_type_id, serialized_param_value))
            field_size += param_size
        serialized_fields.append((field_name, field_size, serialized_params))
        packet_size += field_size
    # Create a buffer for the packet.
    packet_data = bytearray(packet_size)
    buf = memoryview(packet_data)
    # Encode the header.
    buf[:4] = PACKET_HEADER_HEADER
    buf[4:8] = packet_type
    _encode_size(buf[8:12], packet_size)
    _encode_uint(buf[12:16], packet_id)
    _encode_uint(buf[16:20], PacketFormat.standard.value)
    _encode_timestamp(buf[20:28], packet_timestamp)
    buf[28:32] = packet_info
    # Encode the fields.
    field_write_position = PACKET_HEADER_SIZE
    for field_id, serialized_field in enumerate(serialized_fields):
        field_name, field_size, serialized_params = serialized_field
        _encode_field(buf[field_write_position:field_write_position+field_size], field_name, field_size, field_id, serialized_params)
        field_write_position += field_size
    # Encode the footer.
    buf[packet_size-4:packet_size] = PACKET_FOOTER_HEADER
    # All done!
    buf.release()
    return packet_data
<|endoftext|>