<commit_before># Copyright 2019 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import signal

import mock

from neutron_lib import exceptions as n_exc

from neutron.agent.linux import ip_lib
from neutron.agent.linux import utils
from neutron.common import utils as n_utils
from neutron.tests.common import net_helpers
from neutron.tests.functional import base

class NetcatTesterTestCase(base.BaseSudoTestCase):

    def setUp(self):
        super(NetcatTesterTestCase, self).setUp()
        self.ns1 = self.useFixture(net_helpers.NamespaceFixture('nc-')).name
        self.ns2 = self.useFixture(net_helpers.NamespaceFixture('nc-')).name
        self.nc = net_helpers.NetcatTester(self.ns1, self.ns2, '10.30.0.2',
                                           '1234', 'tcp')
        ip_wrapper = ip_lib.IPWrapper(namespace=self.ns1)
        veth1, veth2 = ip_wrapper.add_veth('veth1', 'veth2', self.ns2)
        veth1.link.set_up()
        veth1.addr.add('10.30.0.1/24')
        veth2.link.set_up()
        veth2.addr.add('10.30.0.2/24')

    def test_stop_process(self):
        self.nc.test_connectivity()
        server_pid = self.nc.server_process.child_pid
        client_pid = self.nc.client_process.child_pid
        self.assertTrue(utils.process_is_running(server_pid))
        self.assertTrue(utils.process_is_running(client_pid))

        self.nc.stop_processes()
        self.assertFalse(utils.process_is_running(server_pid))
        self.assertFalse(utils.process_is_running(client_pid))

    def test_stop_process_no_process(self):
        self.nc.test_connectivity()
        client_pid = self.nc.client_process.child_pid
        utils.execute(['kill', '-%d' % signal.SIGKILL, client_pid],
                      run_as_root=True)
        n_utils.wait_until_true(
            lambda: not utils.process_is_running(client_pid), timeout=5)
        with mock.patch.object(net_helpers.RootHelperProcess, 'poll',
                               return_value=None):
            self.assertRaises(n_exc.ProcessExecutionError,
                              self.nc.stop_processes,
                              skip_errors=[])

    def test_stop_process_no_process_skip_no_process_exception(self):
        self.nc.test_connectivity()
        server_pid = self.nc.server_process.child_pid
        utils.execute(['kill', '-%d' % signal.SIGKILL, server_pid],
                      run_as_root=True)

        with mock.patch.object(net_helpers.RootHelperProcess, 'poll',
                               side_effect=[None, True, None, True]):
            self.nc.stop_processes()
<commit_msg>Mark NetcatTesterTestCase tests as unstable<commit_after># Copyright 2019 Red Hat, Inc.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import signal

import mock

from neutron_lib import exceptions as n_exc

from neutron.agent.linux import ip_lib
from neutron.agent.linux import utils
from neutron.common import utils as n_utils
from neutron.tests import base as test_base
from neutron.tests.common import net_helpers
from neutron.tests.functional import base

class NetcatTesterTestCase(base.BaseSudoTestCase):

    def setUp(self):
        super(NetcatTesterTestCase, self).setUp()
        self.ns1 = self.useFixture(net_helpers.NamespaceFixture('nc-')).name
        self.ns2 = self.useFixture(net_helpers.NamespaceFixture('nc-')).name
        self.nc = net_helpers.NetcatTester(self.ns1, self.ns2, '10.30.0.2',
                                           '1234', 'tcp')
        ip_wrapper = ip_lib.IPWrapper(namespace=self.ns1)
        veth1, veth2 = ip_wrapper.add_veth('veth1', 'veth2', self.ns2)
        veth1.link.set_up()
        veth1.addr.add('10.30.0.1/24')
        veth2.link.set_up()
        veth2.addr.add('10.30.0.2/24')

    @test_base.unstable_test("bug 1862927")
    def test_stop_process(self):
        self.nc.test_connectivity()
        server_pid = self.nc.server_process.child_pid
        client_pid = self.nc.client_process.child_pid
        self.assertTrue(utils.process_is_running(server_pid))
        self.assertTrue(utils.process_is_running(client_pid))

        self.nc.stop_processes()
        self.assertFalse(utils.process_is_running(server_pid))
        self.assertFalse(utils.process_is_running(client_pid))

    @test_base.unstable_test("bug 1862927")
    def test_stop_process_no_process(self):
        self.nc.test_connectivity()
        client_pid = self.nc.client_process.child_pid
        utils.execute(['kill', '-%d' % signal.SIGKILL, client_pid],
                      run_as_root=True)
        n_utils.wait_until_true(
            lambda: not utils.process_is_running(client_pid), timeout=5)
        with mock.patch.object(net_helpers.RootHelperProcess, 'poll',
                               return_value=None):
            self.assertRaises(n_exc.ProcessExecutionError,
                              self.nc.stop_processes,
                              skip_errors=[])

    @test_base.unstable_test("bug 1862927")
    def test_stop_process_no_process_skip_no_process_exception(self):
        self.nc.test_connectivity()
        server_pid = self.nc.server_process.child_pid
        utils.execute(['kill', '-%d' % signal.SIGKILL, server_pid],
                      run_as_root=True)

        with mock.patch.object(net_helpers.RootHelperProcess, 'poll',
                               side_effect=[None, True, None, True]):
            self.nc.stop_processes()
<|endoftext|>


<commit_before>class ServerInfo:

    # I have used the defaults specified by RFC1459 here
    SupportedUserModes = ['i', 'o', 's', 'w']
    SupportedChannelListModes = ['b']
    SupportedChannelSetArgsModes = ['l']
    SupportedChannelSetUnsetArgsModes = ['k']
    SupportedChannelNormalModes = ['i','m','n','p','s','t']

    SupportedStatuses = {'o': '@', 'v': '+'}
    SupportedStatusesReverse = {'@': 'o', '+': 'v'}
<commit_msg>I forgot Python can iterate strings<commit_after>class ServerInfo:

    # I have used the defaults specified by RFC1459 here
    SupportedUserModes = 'iosw'
    SupportedChannelListModes = 'b'
    SupportedChannelSetArgsModes = 'l'
    SupportedChannelSetUnsetArgsModes = 'k'
    SupportedChannelNormalModes = 'imnpst'

    SupportedStatuses = {'o': '@', 'v': '+'}
    SupportedStatusesReverse = {'@': 'o', '+': 'v'}
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
    jpegcleaner.py: tool to clean up folders containing invalid and duplicate jpeg images
    jpegcleaner.py is copyright 2020 Jeroen Doggen.
"""

import os
import argparse
import shutil

parser = argparse.ArgumentParser()
parser.add_argument("folder", help="the folder to clean")
args = parser.parse_args()

print("\nStorage space for the folder '" + args.folder + "' before cleanup:")
retVal = os.system('du -sh ' + args.folder)

print("\nRemoving duplicate files with fdupes")
retVal = os.system('fdupes -rdN ' + args.folder)

print("\nRemoving invalid JPEG files")
for subdir, dirs, files in os.walk(args.folder):
    for filename in files:
        filepath = subdir + os.sep + filename

        if filepath.endswith(".jpeg") or filename.endswith(".JPEG") or filename.endswith(".jpg") or filename.endswith(".JPG"):
            #print("JPEG file: " + filename)
            retVal = os.system('jpeginfo -c ' + "\""+ filepath + "\"") # returns the exit status

            if(0 != retVal):
                #os.remove(filepath)
                shutil.move(filepath, './temp/')
                print("Removed invalid JPEG file: " + filepath)

print("\nRemoving duplicate files with jpegdupes")
retVal = os.system('jpegdupes -d -a ' + args.folder)

print("\nStorage space for this folder after cleanup:")
retVal = os.system('du -sh ' + args.folder)
<commit_msg>jpegcleaner.py: show used storage space before & after<commit_after>#!/usr/bin/env python3

"""
    jpegcleaner.py: tool to clean up folders containing invalid and duplicate jpeg images
    jpegcleaner.py is copyright 2020 Jeroen Doggen.
"""

import os
import argparse
import shutil

parser = argparse.ArgumentParser()
parser.add_argument("folder", help="the folder to clean")
args = parser.parse_args()

print("\nStorage space for the folder '" + args.folder + "' before cleanup:")
retVal = os.system('du -sh ' + args.folder + " 2>&1 | tee ./jpegcleaner.log")

print("\nRemoving duplicate files with fdupes")
retVal = os.system('fdupes -rdN ' + args.folder)

print("\nRemoving invalid JPEG files")
for subdir, dirs, files in os.walk(args.folder):
    for filename in files:
        filepath = subdir + os.sep + filename

        if filepath.endswith(".jpeg") or filename.endswith(".JPEG") or filename.endswith(".jpg") or filename.endswith(".JPG"):
            #print("JPEG file: " + filename)
            retVal = os.system('jpeginfo -c ' + "\""+ filepath + "\"") # returns the exit status

            if(0 != retVal):
                #os.remove(filepath)
                shutil.move(filepath, './temp/')
                print("Removed invalid JPEG file: " + filepath)

print("\nRemoving duplicate files with jpegdupes")
retVal = os.system('jpegdupes -d -a ' + args.folder)

print("\nStorage space for this folder:")
print("\nBefore cleanup:")
retVal = os.system('cat ./jpegcleaner.log')
print("\nAfter cleanup:")
retVal = os.system('du -sh ' + args.folder + " 2>&1 | tee -a ./jpegcleaner.log")

<|endoftext|>


<commit_before>from django import template
from django.conf import settings
from django.core import urlresolvers
from django.template import Context, Template
from django.utils.translation import get_language, ugettext_lazy as _
from satchmo.configuration import config_value
from satchmo.product.models import Category
from satchmo.shop.templatetags import get_filter_args

register = template.Library()

def is_producttype(product, ptype):
    """Returns True if product is ptype"""
    if ptype in product.get_subtypes():
        return True
    else:
        return False

register.filter('is_producttype', is_producttype)

def product_images(product, args=""):
    args, kwargs = get_filter_args(args,
        keywords=('include_main', 'maximum'),
        boolargs=('include_main'),
        intargs=('maximum'),
        stripquotes=True)

    q = product.productimage_set
    if kwargs.get('include_main', True):
        q = q.all()
    else:
        main = product.main_image
        q = q.exclude(id = main.id)

    maximum = kwargs.get('maximum', -1)
    if maximum>-1:
        q = list(q)[:maximum]

    return q

register.filter('product_images', product_images)

def smart_attr(product, key):
    """Run the smart_attr function on the spec'd product
    """
    return product.smart_attr(key)

register.filter('smart_attr', smart_attr)

def product_sort_by_price(products):
    """Sort a product list by unit price"""

    fast = [(product.unit_price, product) for product in products]
    fast.sort()
    return zip(*fast)[1]

register.filter('product_sort_by_price', product_sort_by_price)
<commit_msg>Make sure product sorting by price does not error if there are no products. Closes #568<commit_after>from django import template
from django.conf import settings
from django.core import urlresolvers
from django.template import Context, Template
from django.utils.translation import get_language, ugettext_lazy as _
from satchmo.configuration import config_value
from satchmo.product.models import Category
from satchmo.shop.templatetags import get_filter_args

register = template.Library()

def is_producttype(product, ptype):
    """Returns True if product is ptype"""
    if ptype in product.get_subtypes():
        return True
    else:
        return False

register.filter('is_producttype', is_producttype)

def product_images(product, args=""):
    args, kwargs = get_filter_args(args,
        keywords=('include_main', 'maximum'),
        boolargs=('include_main'),
        intargs=('maximum'),
        stripquotes=True)

    q = product.productimage_set
    if kwargs.get('include_main', True):
        q = q.all()
    else:
        main = product.main_image
        q = q.exclude(id = main.id)

    maximum = kwargs.get('maximum', -1)
    if maximum>-1:
        q = list(q)[:maximum]

    return q

register.filter('product_images', product_images)

def smart_attr(product, key):
    """
    Run the smart_attr function on the spec'd product
    """
    return product.smart_attr(key)

register.filter('smart_attr', smart_attr)

def product_sort_by_price(products):
    """
    Sort a product list by unit price

    Example::

        {% for product in products|product_sort_by_price %}
    """

    if products:
        fast = [(product.unit_price, product) for product in products]
        fast.sort()
        return zip(*fast)[1]

register.filter('product_sort_by_price', product_sort_by_price)
<|endoftext|>


<commit_before>#!/usr/bin/python
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
DOCUMENTATION = '''
---
module: iam_policy
short_description: Manage IAM policies for users, groups, and roles
description:
     - Allows uploading or removing IAM policies for IAM users, groups or roles.
version_added: "2.0"
options:
  iam_type:
    description:
      - Type of IAM resource
    required: true
    default: null
    choices: [ "user", "group", "role"]
  iam_name:
    description:
      - Name of IAM resource you wish to target for policy actions. In other words, the user name, group name or role name.
    required: true
  policy_name:
    description:
      - The name label for the policy to create or remove.
    required: true
  policy_document:
    description:
      - The path to the properly json formatted policy file (mutually exclusive with C(policy_json))
    required: false
  policy_json:
    description:
      - A properly json formatted policy as string (mutually exclusive with C(policy_document), see https://github.com/ansible/ansible/issues/7005#issuecomment-42894813 on how to use it properly)
    required: false
  state:
    description:
      - Whether to create or delete the IAM policy.
    required: true
    default: null
    choices: [ "present", "absent"]
  skip_duplicates:
    description:
      - By default the module looks for any policies that match the document you pass in, if there is a match it will not make a new policy object with the same rules. You can override this by specifying false which would allow for two policy objects with different names but same rules.
    required: false
    default: "/"

notes:
  - 'Currently boto does not support the removal of Managed Policies, the module will not work removing/adding managed policies.'
author: "Jonathan I. Davila (@defionscode)"
extends_documentation_fragment: aws
'''

EXAMPLES = '''
# Create and policy with the name of 'Admin' to the group 'administrators'
tasks:
- name: Create two new IAM users with API keys
  iam_policy:
    iam_type: group
    iam_name: administrators
    policy_name: Admin
    state: present
    policy_document: admin_policy.json

# Advanced example, create two new groups and add a READ-ONLY policy to both
# groups.
task:
- name: Create Two Groups, Mario and Luigi
  iam:
    iam_type: group
    name: "{{ item }}"
    state: present
  with_items:
     - Mario
     - Luigi
  register: new_groups

- name:
  iam_policy:
    iam_type: group
    iam_name: "{{ item.created_group.group_name }}"
    policy_name: "READ-ONLY"
    policy_document: readonlypolicy.json
    state: present
  with_items: new_groups.results

# Create a new S3 policy with prefix per user
tasks:
- name: Create S3 policy from template
  iam_policy:
    iam_type: user
    iam_name: "{{ item.user }}"
    policy_name: "s3_limited_access_{{ item.prefix }}"
    state: present
    policy_json: " {{ lookup( 'template', 's3_policy.json.j2') }} "
    with_items:
      - user: s3_user
        prefix: s3_user_prefix

'''
import json
import urllib
try:
    import boto
    import boto.iam
    import boto.ec2
    HAS_BOTO = True
except ImportError:
    HAS_BOTO = False

def boto_exception(err):
    '''generic error message handler'''
    if hasattr(err, 'error_message'):
        error = err.error_message
    elif hasattr(err, 'message'):
        error = err.message
    else:
        error = '%s: %s' % (Exception, err)

    return error

def user_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  try:
    current_policies = [cp for cp in iam.get_all_user_policies(name).
                                        list_user_policies_result.
                                        policy_names]
    for pol in current_policies:
      '''
      urllib is needed here because boto returns url encoded strings instead
      '''
      if urllib.unquote(iam.get_user_policy(name, pol).
                        get_user_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_user_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_user_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_user_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed, msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.get_all_user_policies(name).
                                        list_user_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies

def role_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  try:
    current_policies = [cp for cp in iam.list_role_policies(name).
                                        list_role_policies_result.
                                        policy_names]
    for pol in current_policies:
      if urllib.unquote(iam.get_role_policy(name, pol).
                        get_role_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_role_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_role_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_role_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed,
                           msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.list_role_policies(name).
                                        list_role_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies

def group_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  msg=''
  try:
    current_policies = [cp for cp in iam.get_all_group_policies(name).
                                        list_group_policies_result.
                                        policy_names]
    for pol in current_policies:
      if urllib.unquote(iam.get_group_policy(name, pol).
                        get_group_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_group_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_group_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_group_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed,
                           msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.get_all_group_policies(name).
                                        list_group_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies, msg

def main():
  argument_spec = ec2_argument_spec()
  argument_spec.update(dict(
      iam_type=dict(
          default=None, required=True, choices=['user', 'group', 'role']),
      state=dict(
          default=None, required=True, choices=['present', 'absent']),
      iam_name=dict(default=None, required=False),
      policy_name=dict(default=None, required=True),
      policy_document=dict(default=None, required=False),
      policy_json=dict(default=None, required=False),
      skip_duplicates=dict(type='bool', default=True, required=False)
  ))

  module = AnsibleModule(
      argument_spec=argument_spec,
  )

  if not HAS_BOTO:
    module.fail_json(msg='boto required for this module')

  state = module.params.get('state').lower()
  iam_type = module.params.get('iam_type').lower()
  state = module.params.get('state')
  name = module.params.get('iam_name')
  policy_name = module.params.get('policy_name')
  skip = module.params.get('skip_duplicates')

  if module.params.get('policy_document') != None and module.params.get('policy_json') != None:
      module.fail_json(msg='Only one of "policy_document" or "policy_json" may be set')

  if module.params.get('policy_document') != None:
    with open(module.params.get('policy_document'), 'r') as json_data:
          pdoc = json.dumps(json.load(json_data))
          json_data.close()
  elif module.params.get('policy_json') != None:
      try:
        pdoc = json.dumps(module.params.get('policy_json'))
      except Exception as e:
        module.fail_json(msg=str(e) + '\n' + module.params.get('policy_json'))
  else:
    pdoc=None

  region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)

  try:
    if region:
        iam = boto.iam.connect_to_region(region, **aws_connect_kwargs)
    else:
        iam = boto.iam.connection.IAMConnection(**aws_connect_kwargs)
  except boto.exception.NoAuthHandlerFound, e:
      module.fail_json(msg=str(e))

  changed = False

  if iam_type == 'user':
    changed, user_name, current_policies = user_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, user_name=name, policies=current_policies)
  elif iam_type == 'role':
    changed, role_name, current_policies = role_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, role_name=name, policies=current_policies)
  elif iam_type == 'group':
    changed, group_name, current_policies, msg = group_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, group_name=name, policies=current_policies, msg=msg)

from ansible.module_utils.basic import *
from ansible.module_utils.ec2 import *

main()
<commit_msg>Allow iam_policy to maintain idempotentce if the role referenced has been removed<commit_after>#!/usr/bin/python
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
DOCUMENTATION = '''
---
module: iam_policy
short_description: Manage IAM policies for users, groups, and roles
description:
     - Allows uploading or removing IAM policies for IAM users, groups or roles.
version_added: "2.0"
options:
  iam_type:
    description:
      - Type of IAM resource
    required: true
    default: null
    choices: [ "user", "group", "role"]
  iam_name:
    description:
      - Name of IAM resource you wish to target for policy actions. In other words, the user name, group name or role name.
    required: true
  policy_name:
    description:
      - The name label for the policy to create or remove.
    required: true
  policy_document:
    description:
      - The path to the properly json formatted policy file (mutually exclusive with C(policy_json))
    required: false
  policy_json:
    description:
      - A properly json formatted policy as string (mutually exclusive with C(policy_document), see https://github.com/ansible/ansible/issues/7005#issuecomment-42894813 on how to use it properly)
    required: false
  state:
    description:
      - Whether to create or delete the IAM policy.
    required: true
    default: null
    choices: [ "present", "absent"]
  skip_duplicates:
    description:
      - By default the module looks for any policies that match the document you pass in, if there is a match it will not make a new policy object with the same rules. You can override this by specifying false which would allow for two policy objects with different names but same rules.
    required: false
    default: "/"

notes:
  - 'Currently boto does not support the removal of Managed Policies, the module will not work removing/adding managed policies.'
author: "Jonathan I. Davila (@defionscode)"
extends_documentation_fragment: aws
'''

EXAMPLES = '''
# Create and policy with the name of 'Admin' to the group 'administrators'
tasks:
- name: Create two new IAM users with API keys
  iam_policy:
    iam_type: group
    iam_name: administrators
    policy_name: Admin
    state: present
    policy_document: admin_policy.json

# Advanced example, create two new groups and add a READ-ONLY policy to both
# groups.
task:
- name: Create Two Groups, Mario and Luigi
  iam:
    iam_type: group
    name: "{{ item }}"
    state: present
  with_items:
     - Mario
     - Luigi
  register: new_groups

- name:
  iam_policy:
    iam_type: group
    iam_name: "{{ item.created_group.group_name }}"
    policy_name: "READ-ONLY"
    policy_document: readonlypolicy.json
    state: present
  with_items: new_groups.results

# Create a new S3 policy with prefix per user
tasks:
- name: Create S3 policy from template
  iam_policy:
    iam_type: user
    iam_name: "{{ item.user }}"
    policy_name: "s3_limited_access_{{ item.prefix }}"
    state: present
    policy_json: " {{ lookup( 'template', 's3_policy.json.j2') }} "
    with_items:
      - user: s3_user
        prefix: s3_user_prefix

'''
import json
import urllib
try:
    import boto
    import boto.iam
    import boto.ec2
    HAS_BOTO = True
except ImportError:
    HAS_BOTO = False

def boto_exception(err):
    '''generic error message handler'''
    if hasattr(err, 'error_message'):
        error = err.error_message
    elif hasattr(err, 'message'):
        error = err.message
    else:
        error = '%s: %s' % (Exception, err)

    return error

def user_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  try:
    current_policies = [cp for cp in iam.get_all_user_policies(name).
                                        list_user_policies_result.
                                        policy_names]
    for pol in current_policies:
      '''
      urllib is needed here because boto returns url encoded strings instead
      '''
      if urllib.unquote(iam.get_user_policy(name, pol).
                        get_user_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_user_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_user_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_user_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed, msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.get_all_user_policies(name).
                                        list_user_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies

def role_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  try:
    current_policies = [cp for cp in iam.list_role_policies(name).
                                        list_role_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError as e:
    if e.error_code == "NoSuchEntity":
      # Role doesn't exist so it's safe to assume the policy doesn't either
      module.exit_json(changed=False)
    else:
      module.fail_json(e.message)

  try:
    for pol in current_policies:
      if urllib.unquote(iam.get_role_policy(name, pol).
                        get_role_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_role_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_role_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_role_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed,
                           msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.list_role_policies(name).
                                        list_role_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies

def group_action(module, iam, name, policy_name, skip, pdoc, state):
  policy_match = False
  changed = False
  msg=''
  try:
    current_policies = [cp for cp in iam.get_all_group_policies(name).
                                        list_group_policies_result.
                                        policy_names]
    for pol in current_policies:
      if urllib.unquote(iam.get_group_policy(name, pol).
                        get_group_policy_result.policy_document) == pdoc:
        policy_match = True
        if policy_match:
          msg=("The policy document you specified already exists "
               "under the name %s." % pol)
    if state == 'present' and skip:
      if policy_name not in current_policies and not policy_match:
        changed = True
        iam.put_group_policy(name, policy_name, pdoc)
    elif state == 'present' and not skip:
        changed = True
        iam.put_group_policy(name, policy_name, pdoc)
    elif state == 'absent':
      try:
        iam.delete_group_policy(name, policy_name)
        changed = True
      except boto.exception.BotoServerError, err:
        error_msg = boto_exception(err)
        if 'cannot be found.' in error_msg:
          changed = False
          module.exit_json(changed=changed,
                           msg="%s policy is already absent" % policy_name)

    updated_policies = [cp for cp in iam.get_all_group_policies(name).
                                        list_group_policies_result.
                                        policy_names]
  except boto.exception.BotoServerError, err:
    error_msg = boto_exception(err)
    module.fail_json(changed=changed, msg=error_msg)

  return changed, name, updated_policies, msg

def main():
  argument_spec = ec2_argument_spec()
  argument_spec.update(dict(
      iam_type=dict(
          default=None, required=True, choices=['user', 'group', 'role']),
      state=dict(
          default=None, required=True, choices=['present', 'absent']),
      iam_name=dict(default=None, required=False),
      policy_name=dict(default=None, required=True),
      policy_document=dict(default=None, required=False),
      policy_json=dict(default=None, required=False),
      skip_duplicates=dict(type='bool', default=True, required=False)
  ))

  module = AnsibleModule(
      argument_spec=argument_spec,
  )

  if not HAS_BOTO:
    module.fail_json(msg='boto required for this module')

  state = module.params.get('state').lower()
  iam_type = module.params.get('iam_type').lower()
  state = module.params.get('state')
  name = module.params.get('iam_name')
  policy_name = module.params.get('policy_name')
  skip = module.params.get('skip_duplicates')

  if module.params.get('policy_document') != None and module.params.get('policy_json') != None:
      module.fail_json(msg='Only one of "policy_document" or "policy_json" may be set')

  if module.params.get('policy_document') != None:
    with open(module.params.get('policy_document'), 'r') as json_data:
          pdoc = json.dumps(json.load(json_data))
          json_data.close()
  elif module.params.get('policy_json') != None:
      try:
        pdoc = json.dumps(module.params.get('policy_json'))
      except Exception as e:
        module.fail_json(msg=str(e) + '\n' + module.params.get('policy_json'))
  else:
    pdoc=None

  region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module)

  try:
    if region:
        iam = boto.iam.connect_to_region(region, **aws_connect_kwargs)
    else:
        iam = boto.iam.connection.IAMConnection(**aws_connect_kwargs)
  except boto.exception.NoAuthHandlerFound, e:
      module.fail_json(msg=str(e))

  changed = False

  if iam_type == 'user':
    changed, user_name, current_policies = user_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, user_name=name, policies=current_policies)
  elif iam_type == 'role':
    changed, role_name, current_policies = role_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, role_name=name, policies=current_policies)
  elif iam_type == 'group':
    changed, group_name, current_policies, msg = group_action(module, iam, name,
                                                       policy_name, skip, pdoc,
                                                       state)
    module.exit_json(changed=changed, group_name=name, policies=current_policies, msg=msg)

from ansible.module_utils.basic import *
from ansible.module_utils.ec2 import *

main()
<|endoftext|>


<commit_before>"""VCD Value tracker."""

import re
from typing import Tuple, Optional, Union
from hdltools.vcd import VCDScope
from hdltools.vcd.parser import BaseVCDParser, VCDParserError
from hdltools.vcd.mixins import VCDTriggerMixin
from hdltools.patterns import Pattern
from hdltools.vcd.trigger import VCDTriggerDescriptor
from hdltools.vcd.history import VCDValueHistory, VCDValueHistoryEntry

# TODO: multi value tracker
class VCDValueTracker(BaseVCDParser, VCDTriggerMixin):
    """VCD Value tracker.

    Track a tagged value through system hierarchy.
    """

    def __init__(
        self,
        track: Pattern,
        restrict_src: Optional[Union[Tuple[str], VCDScope]] = None,
        restrict_dest: Optional[Union[Tuple[str], VCDScope]] = None,
        inclusive_src=False,
        inclusive_dest=False,
        ignore_signals: Optional[Tuple[str]] = None,
        ignore_scopes: Optional[Tuple[str]] = None,
        anchors: Optional[Tuple[str, str]] = None,
        preconditions: Optional[Tuple[VCDTriggerDescriptor]] = None,
        postconditions: Optional[Tuple[VCDTriggerDescriptor]] = None,
        time_range: Optional[Tuple[int, int]] = None,
        track_all: bool = False,
    ):
        """Initialize."""
        super().__init__()
        if not isinstance(track, Pattern):
            raise TypeError("track must be a Pattern object")
        self._track_value = track
        self._track_all = track_all
        self._track_history = VCDValueHistory()
        self._complete_value_history = VCDValueHistory()
        self._full_history = VCDValueHistory()
        self._stmt_count = 0
        if isinstance(restrict_src, tuple):
            restrict_src = VCDScope(*restrict_src)
        if isinstance(restrict_dest, tuple):
            restrict_dest = VCDScope(*restrict_dest)
        self._restrict_src = restrict_src
        self._restrict_dest = restrict_dest
        self._inclusive_src = inclusive_src
        self._inclusive_dest = inclusive_dest
        if ignore_signals is not None:
            self._ignore_sig = [re.compile(ign) for ign in ignore_signals]
        else:
            self._ignore_sig = None
        if ignore_scopes is not None:
            self._ignore_scope = [re.compile(ign) for ign in ignore_scopes]
        else:
            self._ignore_scope = None

        # source, destination anchors
        src_anchor, dest_anchor = (
            anchors if anchors is not None else (None, None)
        )
        self._src_anchor = (
            re.compile(src_anchor) if src_anchor is not None else None
        )
        self._dest_anchor = (
            re.compile(dest_anchor) if dest_anchor is not None else None
        )
        self._maybe_src = None
        self._maybe_dest = None
        self._wait_precondition = False
        self._wait_postcondition = False
        self._postconditions = []

        if time_range is not None:
            if not isinstance(time_range, (tuple, list)):
                raise TypeError("time_range must be a list")
            if len(time_range) != 2:
                raise ValueError("time_range must have 2 elements exactly")
            start, end = time_range
            if not isinstance(start, int) or not isinstance(end, int):
                raise TypeError("start and end in time_range must be integers")
            self._hist_start = start
            self._hist_end = end
        else:
            self._hist_start = None
            self._hist_end = None

        if postconditions is not None:
            if isinstance(postconditions, VCDTriggerDescriptor):
                self._postconditions.append(postconditions)
            else:
                for postcondition in postconditions:
                    if not isinstance(postcondition, VCDTriggerDescriptor):
                        raise TypeError(
                            "precondition must be a VCDTriggerDescriptor object"
                        )
                    self._postconditions.append(postcondition)
        if preconditions is not None:
            self._wait_precondition = True
            if isinstance(preconditions, VCDTriggerDescriptor):
                self.add_trigger_level(preconditions)
            else:
                for precondition in preconditions:
                    if not isinstance(precondition, VCDTriggerDescriptor):
                        raise TypeError(
                            "precondition must be a VCDTriggerDescriptor object"
                        )
                    self.add_trigger_level(precondition)

            # arm trigger
            self.trigger_callback = self._precondition_callback
            self.arm_trigger()

    # FIXME: this is a placeholder
    def _precondition_callback(self):
        """Precondition callback."""
        print("DEBUG: reached preconditions")
        self._wait_precondition = False
        # setup postconditions
        if self._postconditions:
            self._wait_postcondition = True
            self.trigger_reset()
            for postcondition in self._postconditions:
                self.add_trigger_level(postcondition)

            self.trigger_callback = self._postcondition_callback
            self.arm_trigger()

    def _postcondition_callback(self):
        """Postcondition callback."""
        print("DEBUG: reached postconditions; stopping")
        self._wait_postcondition = False
        self._abort_parser()

    def _parse_progress(self):
        """Track parsing progress."""
        self._stmt_count += 1
        # print("DEBUG: #lines = {}".format(self._current_line))

    def header_statement_handler(self, stmt, fields):
        """Handle header statements.

        Build variable-scope map.
        """
        self._parse_progress()

    def initial_value_handler(self, stmt, fields):
        """Handle initial values."""
        self._parse_progress()
        var = self.variables[fields["var"]]
        if self._track_all:
            self._add_to_history(var.scope, var.name, 0)
        if self._track_value.match(fields["value"]):
            # found
            self._add_to_tracked_history(var.scope, var.name, 0)
            self._add_to_value_history(var.scope, var.name, 0)

    def value_change_handler(self, stmt, fields):
        """Handle value change."""
        self._parse_progress()
        var = self.variables[fields["var"]]
        if self._track_all:
            self._add_to_history(var.scope, var.name, self.current_time)
        # handle start time
        if self._track_value.match(fields["value"]):
            # add to complete tracking history
            self._add_to_value_history(var.scope, var.name, self.current_time)
        if (
            self._hist_start is not None
            and self.current_time < self._hist_start
        ):
            return
        if self._hist_end is not None and self.current_time > self._hist_end:
            # done
            self._abort_parser()
        if self._wait_precondition and self._triggered is False:
            return
        var_scope = var.scope
        in_src_scope = self._restrict_src is not None and (
            var_scope == self._restrict_src
            or (self._restrict_src.contains(var_scope) and self._inclusive_src)
        )

        in_dest_scope = self._restrict_dest is not None and (
            var_scope == self._restrict_dest
            or (
                self._restrict_dest.contains(var_scope)
                and self._inclusive_dest
            )
        )
        if (self._restrict_src is not None and not in_src_scope) and (
            self._restrict_dest is not None and not in_dest_scope
        ):
            return
        if self._track_value.match(fields["value"]):
            if self._ignore_sig is not None:
                for patt in self._ignore_sig:
                    if (
                        patt.match(self.variables[fields["var"]].name)
                        is not None
                    ):
                        # ignore
                        return
            print("DEBUG: current time = {}".format(self.current_time))
            print(
                "DEBUG: found value match, variable is {}".format(
                    self.variables[fields["var"]]
                )
            )
            if fields["var"] not in self.variables:
                raise VCDParserError(
                    'unknown variable in change event: "{}"'.format(
                        fields["var"]
                    )
                )
            idx = self._add_to_tracked_history(
                var.scope, var.name, self.current_time
            )
            # FIXME: make sure that anchors are not used without scope restriction
            if in_src_scope:
                if self._src_anchor is not None and self._maybe_dest is None:
                    if (
                        self._src_anchor.match(
                            self.variables[fields["var"]].name
                        )
                        is not None
                    ):
                        # new probable source
                        self._maybe_src = idx
                elif self._maybe_dest is None:
                    # anything that appears is probable source
                    self._maybe_src = idx

            if in_dest_scope:
                if self._maybe_dest is None and self._maybe_src is not None:
                    if (
                        self._dest_anchor is not None
                        and self._dest_anchor.match(
                            self.variables[fields["var"]].name
                        )
                        is not None
                    ):
                        self._maybe_dest = idx
                    elif self._dest_anchor is None:
                        self._maybe_dest = idx

    @property
    def history(self):
        """Get tracking history."""
        return self._track_history

    @property
    def full_history(self):
        """Get full history if available."""
        if self._track_all:
            return self._full_history
        return self.history

    @property
    def maybe_src(self):
        """Get probable source."""
        return self._maybe_src

    @property
    def maybe_dest(self):
        """Get probable destination."""
        return self._maybe_dest

    def _add_to_value_history(self, scope, signal, time):
        """Add to value history."""
        self._complete_value_history.add_entry(
            VCDValueHistory(scope, signal, time)
        )
        return len(self._full_history - 1)

    def _add_to_history(self, scope, signal, time):
        """Add to history."""
        self._full_history.add_entry(VCDValueHistoryEntry(scope, signal, time))
        return len(self._full_history) - 1

    def _add_to_tracked_history(self, scope, signal, time):
        """Add to history."""
        self._track_history.add_entry(
            VCDValueHistoryEntry(scope, signal, time)
        )
        return len(self._track_history) - 1

    def parse(self, data):
        """Parse."""
        self._stmt_count = 0
        return super().parse(data)
<commit_msg>get history<commit_after>"""VCD Value tracker."""

import re
from typing import Tuple, Optional, Union
from hdltools.vcd import VCDScope
from hdltools.vcd.parser import BaseVCDParser, VCDParserError
from hdltools.vcd.mixins import VCDTriggerMixin
from hdltools.patterns import Pattern
from hdltools.vcd.trigger import VCDTriggerDescriptor
from hdltools.vcd.history import VCDValueHistory, VCDValueHistoryEntry

# TODO: multi value tracker
# FIXME: avoid expensive duplications of data
class VCDValueTracker(BaseVCDParser, VCDTriggerMixin):
    """VCD Value tracker.

    Track a tagged value through system hierarchy.
    """

    def __init__(
        self,
        track: Pattern,
        restrict_src: Optional[Union[Tuple[str], VCDScope]] = None,
        restrict_dest: Optional[Union[Tuple[str], VCDScope]] = None,
        inclusive_src=False,
        inclusive_dest=False,
        ignore_signals: Optional[Tuple[str]] = None,
        ignore_scopes: Optional[Tuple[str]] = None,
        anchors: Optional[Tuple[str, str]] = None,
        preconditions: Optional[Tuple[VCDTriggerDescriptor]] = None,
        postconditions: Optional[Tuple[VCDTriggerDescriptor]] = None,
        time_range: Optional[Tuple[int, int]] = None,
        track_all: bool = False,
    ):
        """Initialize."""
        super().__init__()
        if not isinstance(track, Pattern):
            raise TypeError("track must be a Pattern object")
        self._track_value = track
        self._track_all = track_all
        self._track_history = VCDValueHistory()
        self._complete_value_history = VCDValueHistory()
        self._full_history = VCDValueHistory()
        self._stmt_count = 0
        if isinstance(restrict_src, tuple):
            restrict_src = VCDScope(*restrict_src)
        if isinstance(restrict_dest, tuple):
            restrict_dest = VCDScope(*restrict_dest)
        self._restrict_src = restrict_src
        self._restrict_dest = restrict_dest
        self._inclusive_src = inclusive_src
        self._inclusive_dest = inclusive_dest
        if ignore_signals is not None:
            self._ignore_sig = [re.compile(ign) for ign in ignore_signals]
        else:
            self._ignore_sig = None
        if ignore_scopes is not None:
            self._ignore_scope = [re.compile(ign) for ign in ignore_scopes]
        else:
            self._ignore_scope = None

        # source, destination anchors
        src_anchor, dest_anchor = (
            anchors if anchors is not None else (None, None)
        )
        self._src_anchor = (
            re.compile(src_anchor) if src_anchor is not None else None
        )
        self._dest_anchor = (
            re.compile(dest_anchor) if dest_anchor is not None else None
        )
        self._maybe_src = None
        self._maybe_dest = None
        self._wait_precondition = False
        self._wait_postcondition = False
        self._postconditions = []

        if time_range is not None:
            if not isinstance(time_range, (tuple, list)):
                raise TypeError("time_range must be a list")
            if len(time_range) != 2:
                raise ValueError("time_range must have 2 elements exactly")
            start, end = time_range
            if not isinstance(start, int) or not isinstance(end, int):
                raise TypeError("start and end in time_range must be integers")
            self._hist_start = start
            self._hist_end = end
        else:
            self._hist_start = None
            self._hist_end = None

        if postconditions is not None:
            if isinstance(postconditions, VCDTriggerDescriptor):
                self._postconditions.append(postconditions)
            else:
                for postcondition in postconditions:
                    if not isinstance(postcondition, VCDTriggerDescriptor):
                        raise TypeError(
                            "precondition must be a VCDTriggerDescriptor object"
                        )
                    self._postconditions.append(postcondition)
        if preconditions is not None:
            self._wait_precondition = True
            if isinstance(preconditions, VCDTriggerDescriptor):
                self.add_trigger_level(preconditions)
            else:
                for precondition in preconditions:
                    if not isinstance(precondition, VCDTriggerDescriptor):
                        raise TypeError(
                            "precondition must be a VCDTriggerDescriptor object"
                        )
                    self.add_trigger_level(precondition)

            # arm trigger
            self.trigger_callback = self._precondition_callback
            self.arm_trigger()

    # FIXME: this is a placeholder
    def _precondition_callback(self):
        """Precondition callback."""
        print("DEBUG: reached preconditions")
        self._wait_precondition = False
        # setup postconditions
        if self._postconditions:
            self._wait_postcondition = True
            self.trigger_reset()
            for postcondition in self._postconditions:
                self.add_trigger_level(postcondition)

            self.trigger_callback = self._postcondition_callback
            self.arm_trigger()

    def _postcondition_callback(self):
        """Postcondition callback."""
        print("DEBUG: reached postconditions; stopping")
        self._wait_postcondition = False
        self._abort_parser()

    def _parse_progress(self):
        """Track parsing progress."""
        self._stmt_count += 1
        # print("DEBUG: #lines = {}".format(self._current_line))

    def header_statement_handler(self, stmt, fields):
        """Handle header statements.

        Build variable-scope map.
        """
        self._parse_progress()

    def initial_value_handler(self, stmt, fields):
        """Handle initial values."""
        self._parse_progress()
        var = self.variables[fields["var"]]
        if self._track_all:
            self._add_to_history(var.scope, var.name, 0)
        if self._track_value.match(fields["value"]):
            # found
            self._add_to_tracked_history(var.scope, var.name, 0)
            self._add_to_value_history(var.scope, var.name, 0)

    def value_change_handler(self, stmt, fields):
        """Handle value change."""
        self._parse_progress()
        var = self.variables[fields["var"]]
        if self._track_all:
            self._add_to_history(var.scope, var.name, self.current_time)
        # handle start time
        if self._track_value.match(fields["value"]):
            # add to complete tracking history
            self._add_to_value_history(var.scope, var.name, self.current_time)
        if (
            self._hist_start is not None
            and self.current_time < self._hist_start
        ):
            return
        if self._hist_end is not None and self.current_time > self._hist_end:
            # done
            self._abort_parser()
        if self._wait_precondition and self._triggered is False:
            return
        var_scope = var.scope
        in_src_scope = self._restrict_src is not None and (
            var_scope == self._restrict_src
            or (self._restrict_src.contains(var_scope) and self._inclusive_src)
        )

        in_dest_scope = self._restrict_dest is not None and (
            var_scope == self._restrict_dest
            or (
                self._restrict_dest.contains(var_scope)
                and self._inclusive_dest
            )
        )
        if (self._restrict_src is not None and not in_src_scope) and (
            self._restrict_dest is not None and not in_dest_scope
        ):
            return
        if self._track_value.match(fields["value"]):
            if self._ignore_sig is not None:
                for patt in self._ignore_sig:
                    if (
                        patt.match(self.variables[fields["var"]].name)
                        is not None
                    ):
                        # ignore
                        return
            print("DEBUG: current time = {}".format(self.current_time))
            print(
                "DEBUG: found value match, variable is {}".format(
                    self.variables[fields["var"]]
                )
            )
            if fields["var"] not in self.variables:
                raise VCDParserError(
                    'unknown variable in change event: "{}"'.format(
                        fields["var"]
                    )
                )
            idx = self._add_to_tracked_history(
                var.scope, var.name, self.current_time
            )
            # FIXME: make sure that anchors are not used without scope restriction
            if in_src_scope:
                if self._src_anchor is not None and self._maybe_dest is None:
                    if (
                        self._src_anchor.match(
                            self.variables[fields["var"]].name
                        )
                        is not None
                    ):
                        # new probable source
                        self._maybe_src = idx
                elif self._maybe_dest is None:
                    # anything that appears is probable source
                    self._maybe_src = idx

            if in_dest_scope:
                if self._maybe_dest is None and self._maybe_src is not None:
                    if (
                        self._dest_anchor is not None
                        and self._dest_anchor.match(
                            self.variables[fields["var"]].name
                        )
                        is not None
                    ):
                        self._maybe_dest = idx
                    elif self._dest_anchor is None:
                        self._maybe_dest = idx

    @property
    def history(self):
        """Get tracking history."""
        return self._track_history

    @property
    def full_history(self):
        """Get full history if available."""
        if self._track_all:
            return self._full_history
        return self.history

    @property
    def full_value_history(self):
        """Get full value match history."""
        return self._complete_value_history

    @property
    def maybe_src(self):
        """Get probable source."""
        return self._maybe_src

    @property
    def maybe_dest(self):
        """Get probable destination."""
        return self._maybe_dest

    def _add_to_value_history(self, scope, signal, time):
        """Add to value history."""
        self._complete_value_history.add_entry(
            VCDValueHistory(scope, signal, time)
        )
        return len(self._full_history - 1)

    def _add_to_history(self, scope, signal, time):
        """Add to history."""
        self._full_history.add_entry(VCDValueHistoryEntry(scope, signal, time))
        return len(self._full_history) - 1

    def _add_to_tracked_history(self, scope, signal, time):
        """Add to history."""
        self._track_history.add_entry(
            VCDValueHistoryEntry(scope, signal, time)
        )
        return len(self._track_history) - 1

    def parse(self, data):
        """Parse."""
        self._stmt_count = 0
        return super().parse(data)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

"""
Poisson point process penalised likelihood regression.
"""

try:
    import autograd
    import autograd.numpy as np
    import autograd.scipy as sp
    have_autograd = True
except ImportError as e:
    import numpy as np
    import scipy as sp
    have_autograd = False

from . import influence
from . import model

class ContinuousExact(object):
    def __init__(
            self,
            debug=False
            ):
        self.debug = debug

    def _debug_print(self, *args, **kwargs):
        if self.debug:
            print(*args, **kwargs)

    def _pack(
            self,
            mu=1.0,
            kappa=0.0,
            tau=None,
            omega=None
            ):
        """
        pack all params into one vector for blackbox optimisation;
        currently unused
        """

        packed = np.zeros(
            1 +
            self.n_phi_bases +
            self.n_tau +
            self.n_omega
        )
        packed[0] = mu
        packed[
            1:self.n_phi_bases + 1
        ] = kappa
        packed[
            self.n_phi_bases + 1:
            self.n_phi_bases + self.n_tau + 1
        ] = tau if tau is not None else 1.0
        packed[
            self.n_phi_bases + self.n_tau + 1:
        ] = omega if omega is not None else 0.0
        return packed

    def _unpack(
            self,
            packed):
        """
        unpack all params from one vector for blackbox optimisation;
        currently unused.
        """
        unpacked = dict(
            mu=packed[0],
            kappa=packed[1:self.n_phi_bases+1],
        )
        # are we fitting time parameters?
        if self.n_tau > 0:
            unpacked = packed[
                self.n_phi_bases + 1:
                self.n_phi_bases + self.n_tau + 1
            ]
        # omega parameters - exogenous variation
        if self.n_omega > 0:
            unpacked = packed[
                self.n_phi_bases + self.n_tau + 1:
            ]
        return unpacked

    def negloglik(
            self,
            mu=1.0,
            kappa=0.0,
            tau=None,
            omega=None,
            **kwargs):

        return -model.loglik(
            ts=self._ts,
            eval_ts=self._eval_ts,
            mu=mu,
            phi_kernel=self.phi_kernel,
            mu_kernel=self.mu_kernel,
            **kwargs
        )

    def _penalty_weight_packed(
            self,
            pi_kappa=0.0,
            pi_omega=1e-8,
            **kwargs
            ):
        return self._pack(
            mu=0.0,
            kappa=pi_kappa,
            omega=pi_omega
        )  # / self.penalty_scale

    def penalty(
            self,
            mu=1.0,
            kappa=0.0,
            tau=1.0,
            omega=0.0,
            pi_kappa=0.0,
            pi_omega=0.0,
            **kwargs
            ):
        # 2 different l_1 penalties
        return np.sum(
            np.abs(kappa) * pi_kappa
        ) + np.sum(
            np.abs(omega) * pi_omega
        )

    def objective(
            self,
            mu=1.0,
            kappa=0.0,
            tau=None,
            omega=None,
            pi_kappa=0.0,
            pi_omega=1e-8,
            **kwargs):

        g_negloglik = self.negloglik(
            mu=mu,
            kappa=kappa,
            tau=tau,
            omega=omega,
            pi_kappa=pi_kappa,
            pi_omega=pi_omega,
            **kwargs
        )
        g_penalty = self.penalty(
            mu=mu,
            kappa=kappa,
            tau=tau,
            omega=omega,
            pi_kappa=pi_kappa,
            pi_omega=pi_omega,
            **kwargs
        )
        penalty_dominated = np.abs(
            g_negloglik
        ) < (
            g_penalty
        )
        return (g_negloglik + g_penalty) * penalty_dominated

    def _setup_graphs(
            self,
            ts,
            phi_kernel=None,
            mu_kernel=None,
            n_phi_bases=1,
            n_mu_bases=0,
            fit_tau=False,
            fit_omega=False,
            t_start=0.0,
            t_end=None,
            param_vector=None,):

        self._t_start = t_start
        self._t_end = t_end or ts[-1]
        self._ts = ts
        # Full data likelihood must evaluate at the t_end also
        if ts[-1] < self._t_end:
            _eval_ts = np.append(
                ts[ts > self._t_start],
                [self._t_end]
            )
        else:
            _eval_ts = ts[
                np.logical_and(
                    ts >= self._t_start,
                    ts < self._t_end
                )
            ]

        self._eval_ts = _eval_ts

        if phi_kernel is None:
            phi_kernel = influence.MaxwellKernel(n_bases=5)
        else:
            phi_kernel = influence.as_influence_kernel(
                phi_kernel, n_bases=n_phi_bases
            )
        self.phi_kernel = phi_kernel
        self.n_phi_bases = phi_kernel.n_bases
        if mu_kernel is None:
            if n_mu_bases > 0:
                mu_kernel = influence.LinearStepKernel(
                    t_start=self._t_start,
                    t_end=self._t_end,
                    n_bases=n_mu_bases)
            else:
                mu_kernel = influence.ConstKernel()
        self.mu_kernel = mu_kernel
        self.n_mu_bases = n_mu_bases

        self._fit_tau = fit_tau
        self._fit_omega = fit_omega
        self.n_tau = self.n_phi_bases * self._fit_tau
        self.n_omega = self.n_mu_bases * self._fit_omega

        if param_vector is None:
            param_vector = self._pack()
        self._param_vector = param_vector

        self._param_floor = self._pack(
            mu=0.0,  # unused?
            mu_kwargs=dict(
                kappa=0.0
            ),
            phi_kwargs=dict(
                kappa=0.0
            )
        )
        self._grad_mu = autograd.grad(
            self._negloglik_packed, 0)
        self._grad_kappa = autograd.grad(
            self._negloglik_packed, 1)
        self._grad_tau = autograd.grad(
            self._negloglik_packed, 2)
        self._grad_omega = autograd.grad(
            self._negloglik_packed, 3)
        # self._hessian_diag_negloglik = autograd.elementwise_grad(
        #     self._grad_negloglik, 0)

    def _teardown_graphs(
            self):

        del(self._ts)
        del(self._eval_ts)

    def fit(
            self,
            ts,
            phi_kernel=None,
            mu_kernel=None,
            n_phi_bases=1,
            n_mu_bases=0,
            fit_tau=False,
            fit_omega=False,
            t_start=0.0,
            t_end=None,
            **kwargs
            ):
        self._setup_graphs(
            ts=ts,
            phi_kernel=phi_kernel,
            mu_kernel=mu_kernel,
            n_phi_bases=n_phi_bases,
            n_mu_bases=n_mu_bases,
            fit_tau=fit_tau,
            fit_omega=fit_omega,
            **kwargs
        )

        param_kwargs = dict()
        for k in [
            'mu', 'tau', 'kappa', 'omega', 'pi_kappa', 'pi_omega'
        ]:
            v = kwargs.get(k, None)
            if v is not None:
                param_kwargs[k] = v
        return self._fit(
            **param_kwargs,
            **kwargs
        )

    def _fit(
            self,
            mu=1.0,
            kappa=0.0,
            tau=None,
            omega=None,
            pi_kappa=0.0,
            pi_omega=1e-8,
            max_steps=10,
            eps=1e-8,
            **kwargs
            ):
        """
        fit by Truncated Newton in each coordinate
        """

        for i in range(max_steps):
            self._debug_print(i, 'param', mu, tau, kappa, omega, 'grad', g)
            res = scipy.minimize(

    # def _fit_path(
    #         self,
    #         param_vector=None,
    #         pi_kappa=0.0,
    #         pi_omega=1e-8,
    #         max_steps=50,
    #         step_iter=20,
    #         lr=0.1,
    #         gamma=0.9,
    #         eps=1e-8,
    #         backoff=0.75,
    #         warm_start=False,
    #         **kwargs
    #         ):
    #
    #     n_params = param_vector.size
    #     param_path = np.zeros((n_params, max_steps))
    #     pi_kappa_path = np.zeros(max_steps)
    #     pi_omega_path = np.zeros(max_steps)
    #     loglik_path = np.zeros(max_steps)
    #     dof_path = np.zeros(max_steps)
    #     aic_path = np.zeros(max_steps)
    #
    #     self._avg_sq_grad = avg_sq_grad = np.ones_like(param_vector)
    #
    #     for j in range(max_steps):
    #         loss = self._objective_packed(param_vector)
    #         best_loss = loss
    #         local_lr = lr
    #         best_param_vector = np.array(param_vector)
    #
    #         for i in range(step_iter):
    #             g_negloglik = self._grad_negloglik(param_vector)
    #             g_penalty = self._grad_penalty(param_vector, pi_kappa, pi_omega)
    #             g = g_negloglik + g_penalty
    #             self._debug_print(j, i, 'param', param_vector, 'grad', g)
    #             avg_sq_grad[:] = avg_sq_grad * gamma + g**2 * (1 - gamma)
    #
    #             velocity = lr * g * (
    #                     np.sqrt(avg_sq_grad) + eps
    #             ) / (0.1 * (i + j + 10.0))
    #             # watch out, nans
    #             velocity[np.logical_not(np.isfinite(velocity))] = 0.0
    #
    #             penalty_dominant = np.abs(
    #                 g_negloglik
    #             ) < (
    #                 self._penalty_weight_packed(pi_kappa, pi_omega)
    #             )
    #             velocity[penalty_dominant * (velocity == 0)] = 0.0
    #             new_param_vector = param_vector - velocity * local_lr
    #             # coefficients that pass through 0 must stop there
    #             new_param_vector[
    #                 np.abs(
    #                     np.sign(new_param_vector) -
    #                     np.sign(param_vector)
    #                 ) == 2
    #             ] = 0.0
    #             new_param_vector[:] = np.maximum(new_param_vector, self._param_floor)
    #             new_loss = self._objective_packed(new_param_vector)
    #             if new_loss < loss:
    #                 # print('good', loss, '=>', new_loss, local_lr)
    #                 loss = new_loss
    #                 param_vector = new_param_vector
    #                 self._param_vector = new_param_vector
    #             else:
    #                 # print('bad', loss, '=>', new_loss, local_lr)
    #                 local_lr = local_lr * backoff
    #                 new_param_vector = param_vector + backoff * (
    #                     new_param_vector - param_vector
    #                 )
    #                 loss = self._objective_packed(new_param_vector)
    #             if loss < best_loss:
    #                 best_param_vector = np.array(param_vector)
    #                 best_loss = loss
    #
    #             if local_lr < 1e-3:
    #                 self._debug_print('nope', j, i, max_steps)
    #                 break
    #
    #         this_loglik = -self._negloglik_packed(best_param_vector)
    #         this_dof = self._dof_packed(best_param_vector)
    #         param_path[:, j] = best_param_vector
    #         pi_kappa_path[j] = pi_kappa
    #         pi_omega_path[j] = pi_omega
    #         loglik_path[j] = this_loglik
    #         dof_path[j] = this_dof
    #         aic_path[j] = 2 * this_loglik - 2 * this_dof
    #
    #         # # regularisation parameter selection
    #         # all_grad = self._unpack(
    #         #     np.abs(
    #         #         grad_objective(best_param_vector) *
    #         #         (best_param_vector != 0.0)
    #         #     )
    #         # )
    #         # kappa_grad = all_grad['kappa']
    #         # log_omega_grad = all_grad.get('omega', None)
    #         #
    #         # if (
    #         #     np.random.random() < (
    #         #         np.sqrt(log_omega_grad.size) /
    #         #         (np.sqrt(kappa_grad.size) +
    #         #         np.sqrt(log_omega_grad.size))
    #         #         )):
    #         #     print('log_omega_grad', log_omega_grad)
    #         #     pi_omega += max(
    #         #         np.amin(log_omega_grad[log_omega_grad > 0])
    #         #         * j/max_steps,
    #         #         pi_omega * 0.1
    #         #     )
    #         # else:
    #         #     print('kappa_grad', kappa_grad)
    #         #     pi_kappa += max(
    #         #         np.amin(kappa_grad[kappa_grad > 0]) * j / max_steps,
    #         #         pi_kappa * 0.1
    #         #     )
    #     return FitHistory(
    #         param_path=param_path[:, :j],
    #         pi_kappa_path=pi_kappa_path[:j],
    #         pi_omega_path=pi_omega_path[:j],
    #         loglik_path=loglik_path[:j],
    #         dof_path=dof_path[:j],
    #         aic_path=aic_path[:j],
    #         unpack=lambda v: self._unpack(v),
    #         param_vector=best_param_vector,
    #         param=self._unpack(best_param_vector),
    #     )
    #
    # def coef_(self):
    #     return self._unpack(self._param_vector)
    #

class FitHistory(object):
    def __init__(self, *args, **kwargs):
        for key in kwargs:
            setattr(self, key, kwargs[key])
        self._keys = kwargs.keys()

    def __repr__(self):
        return 'FitHistory({param!r})'.format(param=self.param)
<commit_msg><commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

"""
Poisson point process penalised likelihood regression.
"""

try:
    import autograd
    import autograd.numpy as np
    # import autograd.scipy as sp
    have_autograd = True
except ImportError as e:
    import numpy as np
    # import scipy as sp
    have_autograd = False

from scipy.optimize import minimize

from . import influence
from . import model

class ContinuousExact(object):
    def __init__(
            self,
            debug=False
            ):
        self.debug = debug

    def _debug_print(self, *args, **kwargs):
        if self.debug:
            print(*args, **kwargs)

    def _pack(
            self,
            mu=1.0,
            kappa=0.0,
            tau=None,
            omega=None
            ):
        """
        pack all params into one vector for blackbox optimisation;
        currently unused
        """

        packed = np.zeros(
            1 +
            self.n_phi_bases +
            self.n_tau +
            self.n_omega
        )
        packed[0] = mu
        packed[
            1:self.n_phi_bases + 1
        ] = kappa
        packed[
            self.n_phi_bases + 1:
            self.n_phi_bases + self.n_tau + 1
        ] = tau if tau is not None else 1.0
        packed[
            self.n_phi_bases + self.n_tau + 1:
        ] = omega if omega is not None else 0.0
        return packed

    def _unpack(
            self,
            packed):
        """
        unpack all params from one vector for blackbox optimisation;
        currently unused.
        """
        unpacked = dict(
            mu=packed[0],
            kappa=packed[1:self.n_phi_bases+1],
        )
        # are we fitting time parameters?
        if self.n_tau > 0:
            unpacked = packed[
                self.n_phi_bases + 1:
                self.n_phi_bases + self.n_tau + 1
            ]
        # omega parameters - exogenous variation
        if self.n_omega > 0:
            unpacked = packed[
                self.n_phi_bases + self.n_tau + 1:
            ]
        return unpacked

    def negloglik(
            self,
            mu=1.0,
            **kwargs):

        return -model.loglik(
            ts=self._ts,
            eval_ts=self._eval_ts,
            mu=mu,
            phi_kernel=self.phi_kernel,
            mu_kernel=self.mu_kernel,
            **kwargs
        )

    def _penalty_weight_packed(
            self,
            pi_kappa=0.0,
            pi_omega=1e-8,
            **kwargs
            ):
        return self._pack(
            mu=0.0,
            kappa=pi_kappa,
            omega=pi_omega
        )  # / self.penalty_scale

    def penalty(
            self,
            mu=1.0,
            kappa=0.0,
            tau=1.0,
            omega=0.0,
            pi_kappa=0.0,
            pi_omega=0.0,
            **kwargs
            ):
        # 2 different l_1 penalties
        return np.sum(
            np.abs(kappa) * pi_kappa
        ) + np.sum(
            np.abs(omega) * pi_omega
        )

    def objective(
            self,
            **kwargs):

        g_negloglik = self.negloglik(
            **kwargs
        )
        g_penalty = self.penalty(
            **kwargs
        )
        penalty_dominated = np.abs(
            g_negloglik
        ) < (
            g_penalty
        )
        return (g_negloglik + g_penalty) * penalty_dominated

    def _setup_graphs(
            self,
            ts,
            phi_kernel=None,
            mu_kernel=None,
            n_phi_bases=1,
            n_mu_bases=0,
            fit_tau=False,
            fit_omega=False,
            t_start=0.0,
            t_end=None,
            param_vector=None,
            **kwargs):

        self._t_start = t_start
        self._t_end = t_end or ts[-1]
        self._ts = ts
        self._n_ts = np.sum(ts[
            np.logical_and(
                ts >= self._t_start,
                ts < self._t_end
            )
        ])
        # Full data likelihood must evaluate at the t_end also
        if ts[-1] < self._t_end:
            _eval_ts = np.append(
                ts[ts > self._t_start],
                [self._t_end]
            )
        else:
            _eval_ts = ts[
                np.logical_and(
                    ts >= self._t_start,
                    ts < self._t_end
                )
            ]

        self._eval_ts = _eval_ts

        if phi_kernel is None:
            phi_kernel = influence.MaxwellKernel(n_bases=5)
        else:
            phi_kernel = influence.as_influence_kernel(
                phi_kernel, n_bases=n_phi_bases
            )
        self.phi_kernel = phi_kernel
        self.n_phi_bases = phi_kernel.n_bases
        if mu_kernel is None:
            if n_mu_bases > 0:
                mu_kernel = influence.LinearStepKernel(
                    t_start=self._t_start,
                    t_end=self._t_end,
                    n_bases=n_mu_bases)
            else:
                mu_kernel = influence.ConstKernel()
        self.mu_kernel = mu_kernel
        self.n_mu_bases = n_mu_bases

        self._fit_tau = fit_tau
        self._fit_omega = fit_omega
        self.n_tau = self.n_phi_bases * self._fit_tau
        self.n_omega = self.n_mu_bases * self._fit_omega

    def _teardown_graphs(self):

        del(self._ts)
        del(self._eval_ts)

    def fit(
            self,
            ts,
            phi_kernel=None,
            mu_kernel=None,
            n_phi_bases=1,
            n_mu_bases=0,
            fit_tau=False,
            fit_omega=False,
            t_start=0.0,
            t_end=None,
            **kwargs
            ):
        self._setup_graphs(
            ts=ts,
            phi_kernel=phi_kernel,
            mu_kernel=mu_kernel,
            n_phi_bases=n_phi_bases,
            n_mu_bases=n_mu_bases,
            fit_tau=fit_tau,
            fit_omega=fit_omega,
            **kwargs
        )
        return self._fit(
            **kwargs
        )

    def _fit(
            self,
            mu=None,  # ignored
            kappa=None,
            tau=None,
            omega=None,
            pi_kappa=0.0,
            pi_omega=1e-8,
            max_steps=10,
            step_iter=10,
            eps=1e-8,
            **kwargs
            ):
        """
        fit by Truncated Newton in each coordinate group
        """
        if tau is None:
            tau = np.arange(self.n_tau)
        if omega is None:
            omega = np.zeros(self.n_omega)
        if kappa is None:
            kappa = np.ones(self.n_phi_bases)

        # def obj_mu(mu):
        #     return self.objective(
        #         mu=mu,
        #         kappa=kappa,
        #         tau=tau,
        #         omega=omega,
        #         pi_kappa=pi_kappa,
        #         pi_omega=pi_omega)
        #
        # grad_mu = autograd.value_and_grad(
        #     obj_mu, 0)

        def obj_kappa(kappa):
            return self.objective(
                mu=mu,
                kappa=kappa,
                tau=tau,
                omega=omega,
                pi_kappa=pi_kappa,
                pi_omega=pi_omega)

        grad_kappa = autograd.value_and_grad(
            obj_kappa, 0)
        kappa_bounds = [(0, 1)] * self.n_phi_bases

        def obj_tau(tau):
            return self.objective(
                mu=mu,
                kappa=kappa,
                tau=tau,
                omega=omega,
                pi_kappa=pi_kappa,
                pi_omega=pi_omega)

        grad_tau = autograd.value_and_grad(
            obj_tau, 0)
        tau_bounds = [(0, None)] * self.n_tau

        def obj_omega(mu):
            return self.objective(
                mu=mu,
                kappa=kappa,
                tau=tau,
                omega=omega,
                pi_kappa=pi_kappa,
                pi_omega=pi_omega)

        grad_omega = autograd.value_and_grad(
            obj_omega, 0)

        # self._hessian_diag_negloglik = autograd.elementwise_grad(
        #     self._grad_negloglik, 0)

        for i in range(max_steps):
            fit = {}
            if self._fit_tau:
                res = minimize(
                    grad_tau,
                    tau,
                    method='L-BFGS-B',  # ?
                    jac=True,
                    bounds=tau_bounds,
                    callback=lambda x: self._debug_print('tau', x),
                    options=dict(
                        maxiter=step_iter,
                        disp=self.debug
                    )
                )
                tau = res.x
                fit['tau'] = tau
            res = minimize(
                grad_kappa,
                kappa,
                method='TNC',
                jac=True,
                bounds=kappa_bounds,
                callback=lambda x: self._debug_print('kappa', x),
                options=dict(
                    maxiter=step_iter,
                    disp=self.debug
                )
            )
            kappa = res.x
            fit['kappa'] = kappa

            if self._fit_omega:
                omega_bounds = [(0, None)] * self.n_omega
                res = minimize(
                    grad_omega,
                    omega,
                    method='TNC',
                    jac=True,
                    bounds=omega_bounds,
                    callback=lambda x: self._debug_print('omega', x),
                    options=dict(
                        maxiter=step_iter,
                        disp=self.debug
                    )
                )
                omega = res.x
                fit['omega'] = omega

            # one-step mu update
            big_lam = model.big_lam_hawkes(
                mu=0.0,
                ts=self._ts,
                eval_ts=self._eval_ts,
                phi_kernel=self.phi_kernel,
                mu_kernel=self.mu_kernel,
                **fit,
                **kwargs
            )
            mu = (self._n_ts - big_lam)/(self._t_end-self._t_start)
            self._debug_print('mu', mu)
            fit['mu'] = mu

        return fit

    # def _fit_path(
    #         self,
    #         param_vector=None,
    #         pi_kappa=0.0,
    #         pi_omega=1e-8,
    #         max_steps=50,
    #         step_iter=20,
    #         lr=0.1,
    #         gamma=0.9,
    #         eps=1e-8,
    #         backoff=0.75,
    #         warm_start=False,
    #         **kwargs
    #         ):
    #
    #     n_params = param_vector.size
    #     param_path = np.zeros((n_params, max_steps))
    #     pi_kappa_path = np.zeros(max_steps)
    #     pi_omega_path = np.zeros(max_steps)
    #     loglik_path = np.zeros(max_steps)
    #     dof_path = np.zeros(max_steps)
    #     aic_path = np.zeros(max_steps)
    #
    #     self._avg_sq_grad = avg_sq_grad = np.ones_like(param_vector)
    #
    #     for j in range(max_steps):
    #         loss = self._objective_packed(param_vector)
    #         best_loss = loss
    #         local_lr = lr
    #         best_param_vector = np.array(param_vector)
    #
    #         for i in range(step_iter):
    #             g_negloglik = self._grad_negloglik(param_vector)
    #             g_penalty = self._grad_penalty(param_vector, pi_kappa, pi_omega)
    #             g = g_negloglik + g_penalty
    #             self._debug_print(j, i, 'param', param_vector, 'grad', g)
    #             avg_sq_grad[:] = avg_sq_grad * gamma + g**2 * (1 - gamma)
    #
    #             velocity = lr * g * (
    #                     np.sqrt(avg_sq_grad) + eps
    #             ) / (0.1 * (i + j + 10.0))
    #             # watch out, nans
    #             velocity[np.logical_not(np.isfinite(velocity))] = 0.0
    #
    #             penalty_dominant = np.abs(
    #                 g_negloglik
    #             ) < (
    #                 self._penalty_weight_packed(pi_kappa, pi_omega)
    #             )
    #             velocity[penalty_dominant * (velocity == 0)] = 0.0
    #             new_param_vector = param_vector - velocity * local_lr
    #             # coefficients that pass through 0 must stop there
    #             new_param_vector[
    #                 np.abs(
    #                     np.sign(new_param_vector) -
    #                     np.sign(param_vector)
    #                 ) == 2
    #             ] = 0.0
    #             new_param_vector[:] = np.maximum(new_param_vector, self._param_floor)
    #             new_loss = self._objective_packed(new_param_vector)
    #             if new_loss < loss:
    #                 # print('good', loss, '=>', new_loss, local_lr)
    #                 loss = new_loss
    #                 param_vector = new_param_vector
    #                 self._param_vector = new_param_vector
    #             else:
    #                 # print('bad', loss, '=>', new_loss, local_lr)
    #                 local_lr = local_lr * backoff
    #                 new_param_vector = param_vector + backoff * (
    #                     new_param_vector - param_vector
    #                 )
    #                 loss = self._objective_packed(new_param_vector)
    #             if loss < best_loss:
    #                 best_param_vector = np.array(param_vector)
    #                 best_loss = loss
    #
    #             if local_lr < 1e-3:
    #                 self._debug_print('nope', j, i, max_steps)
    #                 break
    #
    #         this_loglik = -self._negloglik_packed(best_param_vector)
    #         this_dof = self._dof_packed(best_param_vector)
    #         param_path[:, j] = best_param_vector
    #         pi_kappa_path[j] = pi_kappa
    #         pi_omega_path[j] = pi_omega
    #         loglik_path[j] = this_loglik
    #         dof_path[j] = this_dof
    #         aic_path[j] = 2 * this_loglik - 2 * this_dof
    #
    #         # # regularisation parameter selection
    #         # all_grad = self._unpack(
    #         #     np.abs(
    #         #         grad_objective(best_param_vector) *
    #         #         (best_param_vector != 0.0)
    #         #     )
    #         # )
    #         # kappa_grad = all_grad['kappa']
    #         # log_omega_grad = all_grad.get('omega', None)
    #         #
    #         # if (
    #         #     np.random.random() < (
    #         #         np.sqrt(log_omega_grad.size) /
    #         #         (np.sqrt(kappa_grad.size) +
    #         #         np.sqrt(log_omega_grad.size))
    #         #         )):
    #         #     print('log_omega_grad', log_omega_grad)
    #         #     pi_omega += max(
    #         #         np.amin(log_omega_grad[log_omega_grad > 0])
    #         #         * j/max_steps,
    #         #         pi_omega * 0.1
    #         #     )
    #         # else:
    #         #     print('kappa_grad', kappa_grad)
    #         #     pi_kappa += max(
    #         #         np.amin(kappa_grad[kappa_grad > 0]) * j / max_steps,
    #         #         pi_kappa * 0.1
    #         #     )
    #     return FitHistory(
    #         param_path=param_path[:, :j],
    #         pi_kappa_path=pi_kappa_path[:j],
    #         pi_omega_path=pi_omega_path[:j],
    #         loglik_path=loglik_path[:j],
    #         dof_path=dof_path[:j],
    #         aic_path=aic_path[:j],
    #         unpack=lambda v: self._unpack(v),
    #         param_vector=best_param_vector,
    #         param=self._unpack(best_param_vector),
    #     )
    #
    # def coef_(self):
    #     return self._unpack(self._param_vector)
    #

class FitHistory(object):
    def __init__(self, *args, **kwargs):
        for key in kwargs:
            setattr(self, key, kwargs[key])
        self._keys = kwargs.keys()

    def __repr__(self):
        return 'FitHistory({param!r})'.format(param=self.param)
<|endoftext|>