<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from cms.apphook_pool import apphook_pool
from cms.menu_bases import CMSAttachMenu
from django.core.urlresolvers import resolve
from django.db.models.signals import post_delete, post_save
from django.utils.translation import get_language_from_request, ugettext_lazy as _
from menus.base import Modifier, NavigationNode
from menus.menu_pool import menu_pool

from .cms_appconfig import BlogConfig
from .models import BlogCategory, Post
from .settings import MENU_TYPE_CATEGORIES, MENU_TYPE_COMPLETE, MENU_TYPE_POSTS, get_setting

class BlogCategoryMenu(CMSAttachMenu):
    """
    Main menu class

    Handles all types of blog menu
    """
    name = _('Blog menu')

    def get_nodes(self, request):
        """
        Generates the nodelist

        :param request:
        :return: list of nodes
        """
        nodes = []

        language = get_language_from_request(request, check_path=True)

        categories_menu = False
        posts_menu = False
        config = False
        if hasattr(self, 'instance') and self.instance:
            config = BlogConfig.objects.get(namespace=self.instance.application_namespace)
        if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_CATEGORIES):
            categories_menu = True
        if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_POSTS):
            posts_menu = True

        if categories_menu:
            categories = BlogCategory.objects
            if config:
                categories = categories.namespace(self.instance.application_namespace)
            categories = categories.active_translations(language).distinct()
            categories = categories.order_by('parent__id', 'translations__name')
            for category in categories:
                node = NavigationNode(
                    category.name,
                    category.get_absolute_url(),
                    '%s-%s' % (category.__class__.__name__, category.pk),
                    ('%s-%s' % (category.__class__.__name__, category.parent.id) if category.parent
                     else None)
                )
                nodes.append(node)

        if posts_menu:
            posts = Post.objects
            if hasattr(self, 'instance') and self.instance:
                posts = posts.namespace(self.instance.application_namespace)
            posts = posts.active_translations(language).distinct()
            for post in posts:
                post_id = None
                parent = None
                if categories_menu:
                    category = post.categories.first()
                    if category:
                        parent = '%s-%s' % (category.__class__.__name__, category.pk)
                        post_id = '%s-%s' % (post.__class__.__name__, post.pk),
                else:
                    post_id = '%s-%s' % (post.__class__.__name__, post.pk),
                if post_id:
                    node = NavigationNode(
                        post.get_title(),
                        post.get_absolute_url(language),
                        post_id,
                        parent
                    )
                    nodes.append(node)

        return nodes

menu_pool.register_menu(BlogCategoryMenu)

class BlogNavModifier(Modifier):
    """
    This navigation modifier makes sure that when
    a particular blog post is viewed,
    a corresponding category is selected in menu
    """
    def modify(self, request, nodes, namespace, root_id, post_cut, breadcrumb):
        app = None
        config = None
        if getattr(request, 'current_page', None) and request.current_page.application_urls:
            app = apphook_pool.get_apphook(request.current_page.application_urls)

        if app and app.app_config:
            namespace = resolve(request.path).namespace
            config = app.get_config(namespace)
        if config and config.menu_structure != MENU_TYPE_CATEGORIES:
            return nodes
        if post_cut:
            return nodes
        current_post = getattr(request, get_setting('CURRENT_POST_IDENTIFIER'), None)
        category = None
        if current_post and current_post.__class__ == Post:
            category = current_post.categories.first()
        if not category:
            return nodes

        for node in nodes:
            if '%s-%s' % (category.__class__.__name__, category.pk) == node.id:
                node.selected = True
        return nodes

menu_pool.register_modifier(BlogNavModifier)

def clear_menu_cache(**kwargs):
    """
    Empty menu cache when saving categories
    """
    menu_pool.clear(all=True)

post_save.connect(clear_menu_cache, sender=BlogCategory)
post_delete.connect(clear_menu_cache, sender=BlogCategory)
<commit_msg>Fix string formatting<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, unicode_literals

from cms.apphook_pool import apphook_pool
from cms.menu_bases import CMSAttachMenu
from django.core.urlresolvers import resolve
from django.db.models.signals import post_delete, post_save
from django.utils.translation import get_language_from_request, ugettext_lazy as _
from menus.base import Modifier, NavigationNode
from menus.menu_pool import menu_pool

from .cms_appconfig import BlogConfig
from .models import BlogCategory, Post
from .settings import MENU_TYPE_CATEGORIES, MENU_TYPE_COMPLETE, MENU_TYPE_POSTS, get_setting

class BlogCategoryMenu(CMSAttachMenu):
    """
    Main menu class

    Handles all types of blog menu
    """
    name = _('Blog menu')

    def get_nodes(self, request):
        """
        Generates the nodelist

        :param request:
        :return: list of nodes
        """
        nodes = []

        language = get_language_from_request(request, check_path=True)

        categories_menu = False
        posts_menu = False
        config = False
        if hasattr(self, 'instance') and self.instance:
            config = BlogConfig.objects.get(namespace=self.instance.application_namespace)
        if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_CATEGORIES):
            categories_menu = True
        if config and config.menu_structure in (MENU_TYPE_COMPLETE, MENU_TYPE_POSTS):
            posts_menu = True

        if categories_menu:
            categories = BlogCategory.objects
            if config:
                categories = categories.namespace(self.instance.application_namespace)
            categories = categories.active_translations(language).distinct()
            categories = categories.order_by('parent__id', 'translations__name')
            for category in categories:
                node = NavigationNode(
                    category.name,
                    category.get_absolute_url(),
                    '{0}-{1}'.format(category.__class__.__name__, category.pk),
                    (
                        '{0}-{1}'.format(
                            category.__class__.__name__, category.parent.id
                        ) if category.parent else None
                    )
                )
                nodes.append(node)

        if posts_menu:
            posts = Post.objects
            if hasattr(self, 'instance') and self.instance:
                posts = posts.namespace(self.instance.application_namespace)
            posts = posts.active_translations(language).distinct()
            for post in posts:
                post_id = None
                parent = None
                if categories_menu:
                    category = post.categories.first()
                    if category:
                        parent = '{0}-{1}'.format(category.__class__.__name__, category.pk)
                        post_id = '{0}-{1}'.format(post.__class__.__name__, post.pk),
                else:
                    post_id = '{0}-{1}'.format(post.__class__.__name__, post.pk),
                if post_id:
                    node = NavigationNode(
                        post.get_title(),
                        post.get_absolute_url(language),
                        post_id,
                        parent
                    )
                    nodes.append(node)

        return nodes

menu_pool.register_menu(BlogCategoryMenu)

class BlogNavModifier(Modifier):
    """
    This navigation modifier makes sure that when
    a particular blog post is viewed,
    a corresponding category is selected in menu
    """
    def modify(self, request, nodes, namespace, root_id, post_cut, breadcrumb):
        """
        Actual modifier function
        :param request: request
        :param nodes: complete list of nodes
        :param namespace: Menu namespace
        :param root_id: eventual root_id
        :param post_cut: flag for modifier stage
        :param breadcrumb: flag for modifier stage
        :return: nodeslist
        """
        app = None
        config = None
        if getattr(request, 'current_page', None) and request.current_page.application_urls:
            app = apphook_pool.get_apphook(request.current_page.application_urls)

        if app and app.app_config:
            namespace = resolve(request.path).namespace
            config = app.get_config(namespace)
        if config and config.menu_structure != MENU_TYPE_CATEGORIES:
            return nodes
        if post_cut:
            return nodes
        current_post = getattr(request, get_setting('CURRENT_POST_IDENTIFIER'), None)
        category = None
        if current_post and current_post.__class__ == Post:
            category = current_post.categories.first()
        if not category:
            return nodes

        for node in nodes:
            if '{0}-{1}'.format(category.__class__.__name__, category.pk) == node.id:
                node.selected = True
        return nodes

menu_pool.register_modifier(BlogNavModifier)

def clear_menu_cache(**kwargs):
    """
    Empty menu cache when saving categories
    """
    menu_pool.clear(all=True)

post_save.connect(clear_menu_cache, sender=BlogCategory)
post_delete.connect(clear_menu_cache, sender=BlogCategory)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

def test_bar_fixture(testdir):
    """Make sure that pytest accepts our fixture."""

    # create a temporary pytest test module
    testdir.makepyfile("""
        def test_sth(bar):
            assert bar == "europython2015"
    """)

    # run pytest with the following cmd args
    result = testdir.runpytest(
        '--foo=europython2015',
        '-v'
    )

    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        '*::test_sth PASSED',
    ])

    # make sure that that we get a '0' exit code for the testsuite
    assert result.ret == 0

def test_help_message(testdir):
    result = testdir.runpytest(
        '--help',
    )
    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        'prob:',
        '*--foo=DEST_FOO*Set the value for the fixture "bar".',
    ])

def test_hello_ini_setting(testdir):
    testdir.makeini("""
        [pytest]
        HELLO = world
    """)

    testdir.makepyfile("""
        import pytest

        @pytest.fixture
        def hello(request):
            return request.config.getini('HELLO')

        def test_hello_world(hello):
            assert hello == 'world'
    """)

    result = testdir.runpytest('-v')

    # fnmatch_lines does an assertion internally
    result.stdout.fnmatch_lines([
        '*::test_hello_world PASSED',
    ])

    # make sure that that we get a '0' exit code for the testsuite
    assert result.ret == 0
<commit_msg>simple testcases <commit_after># -*- coding: utf-8 -*-

def test_simple_predicate(testdir):
    testdir.makeconftest('pytest_plugins = "prob"')
    testdir.makefile('.yml', test_truth="""
        test_predicate:
            test: "1=1"
    """)
    result = testdir.runpytest('-v')

    result.stdout.fnmatch_lines([
        '*::test_predicate PASSED',
    ])

def test_simple_false_predicate(testdir):
    testdir.makeconftest('pytest_plugins = "prob"')
    testdir.makefile('.yml', test_truth="""
        test_predicate:
            test: "1 /= 1"
    """)
    result = testdir.runpytest('-v')

    result.stdout.fnmatch_lines([
        '*::test_predicate FAILED',
    ])

def test_truth(testdir):
    testdir.makeconftest('pytest_plugins = "prob"')
    testdir.makefile('.yml', test_truth="""
        test_truth:
            test: "TRUE"
    """)
    result = testdir.runpytest('-v')

    result.stdout.fnmatch_lines([
        '*::test_truth PASSED',
    ])
<|endoftext|>


<commit_before>import settings
from django import forms
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User, Group, Permission

from djangotoolbox.auth.utils import update_permissions_user, \
     update_user_groups, update_permissions_group
from djangotoolbox.auth.models import UserPermissionList, GroupList, \
     GroupPermissionList

class UserForm(forms.ModelForm):
    class Meta:
        model = User
        fields = ('username', 'email', 'first_name', 'last_name', 'is_active',
                  'is_staff', 'is_superuser')

class NonrelPermissionUserForm(UserForm):
    permissions = forms.MultipleChoiceField(required=False)
    groups = forms.MultipleChoiceField(required=False)

    def __init__(self, *args, **kwargs):
        super(NonrelPermissionUserForm, self).__init__(*args, **kwargs)

        self.fields['permissions'] = forms.MultipleChoiceField(required=False)
        self.fields['groups'] = forms.MultipleChoiceField(required=False)

        permissions_objs = Permission.objects.all().order_by('name')
        choices = list ()
        for perm_obj in permissions_objs:
            choices.append([perm_obj.id, perm_obj.name])
        self.fields['permissions'].choices = choices

        group_objs = Group.objects.all()
        choices = list ()
        for group_obj in group_objs:
            choices.append([group_obj.id, group_obj.name])
        self.fields['groups'].choices = choices

        try:
            user_perm_list = UserPermissionList.objects.get(
                user=kwargs['instance'])
            self.fields['permissions'].initial = user_perm_list.fk_list
        except (UserPermissionList.DoesNotExist, KeyError):
            self.fields['permissions'].initial = list()

        try:
            user_group_list = GroupList.objects.get(
                user=kwargs['instance'])
            self.fields['groups'].initial = user_group_list.fk_list
        except (GroupList.DoesNotExist, KeyError):
            self.fields['groups'].initial = list()

class CustomUserAdmin(UserAdmin):
    fieldsets = None
    form = UserForm

class NonrelPermissionCustomUserAdmin(UserAdmin):
    fieldsets = None
    form = NonrelPermissionUserForm

    def save_model(self, request, obj, form, change):
        super(NonrelPermissionCustomUserAdmin, self).save_model(request, obj, form, change)

        if len(form.cleaned_data['permissions']) > 0:
            permissions = list(Permission.objects.filter(
                id__in=form.cleaned_data['permissions']).order_by('name'))
        else:
            permissions = list()

        update_permissions_user(permissions, obj)

        if len(form.cleaned_data['groups']) > 0:
            groups = list(Group.objects.filter(
                id__in=form.cleaned_data['groups']))
        else:
            groups = list()

        update_user_groups(obj, groups)

class PermissionAdmin(admin.ModelAdmin):
    ordering = ('name',)

class GroupForm(forms.ModelForm):
    permissions = forms.MultipleChoiceField(required=False)

    def __init__(self, *args, **kwargs):
        super(GroupForm, self).__init__(*args, **kwargs)

        self.fields['permissions'] = forms.MultipleChoiceField(required=False)

        permissions_objs = Permission.objects.all().order_by('name')
        choices = list ()
        for perm_obj in permissions_objs:
            choices.append([perm_obj.id, perm_obj.name])
        self.fields['permissions'].choices = choices

        try:
            current_perm_list = GroupPermissionList.objects.get(
                group=kwargs['instance'])
            self.fields['permissions'].initial = current_perm_list.fk_list
        except (GroupPermissionList.DoesNotExist, KeyError):
            self.fields['permissions'].initial = list()

    class Meta:
        model = Group
        fields = ('name',)

class CustomGroupAdmin(admin.ModelAdmin):
    form = GroupForm
    fieldsets = None

    def save_model(self, request, obj, form, change):
        super(CustomGroupAdmin, self).save_model(request, obj, form, change)

        if len(form.cleaned_data['permissions']) > 0:
            permissions = list(Permission.objects.filter(
                id__in=form.cleaned_data['permissions']).order_by('name'))
        else:
            permissions = list()

        update_permissions_group(permissions, obj)

admin.site.unregister(User)
admin.site.unregister(Group)

backends = getattr(settings, 'AUTHENTICATION_BACKENDS', list())
backend_name = 'djangotoolbox.auth.backends.NonrelPermissionBackend'
if backend_name in backends:
    admin.site.register(User, NonrelPermissionCustomUserAdmin)
    admin.site.register(Permission, PermissionAdmin)
    admin.site.register(Group, CustomGroupAdmin)
else:
    admin.site.register(User, CustomUserAdmin)

<commit_msg>replace list() with []<commit_after>import settings
from django import forms
from django.contrib import admin
from django.contrib.auth.admin import UserAdmin
from django.contrib.auth.models import User, Group, Permission

from djangotoolbox.auth.utils import update_permissions_user, \
     update_user_groups, update_permissions_group
from djangotoolbox.auth.models import UserPermissionList, GroupList, \
     GroupPermissionList

class UserForm(forms.ModelForm):
    class Meta:
        model = User
        fields = ('username', 'email', 'first_name', 'last_name', 'is_active',
                  'is_staff', 'is_superuser')

class NonrelPermissionUserForm(UserForm):
    permissions = forms.MultipleChoiceField(required=False)
    groups = forms.MultipleChoiceField(required=False)

    def __init__(self, *args, **kwargs):
        super(NonrelPermissionUserForm, self).__init__(*args, **kwargs)

        self.fields['permissions'] = forms.MultipleChoiceField(required=False)
        self.fields['groups'] = forms.MultipleChoiceField(required=False)

        permissions_objs = Permission.objects.all().order_by('name')
        choices = list ()
        for perm_obj in permissions_objs:
            choices.append([perm_obj.id, perm_obj.name])
        self.fields['permissions'].choices = choices

        group_objs = Group.objects.all()
        choices = list ()
        for group_obj in group_objs:
            choices.append([group_obj.id, group_obj.name])
        self.fields['groups'].choices = choices

        try:
            user_perm_list = UserPermissionList.objects.get(
                user=kwargs['instance'])
            self.fields['permissions'].initial = user_perm_list.fk_list
        except (UserPermissionList.DoesNotExist, KeyError):
            self.fields['permissions'].initial = list()

        try:
            user_group_list = GroupList.objects.get(
                user=kwargs['instance'])
            self.fields['groups'].initial = user_group_list.fk_list
        except (GroupList.DoesNotExist, KeyError):
            self.fields['groups'].initial = list()

class CustomUserAdmin(UserAdmin):
    fieldsets = None
    form = UserForm

class NonrelPermissionCustomUserAdmin(UserAdmin):
    fieldsets = None
    form = NonrelPermissionUserForm

    def save_model(self, request, obj, form, change):
        super(NonrelPermissionCustomUserAdmin, self).save_model(request, obj, form, change)

        if len(form.cleaned_data['permissions']) > 0:
            permissions = [Permission.objects.filter(
                id__in=form.cleaned_data['permissions']).order_by('name')]
        else:
            permissions = []

        update_permissions_user(permissions, obj)

        if len(form.cleaned_data['groups']) > 0:
            groups = [Group.objects.filter(
                id__in=form.cleaned_data['groups'])]
        else:
            groups = []

        update_user_groups(obj, groups)

class PermissionAdmin(admin.ModelAdmin):
    ordering = ('name',)

class GroupForm(forms.ModelForm):
    permissions = forms.MultipleChoiceField(required=False)

    def __init__(self, *args, **kwargs):
        super(GroupForm, self).__init__(*args, **kwargs)

        self.fields['permissions'] = forms.MultipleChoiceField(required=False)

        permissions_objs = Permission.objects.all().order_by('name')
        choices = []
        for perm_obj in permissions_objs:
            choices.append([perm_obj.id, perm_obj.name])
        self.fields['permissions'].choices = choices

        try:
            current_perm_list = GroupPermissionList.objects.get(
                group=kwargs['instance'])
            self.fields['permissions'].initial = current_perm_list.fk_list
        except (GroupPermissionList.DoesNotExist, KeyError):
            self.fields['permissions'].initial = []

    class Meta:
        model = Group
        fields = ('name',)

class CustomGroupAdmin(admin.ModelAdmin):
    form = GroupForm
    fieldsets = None

    def save_model(self, request, obj, form, change):
        super(CustomGroupAdmin, self).save_model(request, obj, form, change)

        if len(form.cleaned_data['permissions']) > 0:
            permissions = [Permission.objects.filter(
                id__in=form.cleaned_data['permissions']).order_by('name')]
        else:
            permissions = []

        update_permissions_group(permissions, obj)

admin.site.unregister(User)
admin.site.unregister(Group)

backends = getattr(settings, 'AUTHENTICATION_BACKENDS', list())
backend_name = 'djangotoolbox.auth.backends.NonrelPermissionBackend'
if backend_name in backends:
    admin.site.register(User, NonrelPermissionCustomUserAdmin)
    admin.site.register(Permission, PermissionAdmin)
    admin.site.register(Group, CustomGroupAdmin)
else:
    admin.site.register(User, CustomUserAdmin)

<|endoftext|>


<commit_before>import string
import datetime
import os
import io
import sys
import pysftp
import csv
import datetime
from datetime import date
from datetime import datetime, timedelta
import time
from time import strftime
import argparse
import uuid
from sqlalchemy import text
import shutil
import re
import logging
from logging.handlers import SMTPHandler
#import adp_ftp
import codecs

# python path
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
sys.path.append('/djequis/adp')

# django settings for shell environment
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djequis.settings")

# prime django
import django
django.setup()

# django settings for script
from django.conf import settings
from django.db import connections

# informix environment
os.environ['INFORMIXSERVER'] = settings.INFORMIXSERVER
os.environ['DBSERVERNAME'] = settings.DBSERVERNAME
os.environ['INFORMIXDIR'] = settings.INFORMIXDIR
os.environ['ODBCINI'] = settings.ODBCINI
os.environ['ONCONFIG'] = settings.ONCONFIG
os.environ['INFORMIXSQLHOSTS'] = settings.INFORMIXSQLHOSTS
os.environ['LD_LIBRARY_PATH'] = settings.LD_LIBRARY_PATH
os.environ['LD_RUN_PATH'] = settings.LD_RUN_PATH

# from djequis.core.utils import sendmail
from djequis.adp.utilities import fn_validate_field, fn_check_duplicates
from djzbar.utils.informix import do_sql
from djzbar.utils.informix import get_engine
from djzbar.settings import INFORMIX_EARL_TEST
from djzbar.settings import INFORMIX_EARL_PROD
from djzbar.settings import INFORMIX_EARL_SANDBOX
from djtools.fields import TODAY

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""
parser = argparse.ArgumentParser(description=desc)

parser.add_argument(
    "--test",
    action='store_true',
    help="Dry run?",
    dest="test"
)
parser.add_argument(
    "-d", "--database",
    help="database name.",
    dest="database"
)

# set global variable
global EARL
# determines which database is being called from the command line
# if database == 'cars':
#    EARL = INFORMIX_EARL_PROD
# elif database == 'train':
# EARL = INFORMIX_EARL_TEST
# elif database == 'sandbox':
EARL = INFORMIX_EARL_SANDBOX
# else:
    # this will raise an error when we call get_engine()
    # below but the argument parser should have taken
    # care of this scenario and we will never arrive here.
#    EARL = None
# establish database connection
engine = get_engine(EARL)

def fn_process_cvid(carthid, adpid, ssn, adp_assoc_id):
    # create logger
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    # create console handler and set level to info
    handler = logging.FileHandler(
        '{0}apdtocx.log'.format(settings.LOG_FILEPATH))
    handler.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s',
                                  datefmt='%m/%d/%Y %I:%M:%S %p')
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    # create error file handler and set level to error
    handler = logging.FileHandler(
        '{0}apdtocx_error.log'.format(settings.LOG_FILEPATH))
    handler.setLevel(logging.ERROR)
    formatter = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s',
                                  datefmt='%m/%d/%Y %I:%M:%S %p')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    try:
        ##############################################################
        # Inserts or updates as needed into cvid_rec
        ##############################################################

        # print(carthid, adpid, ssn, adp_assoc_id)

        # Validate the cx_id
        v_cx_id = fn_validate_field(carthid, "cx_id", "cx_id", "cvid_rec",
                    "integer")
        print("CX_ID = " + str(carthid))
        print("v_CX_ID = " + str(v_cx_id))

        # Should also check for duplicates of the adp_id and associate_id
        # What to do in that case?
        v_adp_match = fn_check_duplicates(adpid, "adp_id", "cx_id", "cvid_rec",
                                 v_cx_id, "char")
        print("Found_ID = " + str(v_adp_match))

        # By definition, associate ID cannot be a duplicate in ADP, but
        # possibility of duplicate might exist in CX
        v_assoc_match = fn_check_duplicates(adp_assoc_id, "adp_associate_id",
                       "cx_id", "cvid_rec", v_cx_id, "char")
        print("Found ID = " + str(v_assoc_match))

        if v_cx_id == 0 and v_assoc_match == 0 and v_adp_match == 0:
            # Insert or update as needed to ID_rec
            # Insert works 06/12/18

            q_insert_cvid_rec = '''INSERT INTO cvid_rec (old_id, old_id_num,
              adp_id, ssn, cx_id, cx_id_char, adp_associate_id)
              VALUES (?,?,?,?,?,?,?)'''
            args = (carthid, carthid, adpid, ssn, carthid, carthid, adp_assoc_id)

            print(q_insert_cvid_rec)

            engine.execute(q_insert_cvid_rec, args)
            # scr.write(q_insert_cvid_rec + '\n');
            logger.info("Inserted into cvid_rec table");
        elif str(v_cx_id) != v_assoc_match and v_assoc_match != 0:
            print('Duplicate Associate ID found')
        elif str(v_cx_id) != str(v_adp_match) and v_adp_match != 0:
            print('Duplicate ADP ID found')
        else:
            # sql works - 6/12/18
            q_update_cvid_rec = '''UPDATE cvid_rec SET old_id = ?,
                old_id_num = ?, adp_id = ?, ssn = ?, adp_associate_id = ?
                WHERE cx_id = ?'''
            args = (carthid, carthid, adpid, ssn, adp_assoc_id, carthid)
            print(q_update_cvid_rec)
            logger.info("Update cvid_rec table");
            engine.execute(q_update_cvid_rec, args)

    except Exception as e:
        print(e)
<commit_msg>cleaning up code and adding logging<commit_after>import os
import string
import sys
import pysftp
import csv
import datetime
from datetime import date
from datetime import datetime, timedelta
import codecs
import time
from time import strftime
import argparse
#import uuid
from sqlalchemy import text
import shutil
#import re
import logging
from logging.handlers import SMTPHandler

# python path
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
sys.path.append('/djequis/adp')

# django settings for shell environment
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djequis.settings")

# prime django
import django
django.setup()

# django settings for script
from django.conf import settings
from django.db import connections

# informix environment
os.environ['INFORMIXSERVER'] = settings.INFORMIXSERVER
os.environ['DBSERVERNAME'] = settings.DBSERVERNAME
os.environ['INFORMIXDIR'] = settings.INFORMIXDIR
os.environ['ODBCINI'] = settings.ODBCINI
os.environ['ONCONFIG'] = settings.ONCONFIG
os.environ['INFORMIXSQLHOSTS'] = settings.INFORMIXSQLHOSTS
os.environ['LD_LIBRARY_PATH'] = settings.LD_LIBRARY_PATH
os.environ['LD_RUN_PATH'] = settings.LD_RUN_PATH

# from djequis.core.utils import sendmail
from djzbar.utils.informix import do_sql
from djzbar.utils.informix import get_engine
from djzbar.settings import INFORMIX_EARL_TEST
from djzbar.settings import INFORMIX_EARL_PROD
from djzbar.settings import INFORMIX_EARL_SANDBOX

from djtools.fields import TODAY

# Imports for additional modules and functions written as part of this project
from djequis.adp.utilities import fn_validate_field, fn_check_duplicates

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""
parser = argparse.ArgumentParser(description=desc)

parser.add_argument(
    "--test",
    action='store_true',
    help="Dry run?",
    dest="test"
)
parser.add_argument(
    "-d", "--database",
    help="database name.",
    dest="database"
)

# set global variable
global EARL
# determines which database is being called from the command line
# if database == 'cars':
#    EARL = INFORMIX_EARL_PROD
# elif database == 'train':
# EARL = INFORMIX_EARL_TEST
# elif database == 'sandbox':
EARL = INFORMIX_EARL_SANDBOX
# else:
    # this will raise an error when we call get_engine()
    # below but the argument parser should have taken
    # care of this scenario and we will never arrive here.
#    EARL = None
# establish database connection
engine = get_engine(EARL)

# write out the .sql file
scr = open("apdtocx_output.sql", "a")
# set start_time in order to see how long script takes to execute
start_time = time.time()
# create logger
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
# create console handler and set level to info
handler = logging.FileHandler('{0}apdtocx.log'.format(settings.LOG_FILEPATH))
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s',
                              datefmt='%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(formatter)
logger.addHandler(handler)
# create error file handler and set level to error
handler = logging.FileHandler('{0}apdtocx_error.log'.format(settings.LOG_FILEPATH))
handler.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s: %(levelname)s: %(message)s',
                              datefmt='%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(formatter)
logger.addHandler(handler)

def fn_process_cvid(carthid, adpid, ssn, adp_assoc_id):

    try:
        ##############################################################
        # Inserts or updates as needed into cvid_rec
        ##############################################################

        # Validate the cx_id
        v_cx_id = fn_validate_field(carthid, "cx_id", "cx_id", "cvid_rec",
                    "integer")
        #print("CX_ID = " + str(carthid))
        #print("v_CX_ID = " + str(v_cx_id))

        # Should also check for duplicates of the adp_id and associate_id
        # What to do in that case?
        v_adp_match = fn_check_duplicates(adpid, "adp_id", "cx_id", "cvid_rec",
                                 v_cx_id, "char")
        #print("Found_ID = " + str(v_adp_match))

        # By definition, associate ID cannot be a duplicate in ADP, but
        # possibility of duplicate might exist in CX
        v_assoc_match = fn_check_duplicates(adp_assoc_id, "adp_associate_id",
                       "cx_id", "cvid_rec", v_cx_id, "char")
        #print("Found ID = " + str(v_assoc_match))

        if v_cx_id == 0 and v_assoc_match == 0 and v_adp_match == 0:
            # Insert or update as needed to ID_rec
            q_insert_cvid_rec = '''
              INSERT INTO cvid_rec (old_id, old_id_num, adp_id, ssn, cx_id,
              cx_id_char, adp_associate_id)
              VALUES (?,?,?,?,?,?,?)'''
            args = (carthid, carthid, adpid, ssn, carthid, carthid, adp_assoc_id)
            # print(q_insert_cvid_rec)
            engine.execute(q_insert_cvid_rec, args)
            scr.write(q_insert_cvid_rec + '\n');
            logger.info("Inserted into cvid_rec table");
        elif str(v_cx_id) != v_assoc_match and v_assoc_match != 0:
            print('Duplicate Associate ID found')
        elif str(v_cx_id) != str(v_adp_match) and v_adp_match != 0:
            print('Duplicate ADP ID found')
        else:
            q_update_cvid_rec = '''
              UPDATE cvid_rec
              SET old_id = ?, old_id_num = ?, adp_id = ?, ssn = ?,
              adp_associate_id = ?
              WHERE cx_id = ?'''
            args = (carthid, carthid, adpid, ssn, adp_assoc_id, carthid)
            # print(q_update_cvid_rec)
            logger.info("Update cvid_rec table");
            scr.write(q_update_cvid_rec + '\n');
            engine.execute(q_update_cvid_rec, args)

    except Exception as e:
        print(e)
    finally:
        logging.shutdown()
<|endoftext|>


<commit_before># Copyright (c) 2015-2018, Activision Publishing, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys

from assertpy import assert_that, soft_assertions, fail

def test_success():
    with soft_assertions():
        assert_that('foo').is_length(3)
        assert_that('foo').is_not_empty()
        assert_that('foo').is_true()
        assert_that('foo').is_alpha()
        assert_that('123').is_digit()
        assert_that('foo').is_lower()
        assert_that('FOO').is_upper()
        assert_that('foo').is_equal_to('foo')
        assert_that('foo').is_not_equal_to('bar')
        assert_that('foo').is_equal_to_ignoring_case('FOO')
        assert_that({'a': 1}).has_a(1)

def test_failure():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4)
            assert_that('foo').is_empty()
            assert_that('foo').is_false()
            assert_that('foo').is_digit()
            assert_that('123').is_alpha()
            assert_that('foo').is_upper()
            assert_that('FOO').is_lower()
            assert_that('foo').is_equal_to('bar')
            assert_that('foo').is_not_equal_to('foo')
            assert_that('foo').is_equal_to_ignoring_case('BAR')
            assert_that({'a': 1}).has_a(2)
            assert_that({'a': 1}).has_foo(1)
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).contains('Expected <foo> to be empty string, but was not.')
        assert_that(out).contains('Expected <False>, but was not.')
        assert_that(out).contains('Expected <foo> to contain only digits, but did not.')
        assert_that(out).contains('Expected <123> to contain only alphabetic chars, but did not.')
        assert_that(out).contains('Expected <foo> to contain only uppercase chars, but did not.')
        assert_that(out).contains('Expected <FOO> to contain only lowercase chars, but did not.')
        assert_that(out).contains('Expected <foo> to be equal to <bar>, but was not.')
        assert_that(out).contains('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).contains('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')
        assert_that(out).contains('Expected <1> to be equal to <2> on key <a>, but was not.')
        assert_that(out).contains('Expected key <foo>, but val has no key <foo>.')

def test_failure_chain():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4).is_empty().is_false().is_digit().is_upper()\
                .is_equal_to('bar').is_not_equal_to('foo').is_equal_to_ignoring_case('BAR')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).contains('Expected <foo> to be empty string, but was not.')
        assert_that(out).contains('Expected <False>, but was not.')
        assert_that(out).contains('Expected <foo> to contain only digits, but did not.')
        assert_that(out).contains('Expected <foo> to contain only uppercase chars, but did not.')
        assert_that(out).contains('Expected <foo> to be equal to <bar>, but was not.')
        assert_that(out).contains('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).contains('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')

def test_expected_exception_success():
    with soft_assertions():
        assert_that(func_err).raises(RuntimeError).when_called_with('foo').is_equal_to('err')

def test_expected_exception_failure():
    try:
        with soft_assertions():
            assert_that(func_err).raises(RuntimeError).when_called_with('foo').is_equal_to('bar')
            assert_that(func_ok).raises(RuntimeError).when_called_with('baz')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <err> to be equal to <bar>, but was not.')
        assert_that(out).contains("Expected <func_ok> to raise <RuntimeError> when called with ('baz').")

def func_ok(arg):
    pass

def func_err(arg):
    raise RuntimeError('err')

def test_fail():
    try:
        with soft_assertions():
            fail()
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail!')

def test_fail_with_msg():
    try:
        with soft_assertions():
            fail('foobar')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail: foobar!')

def test_fail_with_soft_failing_asserts():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4)
            assert_that('foo').is_empty()
            fail('foobar')
            assert_that('foo').is_not_equal_to('foo')
            assert_that('foo').is_equal_to_ignoring_case('BAR')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail: foobar!')
        assert_that(out).does_not_contain('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).does_not_contain('Expected <foo> to be empty string, but was not.')
        assert_that(out).does_not_contain('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).does_not_contain('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')

def test_double_fail():
    try:
        with soft_assertions():
            fail()
            fail('foobar')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail!')

def test_nested():
    try:
        with soft_assertions():
            assert_that('a').is_equal_to('A')
            with soft_assertions():
                assert_that('b').is_equal_to('B')
                with soft_assertions():
                    assert_that('c').is_equal_to('C')
                assert_that('b').is_equal_to('B2')
            assert_that('a').is_equal_to('A2')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('1. Expected <a> to be equal to <A>, but was not.')
        assert_that(out).contains('2. Expected <b> to be equal to <B>, but was not.')
        assert_that(out).contains('3. Expected <c> to be equal to <C>, but was not.')
        assert_that(out).contains('4. Expected <b> to be equal to <B2>, but was not.')
        assert_that(out).contains('5. Expected <a> to be equal to <A2>, but was not.')
<commit_msg>Add test for soft_assert in a rcursive function<commit_after># Copyright (c) 2015-2018, Activision Publishing, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its contributors
# may be used to endorse or promote products derived from this software without
# specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from assertpy import assert_that, soft_assertions, fail

def test_success():
    with soft_assertions():
        assert_that('foo').is_length(3)
        assert_that('foo').is_not_empty()
        assert_that('foo').is_true()
        assert_that('foo').is_alpha()
        assert_that('123').is_digit()
        assert_that('foo').is_lower()
        assert_that('FOO').is_upper()
        assert_that('foo').is_equal_to('foo')
        assert_that('foo').is_not_equal_to('bar')
        assert_that('foo').is_equal_to_ignoring_case('FOO')
        assert_that({'a': 1}).has_a(1)

def test_failure():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4)
            assert_that('foo').is_empty()
            assert_that('foo').is_false()
            assert_that('foo').is_digit()
            assert_that('123').is_alpha()
            assert_that('foo').is_upper()
            assert_that('FOO').is_lower()
            assert_that('foo').is_equal_to('bar')
            assert_that('foo').is_not_equal_to('foo')
            assert_that('foo').is_equal_to_ignoring_case('BAR')
            assert_that({'a': 1}).has_a(2)
            assert_that({'a': 1}).has_foo(1)
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).contains('Expected <foo> to be empty string, but was not.')
        assert_that(out).contains('Expected <False>, but was not.')
        assert_that(out).contains('Expected <foo> to contain only digits, but did not.')
        assert_that(out).contains('Expected <123> to contain only alphabetic chars, but did not.')
        assert_that(out).contains('Expected <foo> to contain only uppercase chars, but did not.')
        assert_that(out).contains('Expected <FOO> to contain only lowercase chars, but did not.')
        assert_that(out).contains('Expected <foo> to be equal to <bar>, but was not.')
        assert_that(out).contains('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).contains('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')
        assert_that(out).contains('Expected <1> to be equal to <2> on key <a>, but was not.')
        assert_that(out).contains('Expected key <foo>, but val has no key <foo>.')

def test_failure_chain():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4).is_empty().is_false().is_digit().is_upper() \
                .is_equal_to('bar').is_not_equal_to('foo').is_equal_to_ignoring_case('BAR')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).contains('Expected <foo> to be empty string, but was not.')
        assert_that(out).contains('Expected <False>, but was not.')
        assert_that(out).contains('Expected <foo> to contain only digits, but did not.')
        assert_that(out).contains('Expected <foo> to contain only uppercase chars, but did not.')
        assert_that(out).contains('Expected <foo> to be equal to <bar>, but was not.')
        assert_that(out).contains('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).contains('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')

def test_expected_exception_success():
    with soft_assertions():
        assert_that(func_err).raises(RuntimeError).when_called_with('foo').is_equal_to('err')

def test_expected_exception_failure():
    try:
        with soft_assertions():
            assert_that(func_err).raises(RuntimeError).when_called_with('foo').is_equal_to('bar')
            assert_that(func_ok).raises(RuntimeError).when_called_with('baz')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('Expected <err> to be equal to <bar>, but was not.')
        assert_that(out).contains("Expected <func_ok> to raise <RuntimeError> when called with ('baz').")

def func_ok(arg):
    pass

def func_err(arg):
    raise RuntimeError('err')

def test_fail():
    try:
        with soft_assertions():
            fail()
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail!')

def test_fail_with_msg():
    try:
        with soft_assertions():
            fail('foobar')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail: foobar!')

def test_fail_with_soft_failing_asserts():
    try:
        with soft_assertions():
            assert_that('foo').is_length(4)
            assert_that('foo').is_empty()
            fail('foobar')
            assert_that('foo').is_not_equal_to('foo')
            assert_that('foo').is_equal_to_ignoring_case('BAR')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail: foobar!')
        assert_that(out).does_not_contain('Expected <foo> to be of length <4>, but was <3>.')
        assert_that(out).does_not_contain('Expected <foo> to be empty string, but was not.')
        assert_that(out).does_not_contain('Expected <foo> to be not equal to <foo>, but was.')
        assert_that(out).does_not_contain('Expected <foo> to be case-insensitive equal to <BAR>, but was not.')

def test_double_fail():
    try:
        with soft_assertions():
            fail()
            fail('foobar')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).is_equal_to('Fail!')

def test_nested():
    try:
        with soft_assertions():
            assert_that('a').is_equal_to('A')
            with soft_assertions():
                assert_that('b').is_equal_to('B')
                with soft_assertions():
                    assert_that('c').is_equal_to('C')
                assert_that('b').is_equal_to('B2')
            assert_that('a').is_equal_to('A2')
        fail('should have raised error')
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('1. Expected <a> to be equal to <A>, but was not.')
        assert_that(out).contains('2. Expected <b> to be equal to <B>, but was not.')
        assert_that(out).contains('3. Expected <c> to be equal to <C>, but was not.')
        assert_that(out).contains('4. Expected <b> to be equal to <B2>, but was not.')
        assert_that(out).contains('5. Expected <a> to be equal to <A2>, but was not.')

def test_recursive_nesting():
    def recursive(number):
        if number <= 0:
            return
        with soft_assertions():
            recursive(number-1)
            assert_that(number).is_equal_to(7)
    try:
        recursive(6)
    except AssertionError as e:
        out = str(e)
        assert_that(out).contains('1. Expected <1> to be equal to <7>, but was not.')
        assert_that(out).contains('2. Expected <2> to be equal to <7>, but was not.')
        assert_that(out).contains('3. Expected <3> to be equal to <7>, but was not.')
        assert_that(out).contains('4. Expected <4> to be equal to <7>, but was not.')
        assert_that(out).contains('5. Expected <5> to be equal to <7>, but was not.')
        assert_that(out).contains('6. Expected <6> to be equal to <7>, but was not.')

<|endoftext|>


<commit_before>#!/usr/bin/env python3

from functools import reduce
import itertools
from lxml import etree

class AnnotationFile:
    """Given a GATE XML annotation file, returns an AnnotationFile object.
    """
    def __init__(self, filename):
        self._filename = filename
        self._tree = etree.parse(self.filename)
        self._root = self.tree.getroot()
        self._nodes = None
        self._text_with_nodes = None
        self._annotations = None

    @property
    def filename(self):
        return self._filename

    @property
    def tree(self):
        return self._tree

    @property
    def root(self):
        return self._root

    @property
    def text(self):
        return "".join( self.text_with_nodes.itertext() )

    @property
    def nodes(self):
        if not self._nodes:
            nodes = self.text_with_nodes.getchildren()
            self._nodes =  {
                int(node.get("id")) : node for node in nodes
            }
            return self._nodes
        else:
            return self._nodes

    @property
    def text_with_nodes(self):
        if not self._text_with_nodes:
            self._text_with_nodes = self.root.find(".//TextWithNodes")
            return self._text_with_nodes
        else:
            return self._text_with_nodes

    @property
    def annotation_set_names(self):
        if not self._annotation_set_names:
            self._annotation_set_names = [
                annotation_set.get("Name")
                for annotation_set
                in self.root.findall(".//AnnotationSet")
            ]
            return self._annotation_set_names
        else:
            return self._annotation_set_names

    @property
    def annotations(self):
        if not self._annotations:
            self._annotations = [ x for x in self.iter_annotations() ]
            return self._annotations
        else:
            return self._annotations

    def iter_annotations(self):
        annotations = self.root.findall(
            ".//Annotation"
        )
        for x in annotations:
            yield Annotation(x)

class Annotation:
    def __init__(self, annotation):
        self._annotation_element = annotation
        self._type = annotation.get("Type")
        self._id = annotation.get("Id")
        self._start_node = int(annotation.get("StartNode"))
        self._end_node = int(annotation.get("EndNode"))
        self._continuations = []

        annotation_set_name = self._annotation_element.getparent().get("Name")
        if annotation_set_name:
            self._annotation_set_name = annotation_set_name
        else:
            self._annotation_set_name = ""

        if self._type == "Attribution":
            self._caused_event_id = None
            for feature in self.features:
                if feature.name.lower() == "caused_event":
                    self._caused_event_id = feature.value.split()[0]
                    break

    @property
    def type(self):
        return self._type

    @property
    def id(self):
        return self._id

    @property
    def start_node(self):
        return self._start_node

    @property
    def end_node(self):
        return self._end_node

    @property
    def features(self):
        return [ Feature(x) for x in self._annotation_element if x.tag == "Feature" ]

    @property
    def annotation_set_name(self):
        return self._annotation_set_name

    @property
    def continuations(self):
        return self._continuations

    def iter_spans(self):
        return itertools.chain( [self], ( x for x in self.continuations ) )

    def get_text(self, annotation_file):
        return "".join(
            annotation_file.nodes[node].tail
            for node in sorted(
                self.get_concatenated_char_set().intersection(
                    annotation_file.nodes.keys()
                )
            )
        )

    def get_concatenated_text(self, text_with_nodes, separator):
        return separator.join(
            x.get_text(text_with_nodes) for x in self.iter_spans()
        )

    def get_char_set(self):
        return frozenset(
            range(
                self.start_node,
                self.end_node
            )
        )

    def get_concatenated_char_set(self):
        if self.continuations:
            return reduce(
                lambda x,y : frozenset( x | y.get_char_set() ),
                self.iter_spans(),
                next(self.iter_spans()).get_char_set()
            )
        else: return self.get_char_set()

    def add_continuation(self, annotation):
        self._continuations.append(annotation)

class Feature:
    def __init__(self, feature):
        self._feature = feature
        self._name = feature.find("./Name")
        self._value = feature.find("./Value")

    @property
    def name(self):
        return self._name.text

    @name.setter
    def name(self, name):
        self._name.text = name

    @property
    def value(self):
        return self._value.text

    @value.setter
    def value(self, value):
        self._value.text = value

class Schema:
    def __init__(self, filename):
        self.filename = filename
        self.tree = etree.parse(self.filename)
        self.root = self.tree.getroot()
        self.namespace = {
            'schema':'http://www.w3.org/2000/10/XMLSchema'
        }

    def get_attributes(self, annotation_type):
        attributes = self.root.findall(
            ".//schema:element[@name='{}']"
            "//schema:attribute".format(annotation_type),
            namespaces=self.namespace
        )
        return attributes

def find_from_index(index,
                    source_list,
                    match_function,
                    reverse=False,
                    greedy=True):
    if reverse:
        try:
            list_from_index = source_list[index-1::-1]
        except IndexError:
            raise StopIteration()
    else:
        try:
            list_from_index = source_list[index+1::1]
        except IndexError:
            raise StopIteration()
    if greedy:
        for x in list_from_index:
            if match_function(x):
                yield x
    else:
        for x in list_from_index:
            if match_function(x):
                yield x
            else:
                raise StopIteration()

def concatenate_annotations(annotation_iterable):
    """Given an iterable of Annotation objects, returns a list of Annotations
    objects such that each Annotation's continuations list is populated
    appropriately, less all continuation annotations
    """

    annotations = sorted(
        sorted(
            annotation_iterable,
            key=(lambda x: x.annotation_set_name)
        ),
        key=(lambda x: x.end_node)
    )

    for i, annotation in enumerate(annotations):
        if "_continuation" in annotation.type:
            continuation = annotation
            base_annotation_type = (
                continuation.type.replace("_continuation","")
            )
            continued_annotation = next(
                find_from_index(
                    i,
                    annotations,
                    lambda x : x.type == base_annotation_type,
                    reverse=True,
                )
            )
            continued_annotation.add_continuation(annotation)

    return [
        annotation
        for annotation in annotations
        if not annotation.type.endswith("_continuation")
    ]

def filter_annotations_by_type(annotation_iterable,
                               annotation_types,
                               with_continuations=False):
    """Given an iterable of Annotation objects, return a generator which yields
    all Annotations of the given type(s)"""
    base_types = [ x.lower() for x in annotation_types ]
    key_types = []
    for x in base_types:
        key_types.append(x)
        if with_continuations:
            key_types.append(x + "_continuation")

    return [
        annotation
        for annotation in annotation_iterable
        if annotation.type.lower() in key_types
    ]

def get_feature_by_name(name,
                        annotation):
    return next(
        (
            feature
            for feature in annotation.features
            if name.lower() in feature.name.lower()
        ),
        None
    )

def is_overlapping(annotations):
    if len(annotations) == 0:
        raise Exception("Can't compare to nothing!")
    return all(
        not (
            annotation
            .get_concatenated_char_set()
            .isdisjoint(
                annotations[i+1]
                .get_concatenated_char_set()
            )
        )
        for i, annotation in enumerate( annotations[:-1] )
    )
<commit_msg>Correct functioning of get_text; Add default separator to get_concatenated_text<commit_after>#!/usr/bin/env python3

from functools import reduce
import itertools
from lxml import etree

class AnnotationFile:
    """Given a GATE XML annotation file, returns an AnnotationFile object.
    """
    def __init__(self, filename):
        self._filename = filename
        self._tree = etree.parse(self.filename)
        self._root = self.tree.getroot()
        self._nodes = None
        self._text_with_nodes = None
        self._annotations = None

    @property
    def filename(self):
        return self._filename

    @property
    def tree(self):
        return self._tree

    @property
    def root(self):
        return self._root

    @property
    def text(self):
        return "".join( self.text_with_nodes.itertext() )

    @property
    def nodes(self):
        if not self._nodes:
            nodes = self.text_with_nodes.getchildren()
            self._nodes =  {
                int(node.get("id")) : node for node in nodes
            }
            return self._nodes
        else:
            return self._nodes

    @property
    def text_with_nodes(self):
        if not self._text_with_nodes:
            self._text_with_nodes = self.root.find(".//TextWithNodes")
            return self._text_with_nodes
        else:
            return self._text_with_nodes

    @property
    def annotation_set_names(self):
        if not self._annotation_set_names:
            self._annotation_set_names = [
                annotation_set.get("Name")
                for annotation_set
                in self.root.findall(".//AnnotationSet")
            ]
            return self._annotation_set_names
        else:
            return self._annotation_set_names

    @property
    def annotations(self):
        if not self._annotations:
            self._annotations = [ x for x in self.iter_annotations() ]
            return self._annotations
        else:
            return self._annotations

    def iter_annotations(self):
        annotations = self.root.findall(
            ".//Annotation"
        )
        for x in annotations:
            yield Annotation(x)

class Annotation:
    def __init__(self, annotation):
        self._annotation_element = annotation
        self._type = annotation.get("Type")
        self._id = annotation.get("Id")
        self._start_node = int(annotation.get("StartNode"))
        self._end_node = int(annotation.get("EndNode"))
        self._continuations = []

        annotation_set_name = self._annotation_element.getparent().get("Name")
        if annotation_set_name:
            self._annotation_set_name = annotation_set_name
        else:
            self._annotation_set_name = ""

        if self._type == "Attribution":
            self._caused_event_id = None
            for feature in self.features:
                if feature.name.lower() == "caused_event":
                    self._caused_event_id = feature.value.split()[0]
                    break

    @property
    def type(self):
        return self._type

    @property
    def id(self):
        return self._id

    @property
    def start_node(self):
        return self._start_node

    @property
    def end_node(self):
        return self._end_node

    @property
    def features(self):
        return [ Feature(x) for x in self._annotation_element if x.tag == "Feature" ]

    @property
    def annotation_set_name(self):
        return self._annotation_set_name

    @property
    def continuations(self):
        return self._continuations

    def iter_spans(self):
        return itertools.chain( [self], ( x for x in self.continuations ) )

    def get_text(self, annotation_file):
        return "".join(
            annotation_file.nodes[node].tail
            for node in sorted(
                self.get_char_set().intersection(
                    annotation_file.nodes.keys()
                )
            )
        )

    def get_concatenated_text(self,
                              annotation_file,
                              separator=None):
        if not separator:
            separator = " "
        return separator.join(
            x.get_text(annotation_file) for x in self.iter_spans()
        )

    def get_char_set(self):
        return frozenset(
            range(
                self.start_node,
                self.end_node
            )
        )

    def get_concatenated_char_set(self):
        if self.continuations:
            return reduce(
                lambda x,y : frozenset( x | y.get_char_set() ),
                self.iter_spans(),
                next(self.iter_spans()).get_char_set()
            )
        else: return self.get_char_set()

    def add_continuation(self, annotation):
        self._continuations.append(annotation)

class Feature:
    def __init__(self, feature):
        self._feature = feature
        self._name = feature.find("./Name")
        self._value = feature.find("./Value")

    @property
    def name(self):
        return self._name.text

    @name.setter
    def name(self, name):
        self._name.text = name

    @property
    def value(self):
        return self._value.text

    @value.setter
    def value(self, value):
        self._value.text = value

class Schema:
    def __init__(self, filename):
        self.filename = filename
        self.tree = etree.parse(self.filename)
        self.root = self.tree.getroot()
        self.namespace = {
            'schema':'http://www.w3.org/2000/10/XMLSchema'
        }

    def get_attributes(self, annotation_type):
        attributes = self.root.findall(
            ".//schema:element[@name='{}']"
            "//schema:attribute".format(annotation_type),
            namespaces=self.namespace
        )
        return attributes

def find_from_index(index,
                    source_list,
                    match_function,
                    reverse=False,
                    greedy=True):
    if reverse:
        try:
            list_from_index = source_list[index-1::-1]
        except IndexError:
            raise StopIteration()
    else:
        try:
            list_from_index = source_list[index+1::1]
        except IndexError:
            raise StopIteration()
    if greedy:
        for x in list_from_index:
            if match_function(x):
                yield x
    else:
        for x in list_from_index:
            if match_function(x):
                yield x
            else:
                raise StopIteration()

def concatenate_annotations(annotation_iterable):
    """Given an iterable of Annotation objects, returns a list of Annotations
    objects such that each Annotation's continuations list is populated
    appropriately, less all continuation annotations
    """

    annotations = sorted(
        sorted(
            annotation_iterable,
            key=(lambda x: x.annotation_set_name)
        ),
        key=(lambda x: x.end_node)
    )

    for i, annotation in enumerate(annotations):
        if "_continuation" in annotation.type:
            continuation = annotation
            base_annotation_type = (
                continuation.type.replace("_continuation","")
            )
            continued_annotation = next(
                find_from_index(
                    i,
                    annotations,
                    lambda x : x.type == base_annotation_type,
                    reverse=True,
                )
            )
            continued_annotation.add_continuation(annotation)

    return [
        annotation
        for annotation in annotations
        if not annotation.type.endswith("_continuation")
    ]

def filter_annotations_by_type(annotation_iterable,
                               annotation_types,
                               with_continuations=False):
    """Given an iterable of Annotation objects, return a generator which yields
    all Annotations of the given type(s)"""
    base_types = [ x.lower() for x in annotation_types ]
    key_types = []
    for x in base_types:
        key_types.append(x)
        if with_continuations:
            key_types.append(x + "_continuation")

    return [
        annotation
        for annotation in annotation_iterable
        if annotation.type.lower() in key_types
    ]

def get_feature_by_name(name,
                        annotation):
    return next(
        (
            feature
            for feature in annotation.features
            if name.lower() in feature.name.lower()
        ),
        None
    )

def is_overlapping(annotations):
    if len(annotations) == 0:
        raise Exception("Can't compare to nothing!")
    return all(
        not (
            annotation
            .get_concatenated_char_set()
            .isdisjoint(
                annotations[i+1]
                .get_concatenated_char_set()
            )
        )
        for i, annotation in enumerate( annotations[:-1] )
    )
<|endoftext|>


<commit_before>import os
import string
import sys
import pysftp
import csv
import datetime
from datetime import date
from datetime import datetime, timedelta
import codecs
import time
from time import strftime
import argparse
#import uuid
from sqlalchemy import text
import shutil
#import re
# import logging
# from logging.handlers import SMTPHandler

# python path
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
sys.path.append('/djequis/adp')

# django settings for shell environment
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djequis.settings")

# prime django
import django
django.setup()

# django settings for script
from django.conf import settings
from django.db import connections

# informix environment
os.environ['INFORMIXSERVER'] = settings.INFORMIXSERVER
os.environ['DBSERVERNAME'] = settings.DBSERVERNAME
os.environ['INFORMIXDIR'] = settings.INFORMIXDIR
os.environ['ODBCINI'] = settings.ODBCINI
os.environ['ONCONFIG'] = settings.ONCONFIG
os.environ['INFORMIXSQLHOSTS'] = settings.INFORMIXSQLHOSTS
os.environ['LD_LIBRARY_PATH'] = settings.LD_LIBRARY_PATH
os.environ['LD_RUN_PATH'] = settings.LD_RUN_PATH

# from djequis.core.utils import sendmail
from djzbar.utils.informix import do_sql
from djzbar.utils.informix import get_engine
from djzbar.settings import INFORMIX_EARL_TEST
from djzbar.settings import INFORMIX_EARL_PROD
from djzbar.settings import INFORMIX_EARL_SANDBOX

from djtools.fields import TODAY

# Imports for additional modules and functions written as part of this project
from djequis.adp.utilities import fn_validate_field, fn_check_duplicates, \
    fn_write_log

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""
# write out the .sql file
scr = open("apdtocx_output.sql", "a")
# set start_time in order to see how long script takes to execute
start_time = time.time()

################################################
# Start of processing
################################################
def fn_process_cvid(carthid, adpid, ssn, adp_assoc_id, EARL):
    engine = get_engine(EARL)

    try:
        ##############################################################
        # Inserts or updates as needed into cvid_rec
        ##############################################################

        # Validate the cx_id
        v_cx_id = fn_validate_field(carthid, "cx_id", "cx_id", "cvid_rec",
                    "integer", EARL)
        #print("CX_ID = " + str(carthid))
        #print("v_CX_ID = " + str(v_cx_id))

        # Should also check for duplicates of the adp_id and associate_id
        # What to do in that case?
        v_adp_match = fn_check_duplicates(adpid, "adp_id", "cx_id", "cvid_rec",
                                 v_cx_id, "char", EARL)
        #print("Found_ID = " + str(v_adp_match))

        # By definition, associate ID cannot be a duplicate in ADP, but
        # possibility of duplicate might exist in CX
        v_assoc_match = fn_check_duplicates(adp_assoc_id, "adp_associate_id",
                       "cx_id", "cvid_rec", v_cx_id, "char", EARL)
        #print("Found ID = " + str(v_assoc_match))

        if v_cx_id == 0 and v_assoc_match == 0 and v_adp_match == 0:
            # Insert or update as needed to ID_rec
            q_insert_cvid_rec = '''
              INSERT INTO cvid_rec (old_id, old_id_num, adp_id, ssn, cx_id,
              cx_id_char, adp_associate_id)
              VALUES (?,?,?,?,?,?,?)'''
            args = (carthid, carthid, adpid, ssn, carthid, carthid, adp_assoc_id)
            # print(q_insert_cvid_rec)
            engine.execute(q_insert_cvid_rec, args)
            scr.write(q_insert_cvid_rec + '\n');
            fn_write_log("Inserted into cvid_rec table")
            # logger.info("Inserted into cvid_rec table");
        elif str(v_cx_id) != v_assoc_match and v_assoc_match != 0:
            print('Duplicate Associate ID found')
        elif str(v_cx_id) != str(v_adp_match) and v_adp_match != 0:
            print('Duplicate ADP ID found')
        else:
            q_update_cvid_rec = '''
              UPDATE cvid_rec
              SET old_id = ?, old_id_num = ?, adp_id = ?, ssn = ?,
              adp_associate_id = ?
              WHERE cx_id = ?'''
            args = (carthid, carthid, adpid, ssn, adp_assoc_id, carthid)
            # print(q_update_cvid_rec)
            # fn_write_log("Update cvid_rec table")
            # logger.info("Update cvid_rec table");
            scr.write(q_update_cvid_rec + '\n');
            engine.execute(q_update_cvid_rec, args)

        return 1

    except Exception as e:
        print(e)
        return 0
    # finally:
    #     logging.shutdown()
<commit_msg>Expanded some log messages, did some debugging<commit_after>import os
import string
import sys
import pysftp
import csv
import datetime
from datetime import date
from datetime import datetime, timedelta
import codecs
import time
from time import strftime
import argparse
#import uuid
from sqlalchemy import text
import shutil
#import re
# import logging
# from logging.handlers import SMTPHandler

# python path
sys.path.append('/usr/lib/python2.7/dist-packages/')
sys.path.append('/usr/lib/python2.7/')
sys.path.append('/usr/local/lib/python2.7/dist-packages/')
sys.path.append('/data2/django_1.11/')
sys.path.append('/data2/django_projects/')
sys.path.append('/data2/django_third/')
sys.path.append('/djequis/adp')

# django settings for shell environment
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "djequis.settings")

# prime django
import django
django.setup()

# django settings for script
from django.conf import settings
from django.db import connections

# informix environment
os.environ['INFORMIXSERVER'] = settings.INFORMIXSERVER
os.environ['DBSERVERNAME'] = settings.DBSERVERNAME
os.environ['INFORMIXDIR'] = settings.INFORMIXDIR
os.environ['ODBCINI'] = settings.ODBCINI
os.environ['ONCONFIG'] = settings.ONCONFIG
os.environ['INFORMIXSQLHOSTS'] = settings.INFORMIXSQLHOSTS
os.environ['LD_LIBRARY_PATH'] = settings.LD_LIBRARY_PATH
os.environ['LD_RUN_PATH'] = settings.LD_RUN_PATH

# from djequis.core.utils import sendmail
from djzbar.utils.informix import do_sql
from djzbar.utils.informix import get_engine
from djzbar.settings import INFORMIX_EARL_TEST
from djzbar.settings import INFORMIX_EARL_PROD
from djzbar.settings import INFORMIX_EARL_SANDBOX

from djtools.fields import TODAY

# Imports for additional modules and functions written as part of this project
from djequis.adp.utilities import fn_validate_field, fn_check_duplicates, \
    fn_write_log, fn_write_error

DEBUG = settings.INFORMIX_DEBUG

# set up command-line options
desc = """
    Upload ADP data to CX
"""
# write out the .sql file
scr = open("apdtocx_output.sql", "a")
# set start_time in order to see how long script takes to execute
start_time = time.time()

################################################
# Start of processing
################################################
def fn_process_cvid(carthid, adpid, ssn, adp_assoc_id, EARL):
    engine = get_engine(EARL)

    try:
        ##############################################################
        # Inserts or updates as needed into cvid_rec
        ##############################################################

        # Validate the cx_id
        v_cx_id = fn_validate_field(carthid, "cx_id", "cx_id", "cvid_rec",
                    "integer", EARL)
        #print("CX_ID = " + str(carthid))
        #print("v_CX_ID = " + str(v_cx_id))

        # Should also check for duplicates of the adp_id and associate_id
        # What to do in that case?
        v_adp_match = fn_check_duplicates(adpid, "adp_id", "cx_id", "cvid_rec",
                                 v_cx_id, "char", EARL)
        #print("Found_ID = " + str(v_adp_match))

        # By definition, associate ID cannot be a duplicate in ADP, but
        # possibility of duplicate might exist in CX
        v_assoc_match = fn_check_duplicates(adp_assoc_id, "adp_associate_id",
                       "cx_id", "cvid_rec", v_cx_id, "char", EARL)
        #print("Found ID = " + str(v_assoc_match))

        if v_cx_id == 0 and v_assoc_match == 0 and v_adp_match == 0:
            # Insert or update as needed to ID_rec
            q_insert_cvid_rec = '''
              INSERT INTO cvid_rec (old_id, old_id_num, adp_id, ssn, cx_id,
              cx_id_char, adp_associate_id)
              VALUES (?,?,?,?,?,?,?)'''
            args = (carthid, carthid, adpid, ssn, carthid, carthid, adp_assoc_id)
            # print(q_insert_cvid_rec)
            engine.execute(q_insert_cvid_rec, args)
            scr.write(q_insert_cvid_rec + '\n');
            fn_write_log("Inserted into cvid_rec table values " + str(v_cx_id) + ", " + str(adpid) + ", " + adp_assoc_id)

        elif str(v_cx_id) != v_assoc_match and v_assoc_match != 0:
            print('Duplicate Associate ID found')
        elif str(v_cx_id) != str(v_adp_match) and v_adp_match != 0:
            print('Duplicate ADP ID found')
        else:
            q_update_cvid_rec = '''
              UPDATE cvid_rec
              SET old_id = ?, old_id_num = ?, adp_id = ?, ssn = ?,
              adp_associate_id = ?
              WHERE cx_id = ?'''
            args = (carthid, carthid, adpid, ssn, adp_assoc_id, carthid)
            # print(q_update_cvid_rec)
            fn_write_log("Updated cvid_rec table, values " + str(v_cx_id) + ", " + str(adpid) + ", " + adp_assoc_id)
            # logger.info("Update cvid_rec table");
            scr.write(q_update_cvid_rec + '\n');
            engine.execute(q_update_cvid_rec, args)

        return 1

    except Exception as e:
        print(e)
        fn_write_error("Error in cvidrec.py.  Error = " + e.message)
        return 0
    # finally:
    #     logging.shutdown()
<|endoftext|>


<commit_before>import unittest
from checks.jmx_connector import JmxCheck
import logging
import subprocess
import time
import urllib2
from nose.plugins.skip import SkipTest

from tests.common import kill_subprocess, load_check

class JMXTestCase(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        pass

    def testSolrMetrics(self):
        agentConfig = {
            'solr_jmx_instance_1': 'localhost:8090',
            'solr_jmx_instance_2': 'dummyurl:4444:fake_url',
            'version': '0.1',
            'api_key': 'toto'
        }

        config = JmxCheck.parse_agent_config(agentConfig, 'solr')
        config['init_config'] = SOLR_CONFIG
        config['init_config']['domains'] = ['solr/']

        metrics_check = load_check('solr', config, agentConfig)

        timers_first_check = []

        for instance in config['instances']:
            try:
                start = time.time()
                metrics_check.check(instance)
                timers_first_check.append(time.time() - start)
            except Exception,e:
                print e
                continue

        metrics = metrics_check.get_metrics()

        self.assertTrue(type(metrics) == type([]))
        self.assertTrue(len(metrics) > 8, metrics)
        self.assertEquals(len([t for t in metrics if t[3].get('device_name') == "solr" and t[0] == "jvm.thread_count"]), 1, metrics)
        self.assertTrue(len([t for t in metrics if "jvm." in t[0]]) > 4, [t for t in metrics if "jvm." in t[0]])
        self.assertTrue(len([t for t in metrics if "solr." in t[0]]) > 4, [t for t in metrics if "solr." in t[0]])

        timers_second_check = []
        for instance in config['instances']:
            try:
                start = time.time()
                metrics_check.check(instance)
                timers_second_check.append(time.time() - start)
            except Exception,e:
                continue

        self.assertEquals(len([t for t in timers_first_check if t > 10]), 0, timers_first_check)
        self.assertEquals(len([t for t in timers_second_check if t > 2]), 0, timers_second_check)

        metrics_check.kill_jmx_connectors()

SOLR_CONFIG = {
'conf':
[{'include':
{'attribute':
{'maxDoc':
{'alias': 'solr.searcher.maxdoc',
'metric_type': 'gauge'},
'numDocs':
{'alias': 'solr.searcher.numdocs',
'metric_type': 'gauge'},
'warmupTime':
{'alias': 'solr.searcher.warmup',
'metric_type': 'gauge'}},
'type': 'searcher'}},
{'include':
{'attribute':
{'cumulative_evictions':
{'alias': 'solr.cache.evictions',
'metric_type': 'counter'},
'cumulative_hits': {'alias': 'solr.cache.hits',
'metric_type': 'counter'},
'cumulative_inserts': {'alias': 'solr.cache.inserts',
'metric_type': 'counter'},
'cumulative_lookups': {'alias': 'solr.cache.lookups',
'metric_type': 'counter'}},
'id': 'org.apache.solr.search.FastLRUCache'}},
{'include': {'attribute': {'cumulative_evictions': {'alias': 'solr.cache.evictions',
'metric_type': 'counter'},
'cumulative_hits': {'alias': 'solr.cache.hits',
'metric_type': 'counter'},
'cumulative_inserts': {'alias': 'solr.cache.inserts',
'metric_type': 'counter'},
'cumulative_lookups': {'alias': 'solr.cache.lookups',
'metric_type': 'counter'}},
'id': 'org.apache.solr.search.LRUCache'}},
{'include': {'attribute': {'avgRequestsPerSecond': {'alias': 'solr.search_handler.avg_requests_per_sec',
'metric_type': 'gauge'},
'avgTimePerRequest': {'alias': 'solr.search_handler.avg_time_per_req',
'metric_type': 'gauge'},
'errors': {'alias': 'solr.search_handler.errors',
'metric_type': 'counter'},
'requests': {'alias': 'solr.search_handler.requests',
'metric_type': 'counter'},
'timeouts': {'alias': 'solr.search_handler.timeouts',
'metric_type': 'counter'},
'totalTime': {'alias': 'solr.search_handler.time',
'metric_type': 'counter'}},
'id': 'org.apache.solr.handler.component.SearchHandler'}}]}
if __name__ == "__main__":
    unittest.main()
<commit_msg>Fix to the travis test<commit_after>import unittest
from checks.jmx_connector import JmxCheck
import logging
import subprocess
import time
import urllib2
from nose.plugins.skip import SkipTest

from tests.common import kill_subprocess, load_check

class JMXTestCase(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        pass

    def testSolrMetrics(self):
        agentConfig = {
            'solr_jmx_instance_1': 'localhost:8090',
            'solr_jmx_instance_2': 'dummyurl:4444:fake_url',
            'version': '0.1',
            'api_key': 'toto'
        }

        config = JmxCheck.parse_agent_config(agentConfig, 'solr')
        config['init_config'] = SOLR_CONFIG

        metrics_check = load_check('solr', config, agentConfig)

        timers_first_check = []

        for instance in config['instances']:
            try:
                start = time.time()
                metrics_check.check(instance)
                timers_first_check.append(time.time() - start)
            except Exception,e:
                print e
                continue

        metrics = metrics_check.get_metrics()

        self.assertTrue(type(metrics) == type([]))
        self.assertTrue(len(metrics) > 8, metrics)
        self.assertEquals(len([t for t in metrics if t[3].get('device_name') == "solr" and t[0] == "jvm.thread_count"]), 1, metrics)
        self.assertTrue(len([t for t in metrics if "jvm." in t[0]]) > 4, [t for t in metrics if "jvm." in t[0]])
        self.assertTrue(len([t for t in metrics if "solr." in t[0]]) > 4, [t for t in metrics if "solr." in t[0]])

        timers_second_check = []
        for instance in config['instances']:
            try:
                start = time.time()
                metrics_check.check(instance)
                timers_second_check.append(time.time() - start)
            except Exception,e:
                continue

        self.assertEquals(len([t for t in timers_first_check if t > 10]), 0, timers_first_check)
        self.assertEquals(len([t for t in timers_second_check if t > 2]), 0, timers_second_check)

        metrics_check.kill_jmx_connectors()

SOLR_CONFIG = {
'conf':
[{'include':
{'attribute':
{'maxDoc':
{'alias': 'solr.searcher.maxdoc',
'metric_type': 'gauge'},
'numDocs':
{'alias': 'solr.searcher.numdocs',
'metric_type': 'gauge'},
'warmupTime':
{'alias': 'solr.searcher.warmup',
'metric_type': 'gauge'}},
'type': 'searcher'}},
{'include':
{'attribute':
{'cumulative_evictions':
{'alias': 'solr.cache.evictions',
'metric_type': 'counter'},
'cumulative_hits': {'alias': 'solr.cache.hits',
'metric_type': 'counter'},
'cumulative_inserts': {'alias': 'solr.cache.inserts',
'metric_type': 'counter'},
'cumulative_lookups': {'alias': 'solr.cache.lookups',
'metric_type': 'counter'}},
'id': 'org.apache.solr.search.FastLRUCache'}},
{'include': {'attribute': {'cumulative_evictions': {'alias': 'solr.cache.evictions',
'metric_type': 'counter'},
'cumulative_hits': {'alias': 'solr.cache.hits',
'metric_type': 'counter'},
'cumulative_inserts': {'alias': 'solr.cache.inserts',
'metric_type': 'counter'},
'cumulative_lookups': {'alias': 'solr.cache.lookups',
'metric_type': 'counter'}},
'id': 'org.apache.solr.search.LRUCache'}},
{'include': {'attribute': {'avgRequestsPerSecond': {'alias': 'solr.search_handler.avg_requests_per_sec',
'metric_type': 'gauge'},
'avgTimePerRequest': {'alias': 'solr.search_handler.avg_time_per_req',
'metric_type': 'gauge'},
'errors': {'alias': 'solr.search_handler.errors',
'metric_type': 'counter'},
'requests': {'alias': 'solr.search_handler.requests',
'metric_type': 'counter'},
'timeouts': {'alias': 'solr.search_handler.timeouts',
'metric_type': 'counter'},
'totalTime': {'alias': 'solr.search_handler.time',
'metric_type': 'counter'}},
'id': 'org.apache.solr.handler.component.SearchHandler'}}]}
if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import gdata.calendar.client
import gdata.client

class Calendar:
    def __init__(self, username, password, ignore_events):
        try:
            self.username = username[:username.find('@') + 1]
            self.cal_client = gdata.calendar.client.CalendarClient(source='Google-Calendar_Python_Sample-1.0')
            self.cal_client.ClientLogin(username, password, self.cal_client.source)
            self.ignore_events = ignore_events
        except gdata.client.BadAuthentication:
            raise Exception('ERROR: login to Google failed (check credentials)')

    def get_events(self, start_date, end_date):
        """
        Get events from calendar between given dates.
        """
        query = gdata.calendar.client.CalendarEventQuery(start_min=start_date, start_max=end_date, singleevents="true")
        feed = self.cal_client.GetCalendarEventFeed(q=query)
        for i, an_event in zip(range(len(feed.entry)), feed.entry):
            for who in [x for x in an_event.who if self.username in x.email]:
                if who.attendee_status is not None and "declined" not in who.attendee_status.value:
                    for an_when in [x for x in an_event.when if an_event.title.text not in self.ignore_events]:
                        yield {
                            'start_date': an_when.start,
                            'end_date': an_when.end,
                            'title': an_event.title.text
                        }
<commit_msg>Encode calendar event to utf-8<commit_after>import gdata.calendar.client
import gdata.client

class Calendar:
    def __init__(self, username, password, ignore_events):
        try:
            self.username = username[:username.find('@') + 1]
            self.cal_client = gdata.calendar.client.CalendarClient(source='Google-Calendar_Python_Sample-1.0')
            self.cal_client.ClientLogin(username, password, self.cal_client.source)
            self.ignore_events = ignore_events
        except gdata.client.BadAuthentication:
            raise Exception('ERROR: login to Google failed (check credentials)')

    def get_events(self, start_date, end_date):
        """
        Get events from calendar between given dates.
        """
        query = gdata.calendar.client.CalendarEventQuery(start_min=start_date, start_max=end_date, singleevents="true")
        feed = self.cal_client.GetCalendarEventFeed(q=query)
        for i, an_event in zip(range(len(feed.entry)), feed.entry):
            for who in [x for x in an_event.who if self.username in x.email]:
                if who.attendee_status is not None and "declined" not in who.attendee_status.value:
                    for an_when in [x for x in an_event.when if an_event.title.text not in self.ignore_events]:
                        yield {
                            'start_date': an_when.start,
                            'end_date': an_when.end,
                            'title': an_event.title.text.encode('utf8')
                        }
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    vcs
    ~~~

    Various version Control System (vcs) management abstraction layer for
    Python.

    :created_on: Apr 8, 2010
    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
"""

VERSION = (0, 2, 3, 'dev')

__version__ = '.'.join((str(each) for each in VERSION[:4]))

__all__ = [
    'get_version', 'get_repo', 'get_backend',
    'VCSError', 'RepositoryError', 'ChangesetError']

import sys
from vcs.backends import get_repo, get_backend
from vcs.exceptions import VCSError, RepositoryError, ChangesetError

def get_version():
    """
    Returns shorter version (digit parts only) as string.
    """
    return '.'.join((str(each) for each in VERSION[:3]))

def main(argv=None):
    if argv is None:
        argv = sys.argv
    from vcs.cli import ExecutionManager
    manager = ExecutionManager(argv)
    manager.execute()
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<commit_msg>version bump<commit_after># -*- coding: utf-8 -*-
"""
    vcs
    ~~~

    Various version Control System (vcs) management abstraction layer for
    Python.

    :created_on: Apr 8, 2010
    :copyright: (c) 2010-2011 by Marcin Kuzminski, Lukasz Balcerzak.
"""

VERSION = (0, 3, 0, 'dev')

__version__ = '.'.join((str(each) for each in VERSION[:4]))

__all__ = [
    'get_version', 'get_repo', 'get_backend',
    'VCSError', 'RepositoryError', 'ChangesetError']

import sys
from vcs.backends import get_repo, get_backend
from vcs.exceptions import VCSError, RepositoryError, ChangesetError

def get_version():
    """
    Returns shorter version (digit parts only) as string.
    """
    return '.'.join((str(each) for each in VERSION[:3]))

def main(argv=None):
    if argv is None:
        argv = sys.argv
    from vcs.cli import ExecutionManager
    manager = ExecutionManager(argv)
    manager.execute()
    return 0

if __name__ == '__main__':
    sys.exit(main(sys.argv))
<|endoftext|>