<commit_before>import unittest

from datapunt_geosearch import config
from datapunt_geosearch import datasource

class TestBAGDataset(unittest.TestCase):
    def test_query(self):
        x = 120993
        y = 485919

        ds = datasource.BagDataSource(dsn=config.DSN_BAG)
        results = ds.query(x, y)

        self.assertEqual(len(results['features']), 7)
        self.assertIn('distance', results['features'][0]['properties'])

    def test_query_wgs84(self):
        x = 52.36011
        y = 4.88798

        ds = datasource.BagDataSource(dsn=config.DSN_BAG)
        results = ds.query(x, y, rd=False)

        self.assertEqual(len(results['features']), 7)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Remove WKPB from geosearch - also change test<commit_after>import unittest

from datapunt_geosearch import config
from datapunt_geosearch import datasource

class TestBAGDataset(unittest.TestCase):
    def test_query(self):
        x = 120993
        y = 485919

        ds = datasource.BagDataSource(dsn=config.DSN_BAG)
        results = ds.query(x, y)

        self.assertEqual(len(results['features']), 7)
        self.assertIn('distance', results['features'][0]['properties'])

    def test_query_wgs84(self):
        x = 52.36011
        y = 4.88798

        ds = datasource.BagDataSource(dsn=config.DSN_BAG)
        results = ds.query(x, y, rd=False)

        self.assertEqual(len(results['features']), 6)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import time
import al, AL
import string

dev = AL.DEFAULT_DEVICE

parameter_name = [ \
	'input source', 'left input atten', 'right input atten', \
	'input rate', 'output rate', \
	'left speaker gain', 'right speaker gain', \
	'input count', 'output count', 'unused count', \
	'sync input to aes', 'sync output to aes', \
	]
source_name = ['line', 'microphone', 'digital']

params = al.queryparams(dev)
for i in range(1, len(params), 2):
	params[i] = -1
while 1:
	time.millisleep(100)
	old = params[:]
	al.getparams(dev, params)
	if params <> old:
		for i in range(0, len(params), 2):
			if params[i+1] <> old[i+1]:
				name = parameter_name[params[i]]
				if params[i] == AL.INPUT_SOURCE:
					if 0 <= old[i+1] < len(source_name):
						oldval = source_name[old[i+1]]
					else:
						oldval = ''
					newval = source_name[params[i+1]]
				else:
					oldval = `old[i+1]`
					newval = `params[i+1]`
				print string.ljust(name, 20),
				print '(' + string.rjust(oldval, 10) + ')',
				print '-->',
				print string.rjust(newval, 10)
		print
<commit_msg>Get official parameter names<commit_after>import time
import al, AL
import string

dev = AL.DEFAULT_DEVICE

source_name = ['line', 'microphone', 'digital']

params = al.queryparams(dev)
for i in range(1, len(params), 2):
	params[i] = -1
while 1:
	time.millisleep(100)
	old = params[:]
	al.getparams(dev, params)
	if params <> old:
		for i in range(0, len(params), 2):
			if params[i+1] <> old[i+1]:
				name = al.getname(dev, params[i])
				if params[i] == AL.INPUT_SOURCE:
					if 0 <= old[i+1] < len(source_name):
						oldval = source_name[old[i+1]]
					else:
						oldval = ''
					newval = source_name[params[i+1]]
				else:
					oldval = `old[i+1]`
					newval = `params[i+1]`
				print string.ljust(name, 25),
				print '(' + string.rjust(oldval, 10) + ')',
				print '-->',
				print string.rjust(newval, 10)
		print
<|endoftext|>


<commit_before># This script downloads yearly malaria statistics from data.go.id
# It uses urllib and is compatible with both Python 2 and 3

import os
import sys
import logging #logs what goes on

DIRECTORY = '../../data/raw/disease_ID'
OUTFILE = "yearly-malaria.csv"
OUTFILE2015 = "malaria-2015.xlsx"
URL = "http://data.go.id/dataset/cef9b348-91a9-4270-be1d-3cf64eb9d5b0/resource/42f31bb0-af59-4c96-9a74-db3283f9e316/download/kasusmalaria.csv"
URL2015 = "http://data.go.id/dataset/cef9b348-91a9-4270-be1d-3cf64eb9d5b0/resource/2965b760-0f7f-4bd7-9dbe-8d261729e12f/download/jumlahkasusangkakesakitanmalariaper1000pendudukberisiko.xlsx"

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

def download():

    """ Download malaria statistics from data.go.id """
    logger.info("Downloading raw data of Malaria statistics in Indonesia between 2010 - 2015")

    # compatibility check between python 2 and 3
    if sys.version_info < (3, 0):
        # for python 2, use this
        try:
            os.makedirs(DIRECTORY)
        except OSError as e:
            pass
        import urllib as downloader
        from urllib2 import URLError, HTTPError
    else:
        # for python 3, use this
        os.makedirs(DIRECTORY, exist_ok=True)
        import urllib.request as downloader
        from urllib.error import URLError, HTTPError

    output_path = os.path.join(DIRECTORY, OUTFILE)
    output_path_2015 = os.path.join(DIRECTORY, OUTFILE2015)

    # now retrieve the file
    try:
        downloader.urlretrieve(URL, output_path)
        logger.info('Downloaded yearly stats successfully to %s', os.path.abspath(output_path))

        downloader.urlretrieve(URL2015, output_path_2015)
        logger.info('Downloaded 2015 stats successfully to %s', os.path.abspath(output_path_2015))
    except (HTTPError, URLError) as e:
        logger.error('Failed to download: %s', e.reason)

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    download()
<commit_msg>remove empty line<commit_after># This script downloads yearly malaria statistics from data.go.id
# It uses urllib and is compatible with both Python 2 and 3

import os
import sys
import logging #logs what goes on

DIRECTORY = '../../data/raw/disease_ID'
OUTFILE = "yearly-malaria.csv"
OUTFILE2015 = "malaria-2015.xlsx"
URL = "http://data.go.id/dataset/cef9b348-91a9-4270-be1d-3cf64eb9d5b0/resource/42f31bb0-af59-4c96-9a74-db3283f9e316/download/kasusmalaria.csv"
URL2015 = "http://data.go.id/dataset/cef9b348-91a9-4270-be1d-3cf64eb9d5b0/resource/2965b760-0f7f-4bd7-9dbe-8d261729e12f/download/jumlahkasusangkakesakitanmalariaper1000pendudukberisiko.xlsx"

logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())

def download():
    """ Download malaria statistics from data.go.id """
    logger.info("Downloading raw data of Malaria statistics in Indonesia between 2010 - 2015")

    # compatibility check between python 2 and 3
    if sys.version_info < (3, 0):
        # for python 2, use this
        try:
            os.makedirs(DIRECTORY)
        except OSError as e:
            pass
        import urllib as downloader
        from urllib2 import URLError, HTTPError
    else:
        # for python 3, use this
        os.makedirs(DIRECTORY, exist_ok=True)
        import urllib.request as downloader
        from urllib.error import URLError, HTTPError

    output_path = os.path.join(DIRECTORY, OUTFILE)
    output_path_2015 = os.path.join(DIRECTORY, OUTFILE2015)

    # now retrieve the file
    try:
        downloader.urlretrieve(URL, output_path)
        logger.info('Downloaded yearly stats successfully to %s', os.path.abspath(output_path))

        downloader.urlretrieve(URL2015, output_path_2015)
        logger.info('Downloaded 2015 stats successfully to %s', os.path.abspath(output_path_2015))
    except (HTTPError, URLError) as e:
        logger.error('Failed to download: %s', e.reason)

if __name__ == "__main__":
    logging.basicConfig(level=logging.DEBUG)
    download()
<|endoftext|>


<commit_before>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2012:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

from shinken.util import strip_and_uniq

def find(value, lst, key):
    print 'Finding the value', value
    print 'And in the list', lst, 'and key', key

    if value is None:
        return None

    for i in lst:
        v = i.get(key, None)
        print 'MAtch with', v, value
        if v == value:
            return i
    return None

# We will find in lst elements with the key that match elt.prop values (it's a list
# of elements wit the keys in lst[key])
def find_several(lst, elt, prop, key):
    print 'Find several in', lst, 'for element', elt, 'and property', prop
    value = elt.get(prop, None)
    if value is None:
        return []

    values = value.split(',')
    values = strip_and_uniq(values)
    print 'Our values are', values

    res = []
    # Now try to match what it got
    for dbi in lst:
        print 'Try to look at', dbi, 'for property', key
        v = dbi.get(key, None)
        if v is None:
            continue
        v = v.strip()
        print 'MAtch with db',v
        if v  in values:
            res.append(dbi)
    print "Return find_several::", res
    return res

class Helper(object):
    def __init__(self, app):
        self.app = app

    # Return a simple string input
    def get_string_input(self, elt, prop, name, span='span10', innerspan='span2' ,placeholder='', popover=None, editable=''):
        p = ''
        if popover is not None:
            p = '' % prop
            p += '' % (prop, popover)
        s = '''
                   %s
                  <input class="%s" name="%s" type="text" value="%s" placeholder='%s' %s/>
                  %s

            ''' % (span, innerspan, name, editable, prop, elt.get(prop, ''), placeholder, editable, p, prop)
        return s

    def get_bool_input(self, elt, prop, name, editable=''):
        # Ok, let's try to see the value in db first
        v = elt.get(prop, '')

        on = ''
        if v == '1':
            on = 'active'
        off = ''
        if v == '0':
            off = 'active'
        unset = ''
        if not on and not off:
            unset = 'active'

        s = '''

            %s

	  <button class="btn %s %s" type="button" name="%s" value="1" >On</button>
	  <button class="btn %s %s" type="button" name="%s" value="0" >Off</button>
	  <button class="btn %s %s" type="button" name="%s" value="" >Unset</button>

        ''' % (name, prop, editable, on, editable, prop, off, editable, prop, unset, editable, prop)
        return s

    def get_percent_input(self, elt, prop, name, editable=''):
        # Ok, let's try to see the value in db first
        v = elt.get(prop, '')
        value = 0
        active = '0'
        if v != '':
            value = int(v)
            active = '1'

        s = '''

            %s

           %s%%

           Set/Unset
        ''' % (name, prop, prop, value ,prop, editable, prop, prop, active, prop, prop, editable)
        return s

    def get_select_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        value = elt.get(prop, None)
        elt_tp = find(value, tps, key)
        print 'Find a matching element for me?', elt.get(prop), elt_tp

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if elt_tp:
            tpname = elt_tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        # Always add avoid value if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if tp == elt_tp:
                continue

            if not key in tp:
                continue

            tpname = tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_command_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        value = elt.get(prop, None)
        args = ''
        # We split on the first ! of the data
        if value is not None:
            elts = value.split('!',1)
            value = elts[0]
            if len(elts) > 1:
                args = '!'+elts[1]

        elt_tp = find(value, tps, key)
        print 'Find a matching element for me?', elt.get(prop), elt_tp

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if elt_tp:
            tpname = elt_tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        # Always add a void option if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if tp == elt_tp:
                continue

            if not key in tp:
                continue

            tpname = tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s
                 Args <input class='%s' name='args-%s' value='%s' %s></input>

            ''' % (name, select_part, editable, prop, args, editable, prop)
        return s

    def get_multiselect_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        elts_tp = find_several(tps, elt, prop, key)
        print 'Find a matching element for me?', elts_tp

        select_part = '''<SELECT class='%s' name="%s" multiple="multiple" %s>''' % (editable, prop, editable)
        #if elt_tp:
        #    select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        #else:
        #    select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if not key in tp:
                continue

            tpname = tp.get(key)
            if tp in elts_tp:
                select_part += '<OPTION VALUE="%s" selected="selected" >%s</OPTION>' % (tpname, tpname)
            else:
                select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_poller_tag_input(self, elt, prop, name, editable=''):
        value = elt.get(prop, None)
        all_poller_tags = set()
        for p in self.app.conf.pollers:
            print 'Look at poller?', p.__dict__
            for t in getattr(p, 'poller_tags', '').split(','):
                all_poller_tags.add(t.strip())

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if value in all_poller_tags:
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (value, value)
        # Always add a void value, to unset if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for pt in all_poller_tags:
            if value == pt:
                continue

            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (pt, pt)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_realm_input(self, elt, prop, name, editable=''):

        value = elt.get(prop, None)
        all_realms = set()
        for r in self.app.conf.realms:
            print 'Look at realm?', r.__dict__
            all_realms.add(r.get_name())

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if value in all_realms:
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (value, value)
        # Always add a void value, to unset if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for pt in all_realms:
            if value == pt:
                continue

            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (pt, pt)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_customs_inputs(self, app, elt, editable=''):
        print "CUSTOM OF", elt
        s = ''
        customs = {}

        # Get our template names
        tnames = [t.strip() for t in elt.get('use', '').split(',')]

        # Look for our templates custom values, and get only the first one
        for t in tnames:
            for tpl in app.host_templates.values():
                tname = getattr(tpl, 'name', None)
                if tname == t:
                    for (k,v) in tpl.customs.iteritems():
                        print 'My template customs', k, v
                        if k not in customs:
                           customs[k] = {'from' : tname, 'value' : ''}

        # Now the item one, will overwrite any entry
        for (k,v) in elt.iteritems():
            if k.startswith('_') and k != '_id':
                customs[k] = {'from' : '__ITEM__', 'value' : v}

        # Get a sorted macro names
        sorted_names = [k for k in customs]
        sorted_names.sort()
        print 'Sorted names', sorted_names

        # We want to show the how element macros value first
        tnames.insert(0, '__ITEM__')
        for tname in tnames:
            new_template = True
            for k in sorted_names:
                v = customs[k]
                if v['from'] != tname:
                    continue
                if new_template:
                    s += '%s' % tname
                    new_template = False
                print "Looping over template", tname, "customs"
                ctype = 'string'
                founded = False
                for p in app.packs:
                    if founded:
                        break
                    for (m, mv) in p.macros.iteritems():
                        print "COmpare", k, m
                        if k.upper() == m.upper():
                            print 'Match a pack', mv
                            ctype = mv.get('type', 'string').strip()
                            founded = True
                            break
                if ctype == 'percent':
                    s+= self.get_percent_input(elt, k, k[1:], editable=editable)
                else: # if not known, apply string
                    s+= self.get_string_input(elt, k, k[1:], editable=editable)

        return s
<commit_msg>Add : skonf host macros, put in placeholder the template value if available.<commit_after>#!/usr/bin/python

# -*- coding: utf-8 -*-

# Copyright (C) 2009-2012:
#    Gabes Jean, naparuba@gmail.com
#    Gerhard Lausser, Gerhard.Lausser@consol.de
#    Gregory Starck, g.starck@gmail.com
#    Hartmut Goebel, h.goebel@goebel-consult.de
#
# This file is part of Shinken.
#
# Shinken is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Shinken is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with Shinken.  If not, see <http://www.gnu.org/licenses/>.

from shinken.util import strip_and_uniq

def find(value, lst, key):
    print 'Finding the value', value
    print 'And in the list', lst, 'and key', key

    if value is None:
        return None

    for i in lst:
        v = i.get(key, None)
        print 'MAtch with', v, value
        if v == value:
            return i
    return None

# We will find in lst elements with the key that match elt.prop values (it's a list
# of elements wit the keys in lst[key])
def find_several(lst, elt, prop, key):
    print 'Find several in', lst, 'for element', elt, 'and property', prop
    value = elt.get(prop, None)
    if value is None:
        return []

    values = value.split(',')
    values = strip_and_uniq(values)
    print 'Our values are', values

    res = []
    # Now try to match what it got
    for dbi in lst:
        print 'Try to look at', dbi, 'for property', key
        v = dbi.get(key, None)
        if v is None:
            continue
        v = v.strip()
        print 'MAtch with db',v
        if v  in values:
            res.append(dbi)
    print "Return find_several::", res
    return res

class Helper(object):
    def __init__(self, app):
        self.app = app

    # Return a simple string input
    def get_string_input(self, elt, prop, name, span='span10', innerspan='span2' ,placeholder='', popover=None, editable=''):
        p = ''
        if popover is not None:
            p = '' % prop
            p += '' % (prop, popover)
        s = '''
                   %s
                  <input class="%s" name="%s" type="text" value="%s" placeholder='%s' %s/>
                  %s

            ''' % (span, innerspan, name, editable, prop, elt.get(prop, ''), placeholder, editable, p, prop)
        return s

    def get_bool_input(self, elt, prop, name, editable=''):
        # Ok, let's try to see the value in db first
        v = elt.get(prop, '')

        on = ''
        if v == '1':
            on = 'active'
        off = ''
        if v == '0':
            off = 'active'
        unset = ''
        if not on and not off:
            unset = 'active'

        s = '''

            %s

	  <button class="btn %s %s" type="button" name="%s" value="1" >On</button>
	  <button class="btn %s %s" type="button" name="%s" value="0" >Off</button>
	  <button class="btn %s %s" type="button" name="%s" value="" >Unset</button>

        ''' % (name, prop, editable, on, editable, prop, off, editable, prop, unset, editable, prop)
        return s

    def get_percent_input(self, elt, prop, name, editable=''):
        # Ok, let's try to see the value in db first
        v = elt.get(prop, '')
        value = 0
        active = '0'
        if v != '':
            value = int(v)
            active = '1'

        s = '''

            %s

           %s%%

           Set/Unset
        ''' % (name, prop, prop, value ,prop, editable, prop, prop, active, prop, prop, editable)
        return s

    def get_select_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        value = elt.get(prop, None)
        elt_tp = find(value, tps, key)
        print 'Find a matching element for me?', elt.get(prop), elt_tp

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if elt_tp:
            tpname = elt_tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        # Always add avoid value if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if tp == elt_tp:
                continue

            if not key in tp:
                continue

            tpname = tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_command_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        value = elt.get(prop, None)
        args = ''
        # We split on the first ! of the data
        if value is not None:
            elts = value.split('!',1)
            value = elts[0]
            if len(elts) > 1:
                args = '!'+elts[1]

        elt_tp = find(value, tps, key)
        print 'Find a matching element for me?', elt.get(prop), elt_tp

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if elt_tp:
            tpname = elt_tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        # Always add a void option if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if tp == elt_tp:
                continue

            if not key in tp:
                continue

            tpname = tp.get(key)
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s
                 Args <input class='%s' name='args-%s' value='%s' %s></input>

            ''' % (name, select_part, editable, prop, args, editable, prop)
        return s

    def get_multiselect_input(self, elt, prop, name, cls, key, editable=''):
        t = getattr(self.app.db, cls)
        tps = list(t.find({}))

        elts_tp = find_several(tps, elt, prop, key)
        print 'Find a matching element for me?', elts_tp

        select_part = '''<SELECT class='%s' name="%s" multiple="multiple" %s>''' % (editable, prop, editable)
        #if elt_tp:
        #    select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        #else:
        #    select_part += '<OPTION VALUE=""></OPTION>'

        for tp in tps:
            if not key in tp:
                continue

            tpname = tp.get(key)
            if tp in elts_tp:
                select_part += '<OPTION VALUE="%s" selected="selected" >%s</OPTION>' % (tpname, tpname)
            else:
                select_part += '<OPTION VALUE="%s">%s</OPTION>' % (tpname, tpname)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_poller_tag_input(self, elt, prop, name, editable=''):
        value = elt.get(prop, None)
        all_poller_tags = set()
        for p in self.app.conf.pollers:
            print 'Look at poller?', p.__dict__
            for t in getattr(p, 'poller_tags', '').split(','):
                all_poller_tags.add(t.strip())

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if value in all_poller_tags:
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (value, value)
        # Always add a void value, to unset if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for pt in all_poller_tags:
            if value == pt:
                continue

            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (pt, pt)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_realm_input(self, elt, prop, name, editable=''):

        value = elt.get(prop, None)
        all_realms = set()
        for r in self.app.conf.realms:
            print 'Look at realm?', r.__dict__
            all_realms.add(r.get_name())

        select_part = '''<SELECT class='%s' name="%s" %s>''' % (editable, prop, editable)
        if value in all_realms:
            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (value, value)
        # Always add a void value, to unset if need
        select_part += '<OPTION VALUE=""></OPTION>'

        for pt in all_realms:
            if value == pt:
                continue

            select_part += '<OPTION VALUE="%s">%s</OPTION>' % (pt, pt)
        select_part += '</SELECT>'

        s = '''
                   %s
                  %s

            ''' % (name, select_part, prop)
        return s

    def get_customs_inputs(self, app, elt, editable=''):
        print "CUSTOM OF", elt
        s = ''
        customs = {}

        # Get our template names
        tnames = [t.strip() for t in elt.get('use', '').split(',')]

        # Look for our templates custom values, and get only the first one
        for t in tnames:
            for tpl in app.host_templates.values():
                tname = getattr(tpl, 'name', None)
                if tname == t:
                    for (k,v) in tpl.customs.iteritems():
                        print 'My template customs', k, v
                        if k not in customs:
                           customs[k] = {'from' : tname, 'value' : '', 'placeholder' : v}

        # Now the item one, will overwrite any entry
        for (k,v) in elt.iteritems():
            if k.startswith('_') and k != '_id':
                customs[k] = {'from' : '__ITEM__', 'value' : v, 'placeholder' : ''}

        # Get a sorted macro names
        sorted_names = [k for k in customs]
        sorted_names.sort()
        print 'Sorted names', sorted_names

        # We want to show the how element macros value first
        tnames.insert(0, '__ITEM__')
        for tname in tnames:
            new_template = True
            for k in sorted_names:
                v = customs[k]
                if v['from'] != tname:
                    continue
                if new_template:
                    s += '%s' % tname
                    new_template = False
                print "Looping over template", tname, "customs"
                ctype = 'string'
                founded = False
                for p in app.packs:
                    if founded:
                        break
                    for (m, mv) in p.macros.iteritems():
                        print "COmpare", k, m
                        if k.upper() == m.upper():
                            print 'Match a pack', mv
                            ctype = mv.get('type', 'string').strip()
                            founded = True
                            break
                if ctype == 'percent':
                    s+= self.get_percent_input(elt, k, k[1:], editable=editable)
                else: # if not known, apply string
                    s+= self.get_string_input(elt, k, k[1:], editable=editable, placeholder=v['placeholder'])

        return s
<|endoftext|>


<commit_before>"""
Detector utilities
"""

__license__ = """
This file is part of RAPD

Copyright (C) 2016-2017 Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2016-11-21"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Development"

# Standard imports
import glob
import importlib
import os
import shutil
import sys
import tempfile

# Phenix imports
from dxtbx.format.Registry import Registry
from iotbx.detectors import ImageFactory

# RAPD imports
import utils.convert_hdf5_cbf as convert_hdf5_cbf
import utils.text as text
import detector_list

def print_detector_info(image):
    """
    Print out information on the detector given an image
    """

    from iotbx.detectors import ImageFactory

    image_basename = os.path.basename(image)

    try:
        i = ImageFactory(image)
    except IOError as e:
        if "no format support found for" in e.message:
            print "No format support for %s" % image_basename
            return False
        else:
            print e
            return False
    except AttributeError as e:
        if "object has no attribute 'detectorbase'" in e.message:
            print "No format support for %s" % image_basename
            return False
        else:
            print text.red + e.message + text.stop
            return False

    print "\nInformation from iotbx ImageFactory"
    print "%20s::%s" % ("image", image_basename)
    print "%20s::%s" % ("vendortype", str(i.vendortype))
    # print "%20s" % "Parameters"
    for key, val in i.parameters.iteritems():
        print "%20s::%s" % (key, val)

def print_detector_info2(image):
    """
    Print out information on the detector given an image
    """

    format_instance = Registry.find(image)
    instance = format_instance(image)
    # adds parameters (iotbx)
    temp = instance.get_detectorbase()

    print "\nInformation from dxtbx Registry"
    for key, val in temp.parameters.iteritems():
        print "%20s::%s" % (key, val)

def get_detector_files():
    """
    Returns a list of detector files

    Uses the PYTHONPATH to find the sites/detectors directory and the detectors directory
    for this rapd install and then walks them to find all files that have names that match "*.py"
    and do not start with "_" or have the word "secret"
    """

    def look_for_detector_files(directory):
        """
        Look for detector files in the given directory
        """
        potential_files = []
        for filename in glob.glob(directory+"/*.py"):
            # No secret-containing files
            if "secret" in filename:
                continue
            # No files that start with _
            if os.path.basename(filename).startswith("_"):
                continue
            # Filename OK
            potential_files.append(os.path.join(path, filename))

        return potential_files

    possible_files = []

    # Looking for the rapd src directory
    detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "detectors")):
            detectors_dir = os.path.join(path, "detectors")
            break
    # print detectors_dir

    if detectors_dir:
        possible_files += look_for_detector_files(detectors_dir + "/*")
        # print possible_files

    site_detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "sites/detectors")):
            site_detectors_dir = os.path.join(path, "sites/detectors")
            break

    if site_detectors_dir:
        possible_files += look_for_detector_files(site_detectors_dir)

    return possible_files

def get_detector_file(image):
    """
    Returns the RAPD detector file given an image file
    """

    print "get_detector_file %s" % image

    try:
        i = ImageFactory(image)
        # print i.vendortype
        # print i.parameters["DETECTOR_SN"]
    except (IOError, AttributeError, RuntimeError):
        print error
        return False

    # print i.vendortype
    # print i.parameters["DETECTOR_SN"]

    if (i.vendortype, i.parameters["DETECTOR_SN"]) in detector_list.DETECTORS:
        print "%s: %s %s %s" % (image, detector_list.DETECTORS[(i.vendortype, i.parameters["DETECTOR_SN"])], i.vendortype, i.parameters["DETECTOR_SN"])
        return detector_list.DETECTORS[(i.vendortype, i.parameters["DETECTOR_SN"])]
    else:
        return False

def load_detector(detector):
    """
    Search for detector file, load, and return it
    """

    # print "load_detector %s" % detector

    if isinstance(detector, dict):
        detector = detector.get("detector")

    def look_for_detector_file(file, directory):
        """
        Look for a detector file in the given directory
        """

        print "look_for_detector_file %s %s" % (file, directory)

        if "sites" in directory:
            files = glob.glob(directory+"/"+file+".py")
        else:
            files = glob.glob(directory+"/*/"+file+".py")

        if len(files):
            fullpath_file = files[0]
            split_path_file = fullpath_file.split("/")
            if "sites" in split_path_file:
                return ".".join(split_path_file[split_path_file.index("sites"):]).replace(".py", "")
            else:
                return ".".join(split_path_file[split_path_file.index("detectors"):]).replace(".py", "")
        else:
            return False

    # Look for the src/detectors directory
    detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "detectors")):
            detectors_dir = os.path.join(path, "detectors")
            break

    detector_file = False

    # Search the src/detectors dir
    detector_file = look_for_detector_file(detector, detectors_dir)

    # Search the sites/detectors dir
    if not detector_file:
        sites_dir = False
        for path in sys.path:
            if path.endswith("src") and os.path.exists(os.path.join(path, "sites", "detectors")):
                sites_dir = os.path.join(path, "sites", "detectors")
                break
        detector_file = look_for_detector_file(detector, sites_dir)

    # No module found == bad
    if detector_file == False:
        raise Exception, "No detector file found for %s" % detector
    else:
        module = importlib.import_module(detector_file)
        return module

def main(test_images):
    """Print out some detector information"""

    for test_image in test_images:

        tmp_dir = False

        if test_image.endswith(".h5"):
            print "HDF5 file  - converting"

            tmp_dir = tempfile.mkdtemp()

            converter = convert_hdf5_cbf.hdf5_to_cbf_converter(master_file=test_image,
                                                               output_dir=tmp_dir,
                                                               prefix="tmp",
                                                               start_image=1,
                                                               end_image=1)
            converter.run()

            test_image = "%s/tmp00001.cbf" % tmp_dir

        # try:
        print_detector_info(test_image)
        print_detector_info2(test_image)

        print "\nRAPD detector registry"
        detector = get_detector_file(test_image)
        if detector:
            print "%20s::%s" % ("detector", detector)
        else:
            print "%20s::%s" % ("detector", "unknown")
        # except:
        #     print "%20s::%s" % ("error", "Severe error reading %s" % os.path.basename(test_image))

        print "====="

        if tmp_dir:
            shutil.rmtree(tmp_dir)

if __name__ == "__main__":

    # Get image name from the commandline
    if len(sys.argv) > 1:
        test_images = sys.argv[1:]
        main(test_images)
    else:
        print text.red + "No input image" + text.stop
        sys.exit(9)
<commit_msg>Attempting to build an h5 exploring program<commit_after>"""
Detector utilities
"""

__license__ = """
This file is part of RAPD

Copyright (C) 2016-2017 Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2016-11-21"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Development"

# Standard imports
import glob
import importlib
import os
import shutil
import sys
import tempfile

# CCTBX imports
from dxtbx.format.Registry import Registry
import h5py
from iotbx.detectors import ImageFactory

# RAPD imports
import utils.convert_hdf5_cbf as convert_hdf5_cbf
import utils.text as text
import detector_list

def print_detector_info(image):
    """
    Print out information on the detector given an image
    """

    from iotbx.detectors import ImageFactory

    image_basename = os.path.basename(image)

    try:
        i = ImageFactory(image)
    except IOError as e:
        if "no format support found for" in e.message:
            print "No format support for %s" % image_basename
            return False
        else:
            print e
            return False
    except AttributeError as e:
        if "object has no attribute 'detectorbase'" in e.message:
            print "No format support for %s" % image_basename
            return False
        else:
            print text.red + e.message + text.stop
            return False

    print "\nInformation from iotbx ImageFactory"
    print "%20s::%s" % ("image", image_basename)
    print "%20s::%s" % ("vendortype", str(i.vendortype))
    # print "%20s" % "Parameters"
    for key, val in i.parameters.iteritems():
        print "%20s::%s" % (key, val)

def print_detector_info2(image):
    """
    Print out information on the detector given an image
    """

    format_instance = Registry.find(image)
    instance = format_instance(image)
    # adds parameters (iotbx)
    temp = instance.get_detectorbase()

    print "\nInformation from dxtbx Registry"
    for key, val in temp.parameters.iteritems():
        print "%20s::%s" % (key, val)

def get_detector_files():
    """
    Returns a list of detector files

    Uses the PYTHONPATH to find the sites/detectors directory and the detectors directory
    for this rapd install and then walks them to find all files that have names that match "*.py"
    and do not start with "_" or have the word "secret"
    """

    def look_for_detector_files(directory):
        """
        Look for detector files in the given directory
        """
        potential_files = []
        for filename in glob.glob(directory+"/*.py"):
            # No secret-containing files
            if "secret" in filename:
                continue
            # No files that start with _
            if os.path.basename(filename).startswith("_"):
                continue
            # Filename OK
            potential_files.append(os.path.join(path, filename))

        return potential_files

    possible_files = []

    # Looking for the rapd src directory
    detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "detectors")):
            detectors_dir = os.path.join(path, "detectors")
            break
    # print detectors_dir

    if detectors_dir:
        possible_files += look_for_detector_files(detectors_dir + "/*")
        # print possible_files

    site_detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "sites/detectors")):
            site_detectors_dir = os.path.join(path, "sites/detectors")
            break

    if site_detectors_dir:
        possible_files += look_for_detector_files(site_detectors_dir)

    return possible_files

def get_detector_file(image):
    """
    Returns the RAPD detector file given an image file
    """

    print "get_detector_file %s" % image

    try:
        i = ImageFactory(image)
        # print i.vendortype
        # print i.parameters["DETECTOR_SN"]
    except (IOError, AttributeError, RuntimeError):
        print error
        return False

    # print i.vendortype
    # print i.parameters["DETECTOR_SN"]

    if (i.vendortype, i.parameters["DETECTOR_SN"]) in detector_list.DETECTORS:
        print "%s: %s %s %s" % (image, detector_list.DETECTORS[(i.vendortype, i.parameters["DETECTOR_SN"])], i.vendortype, i.parameters["DETECTOR_SN"])
        return detector_list.DETECTORS[(i.vendortype, i.parameters["DETECTOR_SN"])]
    else:
        return False

def load_detector(detector):
    """
    Search for detector file, load, and return it
    """

    # print "load_detector %s" % detector

    if isinstance(detector, dict):
        detector = detector.get("detector")

    def look_for_detector_file(file, directory):
        """
        Look for a detector file in the given directory
        """

        print "look_for_detector_file %s %s" % (file, directory)

        if "sites" in directory:
            files = glob.glob(directory+"/"+file+".py")
        else:
            files = glob.glob(directory+"/*/"+file+".py")

        if len(files):
            fullpath_file = files[0]
            split_path_file = fullpath_file.split("/")
            if "sites" in split_path_file:
                return ".".join(split_path_file[split_path_file.index("sites"):]).replace(".py", "")
            else:
                return ".".join(split_path_file[split_path_file.index("detectors"):]).replace(".py", "")
        else:
            return False

    # Look for the src/detectors directory
    detectors_dir = False
    for path in sys.path:
        if path.endswith("src") and os.path.exists(os.path.join(path, "detectors")):
            detectors_dir = os.path.join(path, "detectors")
            break

    detector_file = False

    # Search the src/detectors dir
    detector_file = look_for_detector_file(detector, detectors_dir)

    # Search the sites/detectors dir
    if not detector_file:
        sites_dir = False
        for path in sys.path:
            if path.endswith("src") and os.path.exists(os.path.join(path, "sites", "detectors")):
                sites_dir = os.path.join(path, "sites", "detectors")
                break
        detector_file = look_for_detector_file(detector, sites_dir)

    # No module found == bad
    if detector_file == False:
        raise Exception, "No detector file found for %s" % detector
    else:
        module = importlib.import_module(detector_file)
        return module

def read_hdf5_header(image):
    """Explore and return information from an hdf5 file"""

    f = h5py.File(image, "r")

    entry = f.get("entry")

    instrument = entry.get("instrument")
    beam = instrument.get("beam")
    detector = instrument.get("detector")

    for k, v in beam.iteritems():
        print k, v

    for k, v in detector.iteritems():
        print k, v.keys()

    sys.exit()

def main(test_images):
    """Print out some detector information"""

    for test_image in test_images:

        tmp_dir = False

        if test_image.endswith(".h5"):

            print "HDF5 file"

            hdf5_header = read_hdf5_header(test_image)

            tmp_dir = tempfile.mkdtemp()

            converter = convert_hdf5_cbf.hdf5_to_cbf_converter(master_file=test_image,
                                                               output_dir=tmp_dir,
                                                               prefix="tmp",
                                                               start_image=1,
                                                               end_image=1,
                                                               overwrite=True)
            converter.run()

            test_image = "%s/tmp00001.cbf" % tmp_dir

        # try:
        print_detector_info(test_image)
        print_detector_info2(test_image)

        print "\nRAPD detector registry"
        detector = get_detector_file(test_image)
        if detector:
            print "%20s::%s" % ("detector", detector)
        else:
            print "%20s::%s" % ("detector", "unknown")
        # except:
        #     print "%20s::%s" % ("error", "Severe error reading %s" % os.path.basename(test_image))

        print "====="

        if tmp_dir:
            shutil.rmtree(tmp_dir)

if __name__ == "__main__":

    # Get image name from the commandline
    if len(sys.argv) > 1:
        test_images = sys.argv[1:]
        main(test_images)
    else:
        print text.red + "No input image" + text.stop
        sys.exit(9)
<|endoftext|>


<commit_before>#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from saml2 import BINDING_HTTP_REDIRECT
from saml2.ident import decode
from saml2.client import Saml2Client
from saml2.response import LogoutResponse
from saml2.metadata import entity_descriptor
from werkzeug.exceptions import Forbidden
from flask import request, redirect, abort, make_response
from flask import current_app, Blueprint, session

from eduid_common.session import session
from eduid_common.authn.utils import get_location
from eduid_common.authn.loa import get_loa
from eduid_common.authn.eduid_saml2 import get_authn_request, get_authn_response
from eduid_common.authn.eduid_saml2 import authenticate
from eduid_common.authn.cache import IdentityCache, StateCache
from eduid_common.authn.acs_registry import get_action, schedule_action
from eduid_common.authn.utils import check_previous_identification
from eduid_common.api.utils import verify_relay_state

authn_views = Blueprint('authn', __name__, url_prefix='')

@authn_views.route('/login')
def login():
    """
    login view, redirects to SAML2 IdP
    """
    return _authn('login-action')

@authn_views.route('/reauthn')
def reauthn():
    """
    login view with force authn, redirects to SAML2 IdP
    """
    session['user_is_logged_in'] = False
    return _authn('reauthn-action', force_authn=True)

@authn_views.route('/chpass')
def chpass():
    """
    Reauthn view, sends a SAML2 reauthn request to the IdP.
    """
    return _authn('change-password-action', force_authn=True)

@authn_views.route('/terminate')
def terminate():
    """
    Reauthn view, sends a SAML2 reauthn request to the IdP.
    """
    return _authn('terminate-account-action', force_authn=True)

def _authn(action, force_authn=False):
    redirect_url = current_app.config.get('SAML2_LOGIN_REDIRECT_URL', '/')
    relay_state = verify_relay_state(request.args.get('next', redirect_url), redirect_url)
    idps = current_app.saml2_config.getattr('idp')
    assert len(idps) == 1
    idp = list(idps.keys())[0]
    idp = request.args.get('idp', idp)
    loa = request.args.get('required_loa', None)
    authn_request = get_authn_request(current_app.config, session,
                                      relay_state, idp, required_loa=loa,
                                      force_authn=force_authn)
    schedule_action(action)
    current_app.logger.info('Redirecting the user to the IdP for ' + action)
    return redirect(get_location(authn_request))

@authn_views.route('/saml2-acs', methods=['POST'])
def assertion_consumer_service():
    """
    Assertion consumer service, receives POSTs from SAML2 IdP's
    """

    if 'SAMLResponse' not in request.form:
        abort(400)

    xmlstr = request.form['SAMLResponse']
    session_info = get_authn_response(current_app.config, session, xmlstr)
    current_app.logger.debug('Trying to locate the user authenticated by the IdP')
    user = authenticate(current_app, session_info)

    if user is None:
        current_app.logger.error('Could not find the user identified by the IdP')
        raise Forbidden("Access not authorized")

    action = get_action()
    return action(session_info, user)

def _get_name_id(session):
    """
    Get the SAML2 NameID of the currently logged in user.
    :param session: The current session object
    :return: NameID
    :rtype: saml2.saml.NameID | None
    """
    try:
        return decode(session['_saml2_session_name_id'])
    except KeyError:
        return None

@authn_views.route('/logout', methods=['GET'])
def logout():
    """
    SAML Logout Request initiator.
    This view initiates the SAML2 Logout request
    using the pysaml2 library to create the LogoutRequest.
    """
    eppn = session.get('user_eppn')

    if eppn is None:
        current_app.logger.info('Session cookie has expired, no logout action needed')
        location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
        return redirect(location)

    user = current_app.central_userdb.get_user_by_eppn(eppn)

    current_app.logger.debug('Logout process started for user {}'.format(user))
    state = StateCache(session)
    identity = IdentityCache(session)

    client = Saml2Client(current_app.saml2_config,
                         state_cache=state,
                         identity_cache=identity)

    # The user doesn't have a SAML2 NameID in the session
    # after a token login from Signup into Dashboard.
    subject_id = _get_name_id(session)
    if subject_id is None:
        current_app.logger.warning(
            'The session does not contain '
            'the subject id for user {}'.format(user))
        session.clear()
        location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')

    else:
        logouts = client.global_logout(subject_id)
        loresponse = list(logouts.values())[0]
        # loresponse is a dict for REDIRECT binding, and LogoutResponse for SOAP binding
        if isinstance(loresponse, LogoutResponse):
            if loresponse.status_ok():
                current_app.logger.debug('Performing local logout for {}'.format(user))
                session.clear()
                location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
                location = verify_relay_state(request.form.get('RelayState', location), location)
                return redirect(location)
            else:
                abort(500)
        headers_tuple = loresponse[1]['headers']
        location = headers_tuple[0][1]
        current_app.logger.info('Redirecting to {!r} to continue the logout process '
                                'for user {}'.format(location, user))

    state.sync()
    return redirect(location)

@authn_views.route('/saml2-ls', methods=['POST'])
def logout_service():
    """SAML Logout Response endpoint
    The IdP will send the logout response to this view,
    which will process it with pysaml2 help and log the user
    out.
    Note that the IdP can request a logout even when
    we didn't initiate the process as a single logout
    request started by another SP.
    """
    current_app.logger.debug('Logout service started')

    state = StateCache(session)
    identity = IdentityCache(session)
    client = Saml2Client(current_app.saml2_config,
                         state_cache=state,
                         identity_cache=identity)

    logout_redirect_url = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
    next_page = session.get('next', logout_redirect_url)
    next_page = request.args.get('next', next_page)
    next_page = request.form.get('RelayState', next_page)
    next_page = verify_relay_state(next_page, logout_redirect_url)

    if 'SAMLResponse' in request.form: # we started the logout
        current_app.logger.debug('Receiving a logout response from the IdP')
        response = client.parse_logout_request_response(
            request.form['SAMLResponse'],
            BINDING_HTTP_REDIRECT
        )
        state.sync()
        if response and response.status_ok():
            session.clear()
            return redirect(next_page)
        else:
            current_app.logger.error('Unknown error during the logout')
            abort(400)

    # logout started by the IdP
    elif 'SAMLRequest' in request.form:
        current_app.logger.debug('Receiving a logout request from the IdP')
        subject_id = _get_name_id(session)
        if subject_id is None:
            current_app.logger.warning(
                'The session does not contain the subject id for user {0} '
                'Performing local logout'.format(
                    session['eduPersonPrincipalName']
                )
            )
            session.clear()
            return redirect(next_page)
        else:
            http_info = client.handle_logout_request(
                request.form['SAMLRequest'],
                subject_id,
                BINDING_HTTP_REDIRECT,
                relay_state=request.form['RelayState']
            )
            state.sync()
            location = get_location(http_info)
            session.clear()
            return redirect(location)
    current_app.logger.error('No SAMLResponse or SAMLRequest parameter found')
    abort(400)

@authn_views.route('/signup-authn', methods=['GET'])
def signup_authn():
    current_app.logger.debug('Authenticating signing up user')
    location_on_fail = current_app.config.get('SIGNUP_AUTHN_FAILURE_REDIRECT_URL')
    location_on_success = current_app.config.get('SIGNUP_AUTHN_SUCCESS_REDIRECT_URL')

    eppn = check_previous_identification(session.actions)
    if eppn is not None:
        loa = get_loa(current_app.config.get('AVAILABLE_LOA'), None)  # With no session_info lowest loa will be returned
        current_app.logger.info("Starting pre-login actions " "for eppn: {})".format(eppn))
        try:
            user = current_app.central_userdb.get_user_by_eppn(eppn)
        except current_app.central_userdb.exceptions.UserDoesNotExist:
            current_app.logger.error('No user with eduPersonPrincipalName = {} found'.format(eppn))
        except current_app.central_userdb.exceptions.MultipleUsersReturned:
            current_app.logger.error("There are more than one user with eduPersonPrincipalName = {}".format(eppn))
        else:
            if user.locked_identity.count > 0:
                # This user has previously verified their account and is not new, this should not happen.
                current_app.logger.error('Not new user {} tried to log in using token login'.format(user))
                return redirect(location_on_fail)
            session['eduPersonPrincipalName'] = user.eppn
            session['user_eppn'] = user.eppn
            session['user_is_logged_in'] = True
            session['eduPersonAssurance'] = loa

            response = redirect(location_on_success)
            #session.set_cookie(response)  # Is this needed as session_interface.save_session will be called?
            current_app.logger.info('Successful token login, redirecting user {} to {}'.format(user,
                                                                                location_on_success))
            return response

    current_app.logger.info('Token login failed, redirecting user to {}'.format(location_on_fail))
    return redirect(location_on_fail)

@authn_views.route('/saml2-metadata')
def metadata():
    """
    Returns an XML with the SAML 2.0 metadata for this
    SP as configured in the saml2_settings.py file.
    """
    metadata = entity_descriptor(current_app.saml2_config)
    response = make_response(metadata.to_string(), 200)
    response.headers['Content-Type'] = "text/xml; charset=utf8"
    return response
<commit_msg>wrong namespace<commit_after>#
# Copyright (c) 2016 NORDUnet A/S
# All rights reserved.
#
#   Redistribution and use in source and binary forms, with or
#   without modification, are permitted provided that the following
#   conditions are met:
#
#     1. Redistributions of source code must retain the above copyright
#        notice, this list of conditions and the following disclaimer.
#     2. Redistributions in binary form must reproduce the above
#        copyright notice, this list of conditions and the following
#        disclaimer in the documentation and/or other materials provided
#        with the distribution.
#     3. Neither the name of the NORDUnet nor the names of its
#        contributors may be used to endorse or promote products derived
#        from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#

from saml2 import BINDING_HTTP_REDIRECT
from saml2.ident import decode
from saml2.client import Saml2Client
from saml2.response import LogoutResponse
from saml2.metadata import entity_descriptor
from werkzeug.exceptions import Forbidden
from flask import request, redirect, abort, make_response
from flask import current_app, Blueprint, session

from eduid_common.session import session
from eduid_common.authn.utils import get_location
from eduid_common.authn.loa import get_loa
from eduid_common.authn.eduid_saml2 import get_authn_request, get_authn_response
from eduid_common.authn.eduid_saml2 import authenticate
from eduid_common.authn.cache import IdentityCache, StateCache
from eduid_common.authn.acs_registry import get_action, schedule_action
from eduid_common.authn.utils import check_previous_identification
from eduid_common.api.utils import verify_relay_state

authn_views = Blueprint('authn', __name__, url_prefix='')

@authn_views.route('/login')
def login():
    """
    login view, redirects to SAML2 IdP
    """
    return _authn('login-action')

@authn_views.route('/reauthn')
def reauthn():
    """
    login view with force authn, redirects to SAML2 IdP
    """
    session['user_is_logged_in'] = False
    return _authn('reauthn-action', force_authn=True)

@authn_views.route('/chpass')
def chpass():
    """
    Reauthn view, sends a SAML2 reauthn request to the IdP.
    """
    return _authn('change-password-action', force_authn=True)

@authn_views.route('/terminate')
def terminate():
    """
    Reauthn view, sends a SAML2 reauthn request to the IdP.
    """
    return _authn('terminate-account-action', force_authn=True)

def _authn(action, force_authn=False):
    redirect_url = current_app.config.get('SAML2_LOGIN_REDIRECT_URL', '/')
    relay_state = verify_relay_state(request.args.get('next', redirect_url), redirect_url)
    idps = current_app.saml2_config.getattr('idp')
    assert len(idps) == 1
    idp = list(idps.keys())[0]
    idp = request.args.get('idp', idp)
    loa = request.args.get('required_loa', None)
    authn_request = get_authn_request(current_app.config, session,
                                      relay_state, idp, required_loa=loa,
                                      force_authn=force_authn)
    schedule_action(action)
    current_app.logger.info('Redirecting the user to the IdP for ' + action)
    return redirect(get_location(authn_request))

@authn_views.route('/saml2-acs', methods=['POST'])
def assertion_consumer_service():
    """
    Assertion consumer service, receives POSTs from SAML2 IdP's
    """

    if 'SAMLResponse' not in request.form:
        abort(400)

    xmlstr = request.form['SAMLResponse']
    session_info = get_authn_response(current_app.config, session, xmlstr)
    current_app.logger.debug('Trying to locate the user authenticated by the IdP')
    user = authenticate(current_app, session_info)

    if user is None:
        current_app.logger.error('Could not find the user identified by the IdP')
        raise Forbidden("Access not authorized")

    action = get_action()
    return action(session_info, user)

def _get_name_id(session):
    """
    Get the SAML2 NameID of the currently logged in user.
    :param session: The current session object
    :return: NameID
    :rtype: saml2.saml.NameID | None
    """
    try:
        return decode(session['_saml2_session_name_id'])
    except KeyError:
        return None

@authn_views.route('/logout', methods=['GET'])
def logout():
    """
    SAML Logout Request initiator.
    This view initiates the SAML2 Logout request
    using the pysaml2 library to create the LogoutRequest.
    """
    eppn = session.get('user_eppn')

    if eppn is None:
        current_app.logger.info('Session cookie has expired, no logout action needed')
        location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
        return redirect(location)

    user = current_app.central_userdb.get_user_by_eppn(eppn)

    current_app.logger.debug('Logout process started for user {}'.format(user))
    state = StateCache(session)
    identity = IdentityCache(session)

    client = Saml2Client(current_app.saml2_config,
                         state_cache=state,
                         identity_cache=identity)

    # The user doesn't have a SAML2 NameID in the session
    # after a token login from Signup into Dashboard.
    subject_id = _get_name_id(session)
    if subject_id is None:
        current_app.logger.warning(
            'The session does not contain '
            'the subject id for user {}'.format(user))
        session.clear()
        location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')

    else:
        logouts = client.global_logout(subject_id)
        loresponse = list(logouts.values())[0]
        # loresponse is a dict for REDIRECT binding, and LogoutResponse for SOAP binding
        if isinstance(loresponse, LogoutResponse):
            if loresponse.status_ok():
                current_app.logger.debug('Performing local logout for {}'.format(user))
                session.clear()
                location = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
                location = verify_relay_state(request.form.get('RelayState', location), location)
                return redirect(location)
            else:
                abort(500)
        headers_tuple = loresponse[1]['headers']
        location = headers_tuple[0][1]
        current_app.logger.info('Redirecting to {!r} to continue the logout process '
                                'for user {}'.format(location, user))

    state.sync()
    return redirect(location)

@authn_views.route('/saml2-ls', methods=['POST'])
def logout_service():
    """SAML Logout Response endpoint
    The IdP will send the logout response to this view,
    which will process it with pysaml2 help and log the user
    out.
    Note that the IdP can request a logout even when
    we didn't initiate the process as a single logout
    request started by another SP.
    """
    current_app.logger.debug('Logout service started')

    state = StateCache(session)
    identity = IdentityCache(session)
    client = Saml2Client(current_app.saml2_config,
                         state_cache=state,
                         identity_cache=identity)

    logout_redirect_url = current_app.config.get('SAML2_LOGOUT_REDIRECT_URL')
    next_page = session.get('next', logout_redirect_url)
    next_page = request.args.get('next', next_page)
    next_page = request.form.get('RelayState', next_page)
    next_page = verify_relay_state(next_page, logout_redirect_url)

    if 'SAMLResponse' in request.form: # we started the logout
        current_app.logger.debug('Receiving a logout response from the IdP')
        response = client.parse_logout_request_response(
            request.form['SAMLResponse'],
            BINDING_HTTP_REDIRECT
        )
        state.sync()
        if response and response.status_ok():
            session.clear()
            return redirect(next_page)
        else:
            current_app.logger.error('Unknown error during the logout')
            abort(400)

    # logout started by the IdP
    elif 'SAMLRequest' in request.form:
        current_app.logger.debug('Receiving a logout request from the IdP')
        subject_id = _get_name_id(session)
        if subject_id is None:
            current_app.logger.warning(
                'The session does not contain the subject id for user {0} '
                'Performing local logout'.format(
                    session['eduPersonPrincipalName']
                )
            )
            session.clear()
            return redirect(next_page)
        else:
            http_info = client.handle_logout_request(
                request.form['SAMLRequest'],
                subject_id,
                BINDING_HTTP_REDIRECT,
                relay_state=request.form['RelayState']
            )
            state.sync()
            location = get_location(http_info)
            session.clear()
            return redirect(location)
    current_app.logger.error('No SAMLResponse or SAMLRequest parameter found')
    abort(400)

@authn_views.route('/signup-authn', methods=['GET'])
def signup_authn():
    current_app.logger.debug('Authenticating signing up user')
    location_on_fail = current_app.config.get('SIGNUP_AUTHN_FAILURE_REDIRECT_URL')
    location_on_success = current_app.config.get('SIGNUP_AUTHN_SUCCESS_REDIRECT_URL')

    eppn = check_previous_identification(session.signup)
    if eppn is not None:
        loa = get_loa(current_app.config.get('AVAILABLE_LOA'), None)  # With no session_info lowest loa will be returned
        current_app.logger.info("Starting pre-login actions " "for eppn: {})".format(eppn))
        try:
            user = current_app.central_userdb.get_user_by_eppn(eppn)
        except current_app.central_userdb.exceptions.UserDoesNotExist:
            current_app.logger.error('No user with eduPersonPrincipalName = {} found'.format(eppn))
        except current_app.central_userdb.exceptions.MultipleUsersReturned:
            current_app.logger.error("There are more than one user with eduPersonPrincipalName = {}".format(eppn))
        else:
            if user.locked_identity.count > 0:
                # This user has previously verified their account and is not new, this should not happen.
                current_app.logger.error('Not new user {} tried to log in using token login'.format(user))
                return redirect(location_on_fail)
            session['eduPersonPrincipalName'] = user.eppn
            session['user_eppn'] = user.eppn
            session['user_is_logged_in'] = True
            session['eduPersonAssurance'] = loa

            response = redirect(location_on_success)
            #session.set_cookie(response)  # Is this needed as session_interface.save_session will be called?
            current_app.logger.info('Successful token login, redirecting user {} to {}'.format(user,
                                                                                location_on_success))
            return response

    current_app.logger.info('Token login failed, redirecting user to {}'.format(location_on_fail))
    return redirect(location_on_fail)

@authn_views.route('/saml2-metadata')
def metadata():
    """
    Returns an XML with the SAML 2.0 metadata for this
    SP as configured in the saml2_settings.py file.
    """
    metadata = entity_descriptor(current_app.saml2_config)
    response = make_response(metadata.to_string(), 200)
    response.headers['Content-Type'] = "text/xml; charset=utf8"
    return response
<|endoftext|>


<commit_before># Copyright (c) 2019-2020, Manfred Moitzi
# License: MIT-License
from typing import TYPE_CHECKING, Iterable, cast, Union, List, Set
from contextlib import contextmanager
import logging
from ezdxf.lldxf import validator, const
from ezdxf.lldxf.attributes import (
    DXFAttr, DXFAttributes, DefSubclass, RETURN_DEFAULT, group_code_mapping,
)
from ezdxf.audit import AuditError
from .dxfentity import base_class, SubclassProcessor, DXFEntity
from .dxfobj import DXFObject
from .factory import register_entity
from .objectcollection import ObjectCollection

logger = logging.getLogger('ezdxf')

if TYPE_CHECKING:
    from ezdxf.eztypes import (
        TagWriter, Drawing, DXFNamespace, Auditor, EntityDB,
    )

__all__ = ['DXFGroup', 'GroupCollection']

acdb_group = DefSubclass('AcDbGroup', {
    # Group description
    'description': DXFAttr(300, default=''),

    # 1 = Unnamed
    # 0 = Named
    'unnamed': DXFAttr(
        70, default=1, validator=validator.is_integer_bool,
        fixer=RETURN_DEFAULT,
    ),

    # 1 = Selectable
    # 0 = Not selectable
    'selectable': DXFAttr(
        71, default=1,
        validator=validator.is_integer_bool,
        fixer=RETURN_DEFAULT,
    ),

    # 340: Hard-pointer handle to entity in group (one entry per object)
})
acdb_group_group_codes = group_code_mapping(acdb_group)
GROUP_ITEM_CODE = 340

@register_entity
class DXFGroup(DXFObject):
    """ Groups are not allowed in block definitions, and each entity can only
    reside in one group, so cloning of groups creates also new entities.

    """
    DXFTYPE = 'GROUP'
    DXFATTRIBS = DXFAttributes(base_class, acdb_group)

    def __init__(self):
        super().__init__()
        self._handles: Set[str] = set()  # only needed at the loading stage
        self._data: List[DXFEntity] = []

    def copy(self):
        raise const.DXFTypeError('Copying of GROUP not supported.')

    def load_dxf_attribs(self,
                         processor: SubclassProcessor = None) -> 'DXFNamespace':
        dxf = super().load_dxf_attribs(processor)
        if processor:
            tags = processor.fast_load_dxfattribs(
                dxf, acdb_group_group_codes, 1, log=False)
            self.load_group(tags)
        return dxf

    def load_group(self, tags):
        for code, value in tags:
            if code == GROUP_ITEM_CODE:
                # First store handles, because at this point, objects
                # are not stored in the EntityDB:
                self._handles.add(value)

    def preprocess_export(self, tagwriter: 'TagWriter') -> bool:
        self.purge(self.doc.entitydb)
        return True  # export even empty groups

    def export_entity(self, tagwriter: 'TagWriter') -> None:
        """ Export entity specific data as DXF tags. """
        super().export_entity(tagwriter)
        tagwriter.write_tag2(const.SUBCLASS_MARKER, acdb_group.name)
        self.dxf.export_dxf_attribs(tagwriter, [
            'description', 'unnamed', 'selectable'])
        self.export_group(tagwriter)

    def export_group(self, tagwriter: 'TagWriter'):
        for entity in self._data:
            tagwriter.write_tag2(GROUP_ITEM_CODE, entity.dxf.handle)

    def __iter__(self) -> Iterable[DXFEntity]:
        """ Iterate over all DXF entities in :class:`DXFGroup` as instances of
        :class:`DXFGraphic` or inherited (LINE, CIRCLE, ...).

        """
        return (e for e in self._data if e.is_alive)

    def __len__(self) -> int:
        """ Returns the count of DXF entities in :class:`DXFGroup`. """
        return len(self._data)

    def __getitem__(self, item):
        """ Returns entities by standard Python indexing and slicing. """
        return self._data[item]

    def __contains__(self, item: Union[str, DXFEntity]) -> bool:
        """ Returns ``True`` if item is in :class:`DXFGroup`. `item` has to be
        a handle string or an object of type :class:`DXFEntity` or inherited.

        """
        handle = item if isinstance(item, str) else item.dxf.handle
        return handle in set(self.handles())

    def handles(self) -> Iterable[str]:
        """ Iterable of handles of all DXF entities in :class:`DXFGroup`. """
        return (entity.dxf.handle for entity in self)

    def post_load_hook(self, doc: 'Drawing') -> None:
        super().post_load_hook(doc)
        db_get = doc.entitydb.get

        def entities():
            for handle in self._handles:
                entity = db_get(handle)
                if entity and entity.is_alive:
                    yield entity

        try:
            self.set_data(entities())
        except const.DXFStructureError as e:
            logger.error(str(e))
        del self._handles  # all referenced entities are stored in _data

    @contextmanager
    def edit_data(self) -> List[DXFEntity]:
        """ Context manager which yields all the group entities as
        standard Python list::

            with group.edit_data() as data:
               # add new entities to a group
               data.append(modelspace.add_line((0, 0), (3, 0)))
               # remove last entity from a group
               data.pop()

        """
        data = list(self)
        yield data
        self.set_data(data)

    def set_data(self, entities: Iterable[DXFEntity]) -> None:
        """  Set `entities` as new group content, entities should be an iterable
        :class:`DXFGraphic` or inherited (LINE, CIRCLE, ...).
        Raises :class:`DXFValueError` if not all entities be on the same layout
        (modelspace or any paperspace layout but not block)

        """
        entities = list(entities)
        if not all_entities_on_same_layout(entities):
            raise const.DXFStructureError(
                "All entities have to be in the same layout and not in "
                "a block layout."
            )
        self.clear()
        self._data = entities

    def extend(self, entities: Iterable[DXFEntity]) -> None:
        """ Add `entities` to :class:`DXFGroup`. """
        self._data.extend(entities)

    def clear(self) -> None:
        """ Remove all entities from :class:`DXFGroup`, does not delete any
        drawing entities referenced by this group.

        """
        self._data = []

    def audit(self, auditor: 'Auditor') -> None:
        """ Remove invalid handles from :class:`DXFGroup`.

        Invalid handles are: deleted entities, not all entities in the same
        layout or entities in a block layout.

        """
        # Remove destroyed or invalid entities:
        self.purge(auditor.entitydb)
        if not all_entities_on_same_layout(self._data):
            auditor.fixed_error(
                code=AuditError.GROUP_ENTITIES_IN_DIFFERENT_LAYOUTS,
                message=f'Cleared {str(self)}, not all entities are located in '
                        f'the same layout.',
            )
            self.clear()

    def _has_valid_owner(self, entity, db: 'EntityDB') -> bool:
        # no owner -> no layout association
        if entity.dxf.owner is None:
            return False
        owner = db.get(entity.dxf.owner)
        # owner does not exist or is destroyed -> no layout association
        if owner is None or not owner.is_alive:
            return False
        # owner block_record.layout is 0 if entity is in a block definition,
        # which is not allowed:
        valid = owner.dxf.layout != '0'
        if not valid:
            logger.debug(
                f'Removed {str(entity)} from {str(self)}, because entity is '
                f'located in a block layout.')
        return valid

    def _filter_invalid_entities(self, db: 'EntityDB') -> List[DXFEntity]:
        assert db is not None
        return [e for e in self._data
                if e.is_alive and self._has_valid_owner(e, db)]

    def purge(self, db: 'EntityDB') -> None:
        """ Remove invalid group entities. """
        self._data = self._filter_invalid_entities(db)

def all_entities_on_same_layout(entities: Iterable[DXFEntity]):
    """ Check if all entities are on the same layout (model space or any paper
    layout but not block).

    """
    owners = set(entity.dxf.owner for entity in entities)
    # 0 for no entities; 1 for all entities on the same layout
    return len(owners) < 2

class GroupCollection(ObjectCollection):
    def __init__(self, doc: 'Drawing'):
        super().__init__(doc, dict_name='ACAD_GROUP', object_type='GROUP')
        self._next_unnamed_number = 0

    def groups(self) -> Iterable[DXFGroup]:
        """ Iterable of all existing groups. """
        for name, group in self:
            yield group

    def next_name(self) -> str:
        name = self._next_name()
        while name in self:
            name = self._next_name()
        return name

    def _next_name(self) -> str:
        self._next_unnamed_number += 1
        return f"*A{self._next_unnamed_number}"

    def new(self, name: str = None, description: str = "",
            selectable: bool = True) -> DXFGroup:
        r""" Creates a new group. If `name` is ``None`` an unnamed group is
        created, which has an automatically generated name like "\*Annnn".

        Args:
            name: group name as string
            description: group description as string
            selectable: group is selectable if ``True``

        """
        if name in self:
            raise const.DXFValueError(f"GROUP '{name}' already exists.")

        if name is None:
            name = self.next_name()
            unnamed = 1
        else:
            unnamed = 0
        # The group name isn't stored in the group entity itself.
        dxfattribs = {
            'description': description,
            'unnamed': unnamed,
            'selectable': int(bool(selectable)),
        }
        return cast(DXFGroup, self._new(name, dxfattribs))

    def delete(self, group: Union[DXFGroup, str]) -> None:
        """ Delete `group`, `group` can be an object of type :class:`DXFGroup`
        or a group name as string.

        """
        # Delete group by name:
        if isinstance(group, str):
            name = group
        elif group.dxftype() == 'GROUP':
            name = get_group_name(group, self.entitydb)
        else:
            raise TypeError(group.dxftype())

        if name in self:
            super().delete(name)
        else:
            raise const.DXFValueError("GROUP not in group table registered.")

    def audit(self, auditor: 'Auditor') -> None:
        """ Removes empty groups and invalid handles from all groups. """
        trash = []
        for name, group in self:
            group.audit(auditor)
            if not len(group):  # remove empty group
                # do not delete groups while iterating over groups!
                trash.append(name)

        # now delete empty groups
        for name in trash:
            auditor.fixed_error(
                code=AuditError.REMOVE_EMPTY_GROUP,
                message=f'Removed empty group "{name}".',
            )
            self.delete(name)

def get_group_name(group: DXFGroup, db: 'EntityDB') -> str:
    """ Get name of `group`. """
    group_table = cast('Dictionary', db[group.dxf.owner])
    for name, entity in group_table.items():
        if entity is group:
            return name
<commit_msg>clarify debug message<commit_after># Copyright (c) 2019-2020, Manfred Moitzi
# License: MIT-License
from typing import TYPE_CHECKING, Iterable, cast, Union, List, Set
from contextlib import contextmanager
import logging
from ezdxf.lldxf import validator, const
from ezdxf.lldxf.attributes import (
    DXFAttr, DXFAttributes, DefSubclass, RETURN_DEFAULT, group_code_mapping,
)
from ezdxf.audit import AuditError
from .dxfentity import base_class, SubclassProcessor, DXFEntity
from .dxfobj import DXFObject
from .factory import register_entity
from .objectcollection import ObjectCollection

logger = logging.getLogger('ezdxf')

if TYPE_CHECKING:
    from ezdxf.eztypes import (
        TagWriter, Drawing, DXFNamespace, Auditor, EntityDB,
    )

__all__ = ['DXFGroup', 'GroupCollection']

acdb_group = DefSubclass('AcDbGroup', {
    # Group description
    'description': DXFAttr(300, default=''),

    # 1 = Unnamed
    # 0 = Named
    'unnamed': DXFAttr(
        70, default=1, validator=validator.is_integer_bool,
        fixer=RETURN_DEFAULT,
    ),

    # 1 = Selectable
    # 0 = Not selectable
    'selectable': DXFAttr(
        71, default=1,
        validator=validator.is_integer_bool,
        fixer=RETURN_DEFAULT,
    ),

    # 340: Hard-pointer handle to entity in group (one entry per object)
})
acdb_group_group_codes = group_code_mapping(acdb_group)
GROUP_ITEM_CODE = 340

@register_entity
class DXFGroup(DXFObject):
    """ Groups are not allowed in block definitions, and each entity can only
    reside in one group, so cloning of groups creates also new entities.

    """
    DXFTYPE = 'GROUP'
    DXFATTRIBS = DXFAttributes(base_class, acdb_group)

    def __init__(self):
        super().__init__()
        self._handles: Set[str] = set()  # only needed at the loading stage
        self._data: List[DXFEntity] = []

    def copy(self):
        raise const.DXFTypeError('Copying of GROUP not supported.')

    def load_dxf_attribs(self,
                         processor: SubclassProcessor = None) -> 'DXFNamespace':
        dxf = super().load_dxf_attribs(processor)
        if processor:
            tags = processor.fast_load_dxfattribs(
                dxf, acdb_group_group_codes, 1, log=False)
            self.load_group(tags)
        return dxf

    def load_group(self, tags):
        for code, value in tags:
            if code == GROUP_ITEM_CODE:
                # First store handles, because at this point, objects
                # are not stored in the EntityDB:
                self._handles.add(value)

    def preprocess_export(self, tagwriter: 'TagWriter') -> bool:
        self.purge(self.doc.entitydb)
        return True  # export even empty groups

    def export_entity(self, tagwriter: 'TagWriter') -> None:
        """ Export entity specific data as DXF tags. """
        super().export_entity(tagwriter)
        tagwriter.write_tag2(const.SUBCLASS_MARKER, acdb_group.name)
        self.dxf.export_dxf_attribs(tagwriter, [
            'description', 'unnamed', 'selectable'])
        self.export_group(tagwriter)

    def export_group(self, tagwriter: 'TagWriter'):
        for entity in self._data:
            tagwriter.write_tag2(GROUP_ITEM_CODE, entity.dxf.handle)

    def __iter__(self) -> Iterable[DXFEntity]:
        """ Iterate over all DXF entities in :class:`DXFGroup` as instances of
        :class:`DXFGraphic` or inherited (LINE, CIRCLE, ...).

        """
        return (e for e in self._data if e.is_alive)

    def __len__(self) -> int:
        """ Returns the count of DXF entities in :class:`DXFGroup`. """
        return len(self._data)

    def __getitem__(self, item):
        """ Returns entities by standard Python indexing and slicing. """
        return self._data[item]

    def __contains__(self, item: Union[str, DXFEntity]) -> bool:
        """ Returns ``True`` if item is in :class:`DXFGroup`. `item` has to be
        a handle string or an object of type :class:`DXFEntity` or inherited.

        """
        handle = item if isinstance(item, str) else item.dxf.handle
        return handle in set(self.handles())

    def handles(self) -> Iterable[str]:
        """ Iterable of handles of all DXF entities in :class:`DXFGroup`. """
        return (entity.dxf.handle for entity in self)

    def post_load_hook(self, doc: 'Drawing') -> None:
        super().post_load_hook(doc)
        db_get = doc.entitydb.get

        def entities():
            for handle in self._handles:
                entity = db_get(handle)
                if entity and entity.is_alive:
                    yield entity

        try:
            self.set_data(entities())
        except const.DXFStructureError as e:
            logger.error(str(e))
        del self._handles  # all referenced entities are stored in _data

    @contextmanager
    def edit_data(self) -> List[DXFEntity]:
        """ Context manager which yields all the group entities as
        standard Python list::

            with group.edit_data() as data:
               # add new entities to a group
               data.append(modelspace.add_line((0, 0), (3, 0)))
               # remove last entity from a group
               data.pop()

        """
        data = list(self)
        yield data
        self.set_data(data)

    def set_data(self, entities: Iterable[DXFEntity]) -> None:
        """  Set `entities` as new group content, entities should be an iterable
        :class:`DXFGraphic` or inherited (LINE, CIRCLE, ...).
        Raises :class:`DXFValueError` if not all entities be on the same layout
        (modelspace or any paperspace layout but not block)

        """
        entities = list(entities)
        if not all_entities_on_same_layout(entities):
            raise const.DXFStructureError(
                "All entities have to be in the same layout and not in "
                "a block layout."
            )
        self.clear()
        self._data = entities

    def extend(self, entities: Iterable[DXFEntity]) -> None:
        """ Add `entities` to :class:`DXFGroup`. """
        self._data.extend(entities)

    def clear(self) -> None:
        """ Remove all entities from :class:`DXFGroup`, does not delete any
        drawing entities referenced by this group.

        """
        self._data = []

    def audit(self, auditor: 'Auditor') -> None:
        """ Remove invalid handles from :class:`DXFGroup`.

        Invalid handles are: deleted entities, not all entities in the same
        layout or entities in a block layout.

        """
        # Remove destroyed or invalid entities:
        self.purge(auditor.entitydb)
        if not all_entities_on_same_layout(self._data):
            auditor.fixed_error(
                code=AuditError.GROUP_ENTITIES_IN_DIFFERENT_LAYOUTS,
                message=f'Cleared {str(self)}, not all entities are located in '
                        f'the same layout.',
            )
            self.clear()

    def _has_valid_owner(self, entity, db: 'EntityDB') -> bool:
        # no owner -> no layout association
        if entity.dxf.owner is None:
            return False
        owner = db.get(entity.dxf.owner)
        # owner does not exist or is destroyed -> no layout association
        if owner is None or not owner.is_alive:
            return False
        # owner block_record.layout is 0 if entity is in a block definition,
        # which is not allowed:
        valid = owner.dxf.layout != '0'
        if not valid:
            logger.debug(
                f"{str(entity)} in {str(self)} is located in a block layout, "
                f"which is not allowed")
        return valid

    def _filter_invalid_entities(self, db: 'EntityDB') -> List[DXFEntity]:
        assert db is not None
        return [e for e in self._data
                if e.is_alive and self._has_valid_owner(e, db)]

    def purge(self, db: 'EntityDB') -> None:
        """ Remove invalid group entities. """
        self._data = self._filter_invalid_entities(db)

def all_entities_on_same_layout(entities: Iterable[DXFEntity]):
    """ Check if all entities are on the same layout (model space or any paper
    layout but not block).

    """
    owners = set(entity.dxf.owner for entity in entities)
    # 0 for no entities; 1 for all entities on the same layout
    return len(owners) < 2

class GroupCollection(ObjectCollection):
    def __init__(self, doc: 'Drawing'):
        super().__init__(doc, dict_name='ACAD_GROUP', object_type='GROUP')
        self._next_unnamed_number = 0

    def groups(self) -> Iterable[DXFGroup]:
        """ Iterable of all existing groups. """
        for name, group in self:
            yield group

    def next_name(self) -> str:
        name = self._next_name()
        while name in self:
            name = self._next_name()
        return name

    def _next_name(self) -> str:
        self._next_unnamed_number += 1
        return f"*A{self._next_unnamed_number}"

    def new(self, name: str = None, description: str = "",
            selectable: bool = True) -> DXFGroup:
        r""" Creates a new group. If `name` is ``None`` an unnamed group is
        created, which has an automatically generated name like "\*Annnn".

        Args:
            name: group name as string
            description: group description as string
            selectable: group is selectable if ``True``

        """
        if name in self:
            raise const.DXFValueError(f"GROUP '{name}' already exists.")

        if name is None:
            name = self.next_name()
            unnamed = 1
        else:
            unnamed = 0
        # The group name isn't stored in the group entity itself.
        dxfattribs = {
            'description': description,
            'unnamed': unnamed,
            'selectable': int(bool(selectable)),
        }
        return cast(DXFGroup, self._new(name, dxfattribs))

    def delete(self, group: Union[DXFGroup, str]) -> None:
        """ Delete `group`, `group` can be an object of type :class:`DXFGroup`
        or a group name as string.

        """
        # Delete group by name:
        if isinstance(group, str):
            name = group
        elif group.dxftype() == 'GROUP':
            name = get_group_name(group, self.entitydb)
        else:
            raise TypeError(group.dxftype())

        if name in self:
            super().delete(name)
        else:
            raise const.DXFValueError("GROUP not in group table registered.")

    def audit(self, auditor: 'Auditor') -> None:
        """ Removes empty groups and invalid handles from all groups. """
        trash = []
        for name, group in self:
            group.audit(auditor)
            if not len(group):  # remove empty group
                # do not delete groups while iterating over groups!
                trash.append(name)

        # now delete empty groups
        for name in trash:
            auditor.fixed_error(
                code=AuditError.REMOVE_EMPTY_GROUP,
                message=f'Removed empty group "{name}".',
            )
            self.delete(name)

def get_group_name(group: DXFGroup, db: 'EntityDB') -> str:
    """ Get name of `group`. """
    group_table = cast('Dictionary', db[group.dxf.owner])
    for name, entity in group_table.items():
        if entity is group:
            return name
<|endoftext|>