<commit_before>"""Tests for cement.ext.ext_memcached."""

from time import sleep
from random import random
from cement.utils import test
from cement.utils.misc import init_defaults

class MemcachedExtTestCase(test.CementTestCase):

    def setUp(self):
        super(MemcachedExtTestCase, self).setUp()
        self.key = "cement-tests-random-key-%s" % random()
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = '127.0.0.1, localhost'
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()

    def tearDown(self):
        super(MemcachedExtTestCase, self).tearDown()
        self.app.cache.delete(self.key)

    def test_memcache_list_type_config(self):
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = ['127.0.0.1', 'localhost']
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()
        self.eq(self.app.config.get('cache.memcached', 'hosts'),
                ['127.0.0.1', 'localhost'])

    def test_memcache_str_type_config(self):
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = '127.0.0.1, localhost'
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()
        self.eq(self.app.config.get('cache.memcached', 'hosts'),
                ['127.0.0.1', 'localhost'])

    def test_memcached_set(self):
        self.app.cache.set(self.key, 1001)
        self.eq(self.app.cache.get(self.key), 1001)

    def test_memcached_get(self):
        # get empty value
        self.app.cache.delete(self.key)
        self.eq(self.app.cache.get(self.key), None)

        # get empty value with fallback
        self.eq(self.app.cache.get(self.key, 1234), 1234)

    def test_memcached_delete(self):
        self.app.cache.delete(self.key)

    def test_memcached_purge(self):
        self.app.cache.set(self.key, 1002)
        self.app.cache.purge()
        self.eq(self.app.cache.get(self.key), None)

    def test_memcache_expire(self):
        self.app.cache.set(self.key, 1003, time=2)
        sleep(3)
        self.eq(self.app.cache.get(self.key), None)
<commit_msg>Support Environment Variable Overrides for Tests<commit_after>"""Tests for cement.ext.ext_memcached."""

import os
from time import sleep
from random import random
from cement.utils import test
from cement.utils.misc import init_defaults

if 'MEMCACHED_HOSTS' in os.environ.keys():
    memcached_hosts = os.environ['MEMCACHED_HOSTS']
else:
    memcached_hosts = 'memcached'

class MemcachedExtTestCase(test.CementTestCase):

    def setUp(self):
        super(MemcachedExtTestCase, self).setUp()
        self.key = "cement-tests-random-key-%s" % random()
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = 'memcached'
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()

    def tearDown(self):
        super(MemcachedExtTestCase, self).tearDown()

    def test_memcache_list_type_config(self):
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = ["127.0.0.1", "localhost"]
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()
        self.eq(self.app.config.get('cache.memcached', 'hosts'),
                ['127.0.0.1', 'localhost'])

    def test_memcache_str_type_config(self):
        defaults = init_defaults('tests', 'cache.memcached')
        defaults['cache.memcached']['hosts'] = "127.0.0.1,localhost"
        self.app = self.make_app('tests',
                                 config_defaults=defaults,
                                 extensions=['memcached'],
                                 cache_handler='memcached',
                                 )
        self.app.setup()
        self.eq(self.app.config.get('cache.memcached', 'hosts'),
                ['127.0.0.1', 'localhost'])

    def test_memcached_set(self):
        self.app.cache.set(self.key, 1001)
        self.eq(self.app.cache.get(self.key), 1001)
        self.app.cache.delete(self.key)

    def test_memcached_get(self):
        # get empty value
        self.app.cache.delete(self.key)
        self.eq(self.app.cache.get(self.key), None)

        # get empty value with fallback
        self.eq(self.app.cache.get(self.key, 1234), 1234)

    def test_memcached_delete(self):
        self.app.cache.delete(self.key)

    def test_memcached_purge(self):
        self.app.cache.set(self.key, 1002)
        self.app.cache.purge()
        self.eq(self.app.cache.get(self.key), None)

    def test_memcache_expire(self):
        self.app.cache.set(self.key, 1003, time=2)
        sleep(3)
        self.eq(self.app.cache.get(self.key), None)
<|endoftext|>


<commit_before>from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

import config

'''
dev code
= db_session.query(Cloud).filter(Cloud.cloud_type=="openstack")
db_session.merge(new_flav)
db_session.commit()
'''

def get_quotas(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Quota = Base.classes.cloud_quotas
    quota_list = db_session.query(Quota)
    return quota_list

def get_vms(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    VM = Base.classes.cloud_vm
    vm_list = db_session.query(VM)
    return vm_list

def get_flavors(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Flavors = Base.classes.cloud_flavors
    flavor_list = db_session.query(Flavors)
    return flavor_list

def get_images(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Images = Base.classes.cloud_images
    image_list = db_session.query(Images)
    return image_list

def get_networks(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Networks = Base.classes.cloud_networks
    network_list = db_session.query(Networks)
    return network_list

def get_groups(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Groups = Base.classes.csv2_groups
    group_list = db_session.query(Groups)
    return group_list

# may be best to query the view instead of the resources table
def get_group_resources(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    GroupResources = Base.classes.csv2_group_resources
    group_resources_list = db_session.query(GroupResources)
    return group_resources_list
<commit_msg>add getters for condor data<commit_after>from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.ext.automap import automap_base

import config

'''
dev code
= db_session.query(Cloud).filter(Cloud.cloud_type=="openstack")
db_session.merge(new_flav)
db_session.commit()
'''

def get_quotas(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Quota = Base.classes.cloud_quotas
    quota_list = db_session.query(Quota)
    return quota_list

def get_vms(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    VM = Base.classes.cloud_vm
    vm_list = db_session.query(VM)
    return vm_list

def get_flavors(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Flavors = Base.classes.cloud_flavors
    flavor_list = db_session.query(Flavors)
    return flavor_list

def get_images(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Images = Base.classes.cloud_images
    image_list = db_session.query(Images)
    return image_list

def get_networks(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Networks = Base.classes.cloud_networks
    network_list = db_session.query(Networks)
    return network_list

def get_groups(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Groups = Base.classes.csv2_groups
    group_list = db_session.query(Groups)
    return group_list

# may be best to query the view instead of the resources table
def get_group_resources(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    GroupResources = Base.classes.csv2_group_resources
    group_resources_list = db_session.query(GroupResources)
    return group_resources_list

def get_condor_jobs(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Jobs = Base.classes.condor_jobs
    job_list = db_session.query(Jobs)
    return job_list

def get_condor_machines(filter=None):
    engine = create_engine("mysql://" + config.db_user + ":" + config.db_password + "@" + config.db_host + ":" + str(config.db_port) + "/" + config.db_name)
    Base.prepare(engine, reflect=True)
    db_session = Session(engine)
    Machines = Base.classes.condor_machines
    machine_list = db_session.query(Machines)
    return machine_list
<|endoftext|>


<commit_before>from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.csrf import requires_csrf_token
from django.http import HttpResponse
from django.core.exceptions import PermissionDenied

from django.contrib.auth.models import User #to get auth_user table
from .models import user as csv2_user
from . import config

from .view_utils import \
    db_close, \
    db_execute, \
    db_open, \
    getAuthUser, \
    getcsv2User, \
    getSuperUserStatus, \
    lno, \
    qt, \
    qt_filter_get, \
    render, \
    set_user_groups, \
    table_fields, \
    validate_fields, \
    verifyUser
from collections import defaultdict
import bcrypt

from sqlalchemy import exists
from sqlalchemy.sql import select
from sqlalchemy.sql import and_
from lib.schema import *
import sqlalchemy.exc

# lno: VV - error code identifier.

#-------------------------------------------------------------------------------

VM_KEYS = {
    'auto_active_group': True,
    # Named argument formats (anything else is a string).
    'format': {
        'vm_selected':         'ignore',
        'vm_option':           ['kill', 'retire', 'manctl', 'sysctl'],

        'csrfmiddlewaretoken': 'ignore',
        'group':               'ignore',
        },
    }

LIST_KEYS = {
    # Named argument formats (anything else is a string).
    'format': {
        'csrfmiddlewaretoken':     'ignore',
        'group':                   'ignore',
        },
    }

#-------------------------------------------------------------------------------

@requires_csrf_token
def list(
    request,
    selector='::::',
    response_code=0,
    message=None,
    active_user=None,
    user_groups=None,
    ):

    if not verifyUser(request):
        raise PermissionDenied

    # open the database.
    db_engine, db_session, db_connection, db_map = db_ctl = db_open()

    # Retrieve the active user, associated group list and optionally set the active group.
    if not active_user:
        rc, msg, active_user, user_groups = set_user_groups(request, db_ctl)
        if rc != 0:
            db_close(db_ctl)
            return render(request, 'csv2/clouds.html', {'response_code': 1, 'message': msg})

    # Validate input fields (should be none).
    if not message:
        rc, msg, fields, tables, columns = validate_fields(request, [LIST_KEYS], db_ctl, [], active_user)
        if rc != 0:
            db_close(db_ctl)
            return render(request, 'csv2/vms.html', {'response_code': 1, 'message': '%s vm list, %s' % (lno('GV06'), msg)})

    # Retrieve VM information.
    s = select([view_vms]).where(view_vms.c.group_name == active_user.active_group)
    vm_list = qt(db_connection.execute(s), filter=qt_filter_get(['cloud_name', 'poller_status', 'hostname'], selector.split('::')))

    # Retrieve available Clouds.
    s = select([view_cloud_status]).where(view_cloud_status.c.group_name == active_user.active_group)
    cloud_list = qt(db_connection.execute(s))

    db_close(db_ctl)

    # Render the page.
    context = {
            'active_user': active_user,
            'active_group': active_user.active_group,
            'user_groups': user_groups,
            'cloud_list': cloud_list,
            'vm_list': vm_list,
            'response_code': response_code,
            'message': message,
            'enable_glint': config.enable_glint
        }

    return render(request, 'csv2/vms.html', context)

#-------------------------------------------------------------------------------

@requires_csrf_token
def update(request, selector='::::'):
    """
    Update VMs.
    """

    if not verifyUser(request):
        raise PermissionDenied

    if request.method == 'POST' and 'vm_option' in request.POST:
        if 'vm_selected' in request.POST:
            # open the database.
            db_engine, db_session, db_connection, db_map = db_ctl = db_open()

            # Retrieve the active user, associated group list and optionally set the active group.
            rc, msg, active_user, user_groups = set_user_groups(request, db_ctl)
            if rc != 0:
                db_close(db_ctl)
                return list(request, response_code=1, message='%s %s' % (lno('CV12'), msg), active_user=active_user, user_groups=user_groups)

            # Validate input fields.
            rc, msg, fields, tables, columns = validate_fields(request, [VM_KEYS], db_ctl, ['csv2_vms,n', 'condor_machines,n'], active_user)
            if rc != 0:
                db_close(db_ctl)
                return list(request, response_code=1, message='%s cloud update %s' % (lno('VV00'), msg), active_user=active_user, user_groups=user_groups)

            if fields['vm_option'] == 'kill':
                table = tables['csv2_vms']
                verb = 'killed'
            elif fields['vm_option'] == 'retire':
                table = tables['condor_machines']
                verb = 'retired'
            elif fields['vm_option'] == 'manctl':
                table = tables['csv2_vms']
                verb = 'set to manual control'
            elif fields['vm_option'] == 'sysctl':
                table = tables['csv2_vms']
                verb = 'set to system control'
            else:
                return list(request, response_code=1, message='%s vm update, option "%s" is invalid.' % (lno('GV41'), fields['vm_option']))

            if 'vm_hostname' in fields:
                if isinstance(fields['vm_hostname'], list):
                    hosts = fields['vm_hostname']
                else:
                    hosts = fields['vm_hostname'].split(',')
            else:
                hosts = None

            # Retrieve VM information.
            s = select([view_vms]).where((view_vms.c.group_name == active_user.active_group) and (view_vms.c.foreign_vm == 0))
            vm_list = qt(db_connection.execute(s), filter=qt_filter_get(['cloud_name', 'poller_status', 'hostname'], selector.split('::')))

            count = 0
            for vm in vm_list:
                if fields['vm_option'] == 'kill':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'terminate': 1})
                elif fields['vm_option'] == 'retire':
                    update = table.update().where(table.c.hostname == vm['hostname']).values({'condor_off': 1})
                elif fields['vm_option'] == 'manctl':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'manual_control': 1})
                elif fields['vm_option'] == 'sysctl':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'manual_control': 0})

                rc, msg = db_execute(db_ctl, table.update().where(table.c.vmid == vm['vmid']).values(control), allow_no_rows=True)
                if rc == 0:
                    count += msg
                else:
                    db_close(db_ctl)
                    return list(request, response_code=1, message='%s VM update (%s) failed - %s' % (lno('GV41'), fields['vm_option'], msg))

            db_close(db_ctl, commit=True)
            return list(request, response_code=0, message='vm update, VMs %s=%s.' % (verb, count))

    ### Bad request.
        else:
            return list(request, response_code=1, message='%s vm update, no VMs were selected to perform the operation %s.' % (lno('CV15'), request.POST['vm_option']))
    else:
        if request.method != 'POST':
            return list(request, response_code=1, message='%s cloud update, invalid method "%s" specified.' % (lno('CV15'), request.method))
        else:
            return list(request, response_code=1, message='%s cloud update, the vm-option is required and must be one of the following: %s.' % (lno('CV15'), VM_KEYS['format']['vm_option']))
<commit_msg>Added logic to create cvs of vm_ids. WPI<commit_after>from django.shortcuts import render, get_object_or_404, redirect
from django.views.decorators.csrf import requires_csrf_token
from django.http import HttpResponse
from django.core.exceptions import PermissionDenied

from django.contrib.auth.models import User #to get auth_user table
from .models import user as csv2_user
from . import config

from .view_utils import \
    db_close, \
    db_execute, \
    db_open, \
    getAuthUser, \
    getcsv2User, \
    getSuperUserStatus, \
    lno, \
    qt, \
    qt_filter_get, \
    render, \
    set_user_groups, \
    table_fields, \
    validate_fields, \
    verifyUser
from collections import defaultdict
import bcrypt

from sqlalchemy import exists
from sqlalchemy.sql import select
from sqlalchemy.sql import and_
from lib.schema import *
import sqlalchemy.exc

# lno: VV - error code identifier.

#-------------------------------------------------------------------------------

VM_KEYS = {
    'auto_active_group': True,
    # Named argument formats (anything else is a string).
    'format': {
        # 'vm_selected':         'ignore',
        'vm_option':           ['kill', 'retire', 'manctl', 'sysctl'],

        'csrfmiddlewaretoken': 'ignore',
        'group':               'ignore',
        },
    }

LIST_KEYS = {
    # Named argument formats (anything else is a string).
    'format': {
        'csrfmiddlewaretoken':     'ignore',
        'group':                   'ignore',
        },
    }

#-------------------------------------------------------------------------------

@requires_csrf_token
def list(
    request,
    selector='::::',
    response_code=0,
    message=None,
    active_user=None,
    user_groups=None,
    ):

    if not verifyUser(request):
        raise PermissionDenied

    # open the database.
    db_engine, db_session, db_connection, db_map = db_ctl = db_open()

    # Retrieve the active user, associated group list and optionally set the active group.
    if not active_user:
        rc, msg, active_user, user_groups = set_user_groups(request, db_ctl)
        if rc != 0:
            db_close(db_ctl)
            return render(request, 'csv2/clouds.html', {'response_code': 1, 'message': msg})

    # Validate input fields (should be none).
    if not message:
        rc, msg, fields, tables, columns = validate_fields(request, [LIST_KEYS], db_ctl, [], active_user)
        if rc != 0:
            db_close(db_ctl)
            return render(request, 'csv2/vms.html', {'response_code': 1, 'message': '%s vm list, %s' % (lno('GV06'), msg)})

    # Retrieve VM information.
    s = select([view_vms]).where(view_vms.c.group_name == active_user.active_group)
    vm_list = qt(db_connection.execute(s), filter=qt_filter_get(['cloud_name', 'poller_status', 'hostname'], selector.split('::')))

    # Retrieve available Clouds.
    s = select([view_cloud_status]).where(view_cloud_status.c.group_name == active_user.active_group)
    cloud_list = qt(db_connection.execute(s))

    db_close(db_ctl)

    # Render the page.
    context = {
            'active_user': active_user,
            'active_group': active_user.active_group,
            'user_groups': user_groups,
            'cloud_list': cloud_list,
            'vm_list': vm_list,
            'response_code': response_code,
            'message': message,
            'enable_glint': config.enable_glint
        }

    return render(request, 'csv2/vms.html', context)

#-------------------------------------------------------------------------------

@requires_csrf_token
def update(request, selector='::::'):
    """
    Update VMs.
    """

    if not verifyUser(request):
        raise PermissionDenied

    if request.method == 'POST' and 'vm_option' in request.POST:
        if 'vm_selected' in request.POST:
            temp_vm_id_list =  request.POST.getlist('vm_selected')
            vm_id_list = []
            # Remove ending "/" for each element
            for vm_id in temp_vm_id_list:
                vm_id_list.append(vm_id.strip('/'))
            # Form csv string of ids and add it to the selector
            vm_ids = ','.join(vm_id_list)
            selector = selector + vm_ids

            # open the database.
            db_engine, db_session, db_connection, db_map = db_ctl = db_open()

            # Retrieve the active user, associated group list and optionally set the active group.
            rc, msg, active_user, user_groups = set_user_groups(request, db_ctl)
            if rc != 0:
                db_close(db_ctl)
                return list(request, response_code=1, message='%s %s' % (lno('CV12'), msg), active_user=active_user, user_groups=user_groups)

            # Validate input fields.
            rc, msg, fields, tables, columns = validate_fields(request, [VM_KEYS], db_ctl, ['csv2_vms,n', 'condor_machines,n'], active_user)
            if rc != 0:
                db_close(db_ctl)
                return list(request, response_code=1, message='%s cloud update %s' % (lno('VV00'), msg), active_user=active_user, user_groups=user_groups)

            if fields['vm_option'] == 'kill':
                table = tables['csv2_vms']
                verb = 'killed'
            elif fields['vm_option'] == 'retire':
                table = tables['condor_machines']
                verb = 'retired'
            elif fields['vm_option'] == 'manctl':
                table = tables['csv2_vms']
                verb = 'set to manual control'
            elif fields['vm_option'] == 'sysctl':
                table = tables['csv2_vms']
                verb = 'set to system control'
            else:
                return list(request, response_code=1, message='%s vm update, option "%s" is invalid.' % (lno('GV41'), fields['vm_option']))

            if 'vm_hostname' in fields:
                if isinstance(fields['vm_hostname'], list):
                    hosts = fields['vm_hostname']
                else:
                    hosts = fields['vm_hostname'].split(',')
            else:
                hosts = None

            # Retrieve VM information.
            s = select([view_vms]).where((view_vms.c.group_name == active_user.active_group) and (view_vms.c.foreign_vm == 0))
            vm_list = qt(db_connection.execute(s), filter=qt_filter_get(['cloud_name', 'poller_status', 'hostname'], selector.split('::')))

            count = 0
            for vm in vm_list:
                if fields['vm_option'] == 'kill':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'terminate': 1})
                elif fields['vm_option'] == 'retire':
                    update = table.update().where(table.c.hostname == vm['hostname']).values({'condor_off': 1})
                elif fields['vm_option'] == 'manctl':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'manual_control': 1})
                elif fields['vm_option'] == 'sysctl':
                    update = table.update().where(table.c.vmid == vm['vmid']).values({'manual_control': 0})

                rc, msg = db_execute(db_ctl, table.update().where(table.c.vmid == vm['vmid']).values(control), allow_no_rows=True)
                if rc == 0:
                    count += msg
                else:
                    db_close(db_ctl)
                    return list(request, response_code=1, message='%s VM update (%s) failed - %s' % (lno('GV41'), fields['vm_option'], msg))

            db_close(db_ctl, commit=True)
            return list(request, response_code=0, message='vm update, VMs %s=%s.' % (verb, count))

    ### Bad request.
        else:
            return list(request, response_code=1, message='%s vm update, no VMs were selected to perform the operation %s.' % (lno('CV15'), request.POST['vm_option']))
    else:
        if request.method != 'POST':
            return list(request, response_code=1, message='%s cloud update, invalid method "%s" specified.' % (lno('CV15'), request.method))
        else:
            return list(request, response_code=1, message='%s cloud update, the vm-option is required and must be one of the following: %s.' % (lno('CV15'), VM_KEYS['format']['vm_option']))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
byceps.blueprints.news_admin.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2015 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from operator import attrgetter

from flask import g, request

from ...util.framework import create_blueprint, flash_error, flash_success
from ...util.templating import templated
from ...util.views import redirect_to

from ..authorization.decorators import permission_required
from ..authorization.registry import permission_registry
from ..brand.models import Brand
from ..news.models import CurrentVersionAssociation, Item, ItemVersion
from ..news import service
from ..news import signals
from ..party.models import Party
from ..snippet.models.snippet import Snippet

from .authorization import NewsItemPermission
from .forms import ItemCreateForm

blueprint = create_blueprint('news_admin', __name__)

permission_registry.register_enum(NewsItemPermission)

@blueprint.route('/')
@permission_required(NewsItemPermission.list)
@templated
def index():
    """List brands to choose from."""
    brands = Brand.query.all()
    return {
        'brands': brands,
    }

@blueprint.route('/<brand_id>')
@permission_required(NewsItemPermission.list)
@templated
def index_for_brand(brand_id):
    """List news items for that brand."""
    brand = get_brand_or_404(brand_id)
    items = Item.query.for_brand(brand).with_current_version().all()
    return {
        'brand': brand,
        'items': items,
    }

@blueprint.route('/for_brand/<brand_id>/create')
@permission_required(NewsItemPermission.create)
@templated
def create_form(brand_id, *, erroneous_form=None):
    """Show form to create a news item."""
    brand = get_brand_or_404(brand_id)

    form = erroneous_form if erroneous_form else ItemCreateForm()

    return {
        'brand': brand,
        'form': form,
    }

@blueprint.route('/for_brand/<brand_id>', methods=['POST'])
@permission_required(NewsItemPermission.create)
def create(brand_id):
    """Create a news item."""
    brand = get_brand_or_404(brand_id)

    form = ItemCreateForm(request.form)

    slug = form.slug.data.strip().lower()
    creator = g.current_user
    title = form.title.data.strip()
    body = form.body.data.strip()
    image_url_path = form.image_url_path.data.strip()

    item = service.create_item(brand, slug, creator, title, body,
        image_url_path=image_url_path)

    flash_success('Die News "{}" wurde angelegt.', item.title)
    signals.item_published.send(None, item=item)

    return redirect_to('.index_for_brand', brand_id=brand.id)

def get_brand_or_404(brand_id):
    return Brand.query.get_or_404(brand_id)
<commit_msg>Removed unused imports.<commit_after># -*- coding: utf-8 -*-

"""
byceps.blueprints.news_admin.views
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:Copyright: 2006-2015 Jochen Kupperschmidt
:License: Modified BSD, see LICENSE for details.
"""

from operator import attrgetter

from flask import g, request

from ...util.framework import create_blueprint, flash_error, flash_success
from ...util.templating import templated
from ...util.views import redirect_to

from ..authorization.decorators import permission_required
from ..authorization.registry import permission_registry
from ..brand.models import Brand
from ..news.models import Item
from ..news import service
from ..news import signals
from ..party.models import Party

from .authorization import NewsItemPermission
from .forms import ItemCreateForm

blueprint = create_blueprint('news_admin', __name__)

permission_registry.register_enum(NewsItemPermission)

@blueprint.route('/')
@permission_required(NewsItemPermission.list)
@templated
def index():
    """List brands to choose from."""
    brands = Brand.query.all()
    return {
        'brands': brands,
    }

@blueprint.route('/<brand_id>')
@permission_required(NewsItemPermission.list)
@templated
def index_for_brand(brand_id):
    """List news items for that brand."""
    brand = get_brand_or_404(brand_id)
    items = Item.query.for_brand(brand).with_current_version().all()
    return {
        'brand': brand,
        'items': items,
    }

@blueprint.route('/for_brand/<brand_id>/create')
@permission_required(NewsItemPermission.create)
@templated
def create_form(brand_id, *, erroneous_form=None):
    """Show form to create a news item."""
    brand = get_brand_or_404(brand_id)

    form = erroneous_form if erroneous_form else ItemCreateForm()

    return {
        'brand': brand,
        'form': form,
    }

@blueprint.route('/for_brand/<brand_id>', methods=['POST'])
@permission_required(NewsItemPermission.create)
def create(brand_id):
    """Create a news item."""
    brand = get_brand_or_404(brand_id)

    form = ItemCreateForm(request.form)

    slug = form.slug.data.strip().lower()
    creator = g.current_user
    title = form.title.data.strip()
    body = form.body.data.strip()
    image_url_path = form.image_url_path.data.strip()

    item = service.create_item(brand, slug, creator, title, body,
        image_url_path=image_url_path)

    flash_success('Die News "{}" wurde angelegt.', item.title)
    signals.item_published.send(None, item=item)

    return redirect_to('.index_for_brand', brand_id=brand.id)

def get_brand_or_404(brand_id):
    return Brand.query.get_or_404(brand_id)
<|endoftext|>


<commit_before>f = open("io/data/bigfile2", "rb")
b = f.read(512)
print(len(b))
print(b)
<commit_msg>tests: Fix typo in file_long_read3.py.<commit_after>f = open("io/data/bigfile1", "rb")
b = f.read(512)
print(len(b))
print(b)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import division

from numpy import absolute as abs_, logical_and as and_, logical_or as or_

from openfisca_france.model.base import *  # noqa analysis:ignore

class ass_precondition_remplie(Variable):
    column = BoolCol
    entity = Individu
    label = u"Éligible à l'ASS"
    definition_period = MONTH

class ass(Variable):
    column = FloatCol
    label = u"Montant de l'ASS pour une famille"
    entity = Famille
    definition_period = MONTH
    set_input = set_input_divide_by_period

    def formula(self, simulation, period):
        ass_base_ressources = simulation.calculate('ass_base_ressources', period)
        ass_eligibilite_i_holder = simulation.compute('ass_eligibilite_individu', period)
        en_couple = simulation.calculate('en_couple', period)
        ass_params = simulation.legislation_at(period.start).prestations.minima_sociaux.ass

        ass_eligibilite_i = self.split_by_roles(ass_eligibilite_i_holder, roles = [CHEF, PART])

        elig = or_(ass_eligibilite_i[CHEF], ass_eligibilite_i[PART])
        montant_journalier = ass_params.montant_plein
        montant_mensuel = 30 * montant_journalier
        plafond_mensuel = montant_journalier * (ass_params.plaf_seul * not_(en_couple) + ass_params.plaf_coup * en_couple)
        revenus = ass_base_ressources / 12

        ass = min_(montant_mensuel, plafond_mensuel - revenus)
        ass = max_(ass, 0)
        ass = ass * elig
        ass = ass * not_(ass < ass_params.montant_plein)  # pas d'ASS si montant mensuel < montant journalier de base

        return ass

class ass_base_ressources(Variable):
    column = FloatCol
    label = u"Base de ressources de l'ASS"
    entity = Famille
    definition_period = MONTH

    def formula(self, simulation, period):
        ass_base_ressources_i_holder = simulation.compute('ass_base_ressources_individu', period)
        ass_base_ressources_demandeur = self.filter_role(ass_base_ressources_i_holder, role = CHEF)
        ass_base_ressources_conjoint_holder = simulation.compute('ass_base_ressources_conjoint', period)
        ass_base_ressources_conjoint = self.filter_role(ass_base_ressources_conjoint_holder, role = PART)

        result = ass_base_ressources_demandeur + ass_base_ressources_conjoint
        return result

class ass_base_ressources_individu(Variable):
    column = FloatCol
    label = u"Base de ressources individuelle de l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(self, simulation, period):
        # Rolling year
        previous_year = period.start.period('year').offset(-1)
        # N-1
        last_year = period.last_year

        salaire_imposable = simulation.calculate_add('salaire_imposable', previous_year)
        salaire_imposable_this_month = simulation.calculate('salaire_imposable', period)
        salaire_imposable_interrompu = (salaire_imposable > 0) * (salaire_imposable_this_month == 0)
        # Le Salaire d'une activité partielle est neutralisé en cas d'interruption
        salaire_imposable = (1 - salaire_imposable_interrompu) * salaire_imposable
        retraite_nette = simulation.calculate_add('retraite_nette', previous_year)

        def revenus_tns():
            revenus_auto_entrepreneur = simulation.calculate_add('tns_auto_entrepreneur_benefice', previous_year)

            # Les revenus TNS hors AE sont estimés en se basant sur le revenu N-1
            tns_micro_entreprise_benefice = simulation.calculate('tns_micro_entreprise_benefice', last_year)
            tns_benefice_exploitant_agricole = simulation.calculate('tns_benefice_exploitant_agricole', last_year)
            tns_autres_revenus = simulation.calculate('tns_autres_revenus', last_year)

            return revenus_auto_entrepreneur + tns_micro_entreprise_benefice + tns_benefice_exploitant_agricole + tns_autres_revenus

        pensions_alimentaires_percues = simulation.calculate_add('pensions_alimentaires_percues', previous_year)
        pensions_alimentaires_versees_individu = simulation.calculate_add(
            'pensions_alimentaires_versees_individu', previous_year
            )

        aah = simulation.calculate_add('aah', previous_year)
        indemnites_stage = simulation.calculate_add('indemnites_stage', previous_year)
        revenus_stage_formation_pro = simulation.calculate_add('revenus_stage_formation_pro', previous_year)

        return (
            salaire_imposable + retraite_nette + pensions_alimentaires_percues - abs_(pensions_alimentaires_versees_individu) +
            aah + indemnites_stage + revenus_stage_formation_pro + revenus_tns()
        )

class ass_base_ressources_conjoint(Variable):
    column = FloatCol
    label = u"Base de ressources individuelle pour le conjoint du demandeur de l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(self, simulation, period):
        last_month = period.start.period('month').offset(-1)
        # Rolling year
        previous_year = period.start.period('year').offset(-1)
        # N-1
        last_year = period.last_year

        has_ressources_substitution = (
            simulation.calculate('chomage_net', last_month) +
            simulation.calculate('indemnites_journalieres', last_month) +
            simulation.calculate('retraite_nette', last_month)
        ) > 0

        def calculateWithAbatement(ressourceName, neutral_totale = False):
            ressource_year = simulation.calculate_add(ressourceName, previous_year)
            ressource_last_month = simulation.calculate(ressourceName, last_month)

            ressource_interrompue = (ressource_year > 0) * (ressource_last_month == 0)

            # Les ressources interrompues sont abattues différement si elles sont substituées ou non.
            # http://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000020398006&cidTexte=LEGITEXT000006072050

            tx_abat_partiel = simulation.legislation_at(period.start).prestations.minima_sociaux.ass.abat_rev_subst_conj
            tx_abat_total = simulation.legislation_at(period.start).prestations.minima_sociaux.ass.abat_rev_non_subst_conj

            abat_partiel = ressource_interrompue * has_ressources_substitution * (1 - neutral_totale)
            abat_total = ressource_interrompue * (1 - abat_partiel)

            tx_abat_applique = abat_partiel * tx_abat_partiel + abat_total * tx_abat_total

            return (1 - tx_abat_applique) * ressource_year

        salaire_imposable = calculateWithAbatement('salaire_imposable')
        indemnites_stage = calculateWithAbatement('indemnites_stage', neutral_totale = True)
        revenus_stage_formation_pro = calculateWithAbatement('revenus_stage_formation_pro')
        chomage_net = calculateWithAbatement('chomage_net', neutral_totale = True)
        indemnites_journalieres = calculateWithAbatement('indemnites_journalieres')
        aah = calculateWithAbatement('aah')
        retraite_nette = calculateWithAbatement('retraite_nette')
        pensions_alimentaires_percues = calculateWithAbatement('pensions_alimentaires_percues')

        def revenus_tns():
            revenus_auto_entrepreneur = simulation.calculate_add('tns_auto_entrepreneur_benefice', previous_year)

            # Les revenus TNS hors AE sont estimés en se basant sur le revenu N-1
            tns_micro_entreprise_benefice = simulation.calculate('tns_micro_entreprise_benefice', last_year)
            tns_benefice_exploitant_agricole = simulation.calculate('tns_benefice_exploitant_agricole', last_year)
            tns_autres_revenus = simulation.calculate_add('tns_autres_revenus', last_year)

            return revenus_auto_entrepreneur + tns_micro_entreprise_benefice + tns_benefice_exploitant_agricole + tns_autres_revenus

        pensions_alimentaires_versees_individu = simulation.calculate_add('pensions_alimentaires_versees_individu', previous_year)

        result = (
            salaire_imposable + pensions_alimentaires_percues - abs_(pensions_alimentaires_versees_individu) +
            aah + indemnites_stage + revenus_stage_formation_pro + retraite_nette + chomage_net +
            indemnites_journalieres + revenus_tns()
        )

        return result

class ass_eligibilite_individu(Variable):
    column = BoolCol
    label = u"Éligibilité individuelle à l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(self, simulation, period):
        # 1 si demandeur d'emploi
        activite = simulation.calculate('activite', period)

        # Indique que l'user a travaillé 5 ans au cours des 10 dernieres années.
        ass_precondition_remplie = simulation.calculate('ass_precondition_remplie', period)

        are_perceived_this_month = simulation.calculate('chomage_net', period)

        return and_(and_(activite == 1, ass_precondition_remplie), are_perceived_this_month == 0)
<commit_msg>Migrate ass.py to v4 syntax<commit_after># -*- coding: utf-8 -*-

from __future__ import division

from numpy import absolute as abs_, logical_and as and_, logical_or as or_

from openfisca_france.model.base import *  # noqa analysis:ignore

class ass_precondition_remplie(Variable):
    column = BoolCol
    entity = Individu
    label = u"Éligible à l'ASS"
    definition_period = MONTH

class ass(Variable):
    column = FloatCol
    label = u"Montant de l'ASS pour une famille"
    entity = Famille
    definition_period = MONTH
    set_input = set_input_divide_by_period

    def formula(famille, period, legislation):
        ass_base_ressources = famille('ass_base_ressources', period)
        en_couple = famille('en_couple', period)
        ass_params = legislation(period).prestations.minima_sociaux.ass

        ass_eligibilite_i = famille.members('ass_eligibilite_individu', period)
        elig = famille.any(ass_eligibilite_i, role = Famille.PARENT)

        montant_journalier = ass_params.montant_plein
        montant_mensuel = 30 * montant_journalier
        plafond_mensuel = montant_journalier * (ass_params.plaf_seul * not_(en_couple) + ass_params.plaf_coup * en_couple)
        revenus = ass_base_ressources / 12

        ass = min_(montant_mensuel, plafond_mensuel - revenus)
        ass = max_(ass, 0)
        ass = ass * elig
        ass = ass * not_(ass < ass_params.montant_plein)  # pas d'ASS si montant mensuel < montant journalier de base

        return ass

class ass_base_ressources(Variable):
    column = FloatCol
    label = u"Base de ressources de l'ASS"
    entity = Famille
    definition_period = MONTH

    def formula(famille, period):
        ass_base_ressources_demandeur = famille.demandeur('ass_base_ressources_individu', period)
        ass_base_ressources_conjoint = famille.conjoint('ass_base_ressources_conjoint', period)

        result = ass_base_ressources_demandeur + ass_base_ressources_conjoint
        return result

class ass_base_ressources_individu(Variable):
    column = FloatCol
    label = u"Base de ressources individuelle de l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(individu, period):
        # Rolling year
        previous_year = period.start.period('year').offset(-1)
        # N-1
        last_year = period.last_year

        salaire_imposable = individu('salaire_imposable', previous_year, options = [ADD])
        salaire_imposable_this_month = individu('salaire_imposable', period)
        salaire_imposable_interrompu = (salaire_imposable > 0) * (salaire_imposable_this_month == 0)
        # Le Salaire d'une activité partielle est neutralisé en cas d'interruption
        salaire_imposable = (1 - salaire_imposable_interrompu) * salaire_imposable
        retraite_nette = individu('retraite_nette', previous_year, options = [ADD])

        def revenus_tns():
            revenus_auto_entrepreneur = individu('tns_auto_entrepreneur_benefice', previous_year, options = [ADD])

            # Les revenus TNS hors AE sont estimés en se basant sur le revenu N-1
            tns_micro_entreprise_benefice = individu('tns_micro_entreprise_benefice', last_year)
            tns_benefice_exploitant_agricole = individu('tns_benefice_exploitant_agricole', last_year)
            tns_autres_revenus = individu('tns_autres_revenus', last_year)

            return revenus_auto_entrepreneur + tns_micro_entreprise_benefice + tns_benefice_exploitant_agricole + tns_autres_revenus

        pensions_alimentaires_percues = individu('pensions_alimentaires_percues', previous_year, options = [ADD])
        pensions_alimentaires_versees_individu = individu(
            'pensions_alimentaires_versees_individu', previous_year, options = [ADD]
            )

        aah = individu('aah', previous_year, options = [ADD])
        indemnites_stage = individu('indemnites_stage', previous_year, options = [ADD])
        revenus_stage_formation_pro = individu('revenus_stage_formation_pro', previous_year, options = [ADD])

        return (
            salaire_imposable + retraite_nette + pensions_alimentaires_percues - abs_(pensions_alimentaires_versees_individu) +
            aah + indemnites_stage + revenus_stage_formation_pro + revenus_tns()
        )

class ass_base_ressources_conjoint(Variable):
    column = FloatCol
    label = u"Base de ressources individuelle pour le conjoint du demandeur de l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(individu, period, legislation):
        last_month = period.start.period('month').offset(-1)
        # Rolling year
        previous_year = period.start.period('year').offset(-1)
        # N-1
        last_year = period.last_year

        has_ressources_substitution = (
            individu('chomage_net', last_month) +
            individu('indemnites_journalieres', last_month) +
            individu('retraite_nette', last_month)
        ) > 0

        def calculateWithAbatement(ressourceName, neutral_totale = False):
            ressource_year = individu(ressourceName, previous_year, options = [ADD])
            ressource_last_month = individu(ressourceName, last_month)

            ressource_interrompue = (ressource_year > 0) * (ressource_last_month == 0)

            # Les ressources interrompues sont abattues différement si elles sont substituées ou non.
            # http://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000020398006&cidTexte=LEGITEXT000006072050

            tx_abat_partiel = legislation(period).prestations.minima_sociaux.ass.abat_rev_subst_conj
            tx_abat_total = legislation(period).prestations.minima_sociaux.ass.abat_rev_non_subst_conj

            abat_partiel = ressource_interrompue * has_ressources_substitution * (1 - neutral_totale)
            abat_total = ressource_interrompue * (1 - abat_partiel)

            tx_abat_applique = abat_partiel * tx_abat_partiel + abat_total * tx_abat_total

            return (1 - tx_abat_applique) * ressource_year

        salaire_imposable = calculateWithAbatement('salaire_imposable')
        indemnites_stage = calculateWithAbatement('indemnites_stage', neutral_totale = True)
        revenus_stage_formation_pro = calculateWithAbatement('revenus_stage_formation_pro')
        chomage_net = calculateWithAbatement('chomage_net', neutral_totale = True)
        indemnites_journalieres = calculateWithAbatement('indemnites_journalieres')
        aah = calculateWithAbatement('aah')
        retraite_nette = calculateWithAbatement('retraite_nette')
        pensions_alimentaires_percues = calculateWithAbatement('pensions_alimentaires_percues')

        def revenus_tns():
            revenus_auto_entrepreneur = individu('tns_auto_entrepreneur_benefice', previous_year, options = [ADD])

            # Les revenus TNS hors AE sont estimés en se basant sur le revenu N-1
            tns_micro_entreprise_benefice = individu('tns_micro_entreprise_benefice', last_year)
            tns_benefice_exploitant_agricole = individu('tns_benefice_exploitant_agricole', last_year)
            tns_autres_revenus = individu('tns_autres_revenus', last_year, options = [ADD])

            return revenus_auto_entrepreneur + tns_micro_entreprise_benefice + tns_benefice_exploitant_agricole + tns_autres_revenus

        pensions_alimentaires_versees_individu = individu('pensions_alimentaires_versees_individu', previous_year, options = [ADD])

        result = (
            salaire_imposable + pensions_alimentaires_percues - abs_(pensions_alimentaires_versees_individu) +
            aah + indemnites_stage + revenus_stage_formation_pro + retraite_nette + chomage_net +
            indemnites_journalieres + revenus_tns()
        )

        return result

class ass_eligibilite_individu(Variable):
    column = BoolCol
    label = u"Éligibilité individuelle à l'ASS"
    entity = Individu
    definition_period = MONTH

    def formula(individu, period):
        # 1 si demandeur d'emploi
        activite = individu('activite', period)

        # Indique que l'user a travaillé 5 ans au cours des 10 dernieres années.
        ass_precondition_remplie = individu('ass_precondition_remplie', period)

        are_perceived_this_month = individu('chomage_net', period)

        return and_(and_(activite == 1, ass_precondition_remplie), are_perceived_this_month == 0)
<|endoftext|>


<commit_before>import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import requests
import time
import sys
import re
import pylons
import json
from pylons import config

import logging

log = logging.getLogger(__name__)

class DiscoursePlugin(plugins.SingletonPlugin):
    """
    Insert javascript fragments into package pages and the home page to
    allow users to view and create comments on any package.
    """
    plugins.implements(plugins.IConfigurable)
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.ITemplateHelpers)

    def configure(self, config):
        """
        Called upon CKAN setup, will pass current configuration dict
        to the plugin to read custom options.
        """
        discourse_url = config.get('discourse.url', None)
        discourse_username = config.get('discourse.username', None)
        discourse_count_cache_age = config.get('discourse.count_cache_age', 300)
        discourse_ckan_category = config.get('discourse.ckan_category', None)
        discourse_topic_suffix = config.get('discourse.topic_suffix', None)
        site_url = config.get('ckan.site_url', None)
        site_title = config.get('ckan.site_title', None)
        if discourse_url is None:
            log.warn("No discourse forum name is set. Please set \
            'discourse.url' in your .ini!")
        if site_url is None:
            log.warn("Discourse needs ckan.site_url set to work. Please set \
            'ckan.site_url' in your .ini (NOTE: No trailing slash)!")
        if discourse_ckan_category is None:
            log.warn("Discourse needs discourse.ckan_category set to work. Please set \
            'discourse.ckan_category' in your .ini!")
        config['pylons.app_globals'].has_commenting = True
        # store these so available to class methods
        self.__class__.discourse_url = discourse_url.rstrip('/') + '/'
        self.__class__.discourse_username = discourse_username
        self.__class__.discourse_count_cache_age = discourse_count_cache_age
        self.__class__.discourse_ckan_category = discourse_ckan_category
        self.__class__.discourse_topic_suffix = discourse_topic_suffix
        self.__class__.site_url = site_url
        self.__class__.site_title = site_title
        self.__class__.next_sync = time.time() + discourse_count_cache_age
        self.__class__.topic_lookup_dict = dict()
        self.__class__.active_conversations = 0
        self.__class__.discourse_sync()

    # IConfigurer
    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'discourse')

    @classmethod
    def discourse_sync(cls):
        """
        retrieves comment counts from discourse
        caches results for 60 seconds by default
        """

        if time.time() < cls.next_sync and len(cls.topic_lookup_dict.keys()) > 0:
            return cls.active_conversations

        topic_lookup_dict = dict()

        try:
            r = requests.get(cls.discourse_url + cls.discourse_ckan_category, verify=False)
            category_dict = r.json()
            topics_dict = category_dict['topic_list']['topics']
            cls.active_conversations = 0

            for topic in topics_dict:
                topic_title = topic['title'][:-len(cls.discourse_topic_suffix)].strip() if topic['title'].endswith(cls.discourse_topic_suffix) else topic['title']
                if topic['posts_count'] > 1:
                    cls.active_conversations += 1
                    topic_lookup_dict[topic_title] = topic['posts_count']

            while 'more_topics_url' in category_dict['topic_list']:
                more_url = cls.discourse_url + category_dict['topic_list']['more_topics_url']
                more_url = more_url.replace("?category_id", ".json?category_id")
                r = requests.get(more_url, verify=False)
                category_dict = r.json()
                topics_dict = category_dict['topic_list']['topics']

                for topic in topics_dict:
                    topic_title = topic['title'][:-len(cls.discourse_topic_suffix)].strip() if topic['title'].endswith(cls.discourse_topic_suffix) else topic['title']
                    if topic['posts_count'] > 1:
                        cls.active_conversations += 1
                        topic_lookup_dict[topic_title] = topic['posts_count']

            cls.next_sync = time.time() + cls.discourse_count_cache_age
        except:
            # dont try again immediately, next sync attempt at least 60 seconds from now
            cls.next_sync = time.time() + 60

        cls.topic_lookup_dict = topic_lookup_dict
        return cls.active_conversations

    @classmethod
    def discourse_comments(cls, canonical_url = ''):
        ''' Adds Discourse comments to the page.'''
        # we need to create a topic_id
        c = plugins.toolkit.c
        try:
            topic_id = c.controller
            if topic_id == 'package':
                topic_id = 'dataset'

            if c.id:
                topic_id += '/' + c.id
            elif c.current_package_id:
                # we do this so we always tie the same comment even if the url is canonical hash
                # or human-readable version
                pkg_dict = plugins.toolkit.c.__getattr__("pkg_dict")
                topic_id += '/' + pkg_dict["id"]
            else:
                # cannot make a topic_id
                topic_id = ''

            if c.action == 'resource_read':
                topic_id = 'dataset-resource::' + c.resource_id
        except:
            topic_id = ''

        if canonical_url:
            discourse_topic = canonical_url
        else:
            discourse_topic = cls.site_url.rstrip('/') + '/' + topic_id

        # ignore locale settings
        lang_code = pylons.request.environ['CKAN_LANG']

        monolingualURL = re.match('(http://.*?/)('+ lang_code +')/(.*)$', discourse_topic)
        if monolingualURL is not None:
            # we strip language code from URL
            discourse_topic = monolingualURL.group(1) + monolingualURL.group(3)
        else:
            monolingualURL = re.match('(http://.*?/)(..)/(.*)$', discourse_topic)
            if monolingualURL is not None:
                discourse_topic = monolingualURL.group(1) + monolingualURL.group(3)

        data = {'topic_id' : discourse_topic,
            'discourse_url' : cls.discourse_url,
            'discourse_username' : cls.discourse_username }
        return plugins.toolkit.render_snippet('discourse_comments.html', data)

    @classmethod
    def discourse_latest(cls, num_comments=5):
        ''' Adds Discourse latest comments to the page. '''
        discourse_latest_feed = feedparser.parse(cls.discourse_url + 'latest.rss')
        data = {'discourse_latest_feed': discourse_latest_feed,
            'discourse_num_comments' : num_comments}
        return plugins.toolkit.render_snippet('discourse_latest.html', data)

    @classmethod
    def discourse_comments_count(cls, topic_id):
        ''' Adds Discourse comment counts for a given Discourse topic '''
        num_comments = cls.topic_lookup_dict.get(topic_id, 1) - 1
        return num_comments

    def get_helpers(self):
        return {'discourse_comments' : self.discourse_comments,
                'discourse_latest' : self.discourse_latest,
                'discourse_comments_count' : self.discourse_comments_count,
                'discourse_sync' : self.discourse_sync}
<commit_msg>FEATURE: smarter plugin parameter processing Automatically adds .json file extension to Discourse Category JSON endpoint; Checks if endpoint is a valid. Implements #1<commit_after>import ckan.plugins as plugins
import ckan.plugins.toolkit as toolkit
import requests
import time
import sys
import re
import pylons
import json
from pylons import config

import logging

log = logging.getLogger(__name__)

class DiscoursePlugin(plugins.SingletonPlugin):
    """
    Insert javascript fragments into package pages and the home page to
    allow users to view and create comments on any package.
    """
    plugins.implements(plugins.IConfigurable)
    plugins.implements(plugins.IConfigurer)
    plugins.implements(plugins.ITemplateHelpers)

    def configure(self, config):
        """
        Called upon CKAN setup, will pass current configuration dict
        to the plugin to read custom options.
        """
        discourse_url = config.get('discourse.url', None)
        discourse_username = config.get('discourse.username', None)
        discourse_count_cache_age = config.get('discourse.count_cache_age', 300)
        discourse_ckan_category = config.get('discourse.ckan_category', None)
        discourse_topic_suffix = config.get('discourse.topic_suffix', None)
        site_url = config.get('ckan.site_url', None)
        site_title = config.get('ckan.site_title', None)

        if discourse_url is None:
            log.warn("No discourse forum name is set. Please set \
            'discourse.url' in your .ini!")
        else:
            discourse_url = discourse_url.rstrip('/') + '/'

        if site_url is None:
            log.warn("Discourse needs ckan.site_url set to work. Please set \
            'ckan.site_url' in your .ini (NOTE: No trailing slash)!")
        if discourse_ckan_category is None:
            log.warn("Discourse needs discourse.ckan_category set to work. Please set \
            'discourse.ckan_category' in your .ini!")
        elif not discourse_ckan_category.lower().endswith('.json'):
            discourse_ckan_category += '.json'

        # check for valid JSON
        try:
            discourse_api = discourse_url + discourse_ckan_category
            r = requests.get(discourse_api, verify=False)
            test_category_dict = r.json()
        except:
            log.warn(discouse_api + " is not a valid Discourse JSON endpoint!")

        config['pylons.app_globals'].has_commenting = True

        # store these so available to class methods
        self.__class__.discourse_url = discourse_url
        self.__class__.discourse_username = discourse_username
        self.__class__.discourse_count_cache_age = discourse_count_cache_age
        self.__class__.discourse_ckan_category = discourse_ckan_category
        self.__class__.discourse_topic_suffix = discourse_topic_suffix
        self.__class__.site_url = site_url
        self.__class__.site_title = site_title
        self.__class__.next_sync = time.time() + discourse_count_cache_age
        self.__class__.topic_lookup_dict = dict()
        self.__class__.active_conversations = 0
        self.__class__.discourse_sync()

    # IConfigurer
    def update_config(self, config_):
        toolkit.add_template_directory(config_, 'templates')
        toolkit.add_public_directory(config_, 'public')
        toolkit.add_resource('fanstatic', 'discourse')

    @classmethod
    def discourse_sync(cls):
        """
        retrieves comment counts from discourse
        caches results for 60 seconds by default
        """

        if time.time() < cls.next_sync and len(cls.topic_lookup_dict.keys()) > 0:
            return cls.active_conversations

        topic_lookup_dict = dict()

        try:
            r = requests.get(cls.discourse_url + cls.discourse_ckan_category, verify=False)
            category_dict = r.json()
            topics_dict = category_dict['topic_list']['topics']
            cls.active_conversations = 0

            for topic in topics_dict:
                topic_title = topic['title'][:-len(cls.discourse_topic_suffix)].strip() if topic['title'].endswith(cls.discourse_topic_suffix) else topic['title']
                if topic['posts_count'] > 1:
                    cls.active_conversations += 1
                    topic_lookup_dict[topic_title] = topic['posts_count']

            while 'more_topics_url' in category_dict['topic_list']:
                more_url = cls.discourse_url + category_dict['topic_list']['more_topics_url']
                more_url = more_url.replace("?category_id", ".json?category_id")
                r = requests.get(more_url, verify=False)
                category_dict = r.json()
                topics_dict = category_dict['topic_list']['topics']

                for topic in topics_dict:
                    topic_title = topic['title'][:-len(cls.discourse_topic_suffix)].strip() if topic['title'].endswith(cls.discourse_topic_suffix) else topic['title']
                    if topic['posts_count'] > 1:
                        cls.active_conversations += 1
                        topic_lookup_dict[topic_title] = topic['posts_count']

            cls.next_sync = time.time() + cls.discourse_count_cache_age
        except:
            # dont try again immediately, next sync attempt at least 60 seconds from now
            cls.next_sync = time.time() + 60

        cls.topic_lookup_dict = topic_lookup_dict
        return cls.active_conversations

    @classmethod
    def discourse_comments(cls, canonical_url = ''):
        ''' Adds Discourse comments to the page.'''
        # we need to create a topic_id
        c = plugins.toolkit.c
        try:
            topic_id = c.controller
            if topic_id == 'package':
                topic_id = 'dataset'

            if c.id:
                topic_id += '/' + c.id
            elif c.current_package_id:
                # we do this so we always tie the same comment even if the url is canonical hash
                # or human-readable version
                pkg_dict = plugins.toolkit.c.__getattr__("pkg_dict")
                topic_id += '/' + pkg_dict["id"]
            else:
                # cannot make a topic_id
                topic_id = ''

            if c.action == 'resource_read':
                topic_id = 'dataset-resource::' + c.resource_id
        except:
            topic_id = ''

        if canonical_url:
            discourse_topic = canonical_url
        else:
            discourse_topic = cls.site_url.rstrip('/') + '/' + topic_id

        # ignore locale settings
        lang_code = pylons.request.environ['CKAN_LANG']

        monolingualURL = re.match('(http://.*?/)('+ lang_code +')/(.*)$', discourse_topic)
        if monolingualURL is not None:
            # we strip language code from URL
            discourse_topic = monolingualURL.group(1) + monolingualURL.group(3)
        else:
            monolingualURL = re.match('(http://.*?/)(..)/(.*)$', discourse_topic)
            if monolingualURL is not None:
                discourse_topic = monolingualURL.group(1) + monolingualURL.group(3)

        data = {'topic_id' : discourse_topic,
            'discourse_url' : cls.discourse_url,
            'discourse_username' : cls.discourse_username }
        return plugins.toolkit.render_snippet('discourse_comments.html', data)

    @classmethod
    def discourse_latest(cls, num_comments=5):
        ''' Adds Discourse latest comments to the page. '''
        discourse_latest_feed = feedparser.parse(cls.discourse_url + 'latest.rss')
        data = {'discourse_latest_feed': discourse_latest_feed,
            'discourse_num_comments' : num_comments}
        return plugins.toolkit.render_snippet('discourse_latest.html', data)

    @classmethod
    def discourse_comments_count(cls, topic_id):
        ''' Adds Discourse comment counts for a given Discourse topic '''
        num_comments = cls.topic_lookup_dict.get(topic_id, 1) - 1
        return num_comments

    def get_helpers(self):
        return {'discourse_comments' : self.discourse_comments,
                'discourse_latest' : self.discourse_latest,
                'discourse_comments_count' : self.discourse_comments_count,
                'discourse_sync' : self.discourse_sync}
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import traceback

from ckan.lib.helpers import json
from ckanext.harvest.model import HarvestObject, HarvestObjectExtra
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.geocat.utils import search_utils, csw_processor, ogdch_map_utils, csw_mapping  # noqa
from ckan.logic.schema import default_update_package_schema,\
    default_create_package_schema
from ckan.lib.navl.validators import ignore
import ckan.plugins.toolkit as tk
from ckan import model
from ckan.model import Session
import uuid

import logging
log = logging.getLogger(__name__)

DEFAULT_PERMA_LINK_URL = 'https://www.geocat.ch/geonetwork/srv/ger/md.viewer#/full_view/'  # noqa
DEFAULT_PERMA_LINK_LABEL = 'geocat.ch Permalink'
HARVEST_USER = 'harvest'

class GeocatHarvester(HarvesterBase):
    '''
    The harvester for geocat
    '''
    def info(self):
        return {
            'name': 'geocat_harvester',
            'title': 'Geocat harvester',
            'description': (
                'Harvests metadata from geocat (CSW)'
            ),
            'form_config_interface': 'Text'
        }

    def validate_config(self, config):
        if not config:
            return config
        try:
            config_obj = json.loads(config)
            if 'delete_missing_datasets' in config_obj:
                if not isinstance(config_obj['delete_missing_datasets'], bool):
                    raise ValueError('delete_missing_dataset must be boolean')
        except Exception as e:
            raise e
        return config

    def _set_config(self, config_str, harvest_source_id):
        if config_str:
            self.config = json.loads(config_str)
        else:
            self.config = {}

        self.config['user'] = self.config.get('user', HARVEST_USER)
        self.config['delete_missing_datasets'] = \
            self.config.get('delete_missing_datasets', False)

        self.config['geocat_perma_link_label'] = \
            tk.config.get('ckanext.geocat.permalink_title',
                          DEFAULT_PERMA_LINK_LABEL)
        self.config['geocat_perma_link_url'] = \
            self.config.get('geocat_perma_link_url',
                            tk.config.get('geocat_perma_link_url',
                                          DEFAULT_PERMA_LINK_URL))

        self.config['legal_basis_url'] = \
            self.config.get('legal_basis_url', None)

        organization_slug = \
            search_utils.get_organization_slug_for_harvest_source(
                harvest_source_id)
        self.config['organization'] = organization_slug

        log.debug('Using config: %r' % self.config)

    def gather_stage(self, harvest_job):
        log.debug('In GeocatHarvester gather_stage')
        self._set_config(harvest_job.source.config, harvest_job.source.id)

        csw_url = harvest_job.source.url
        harvest_obj_ids = []

        try:
            csw_data = csw_processor.GeocatCatalogueServiceWeb(url=csw_url)
            gathered_geocat_identifiers = csw_data.get_geocat_id_from_csw()
        except Exception as e:
            self._save_gather_error(
                'Unable to get content for URL: %s: %s / %s'
                % (csw_url, str(e), traceback.format_exc()),
                harvest_job
            )
            return []

        existing_dataset_infos = \
            search_utils.get_dataset_infos_for_organization(
                organization_name=self.config['organization'],
                harvest_source_id=harvest_job.source_id,
            )

        gathered_ogdch_identifiers = \
            [ogdch_map_utils.map_geocat_to_ogdch_identifier(
                geocat_identifier=geocat_identifier,
                organization_slug=self.config['organization'])
             for geocat_identifier in gathered_geocat_identifiers]

        all_ogdch_identifiers = \
            set(gathered_ogdch_identifiers + existing_dataset_infos.keys())

        packages_to_delete = search_utils.get_packages_to_delete(
            existing_dataset_infos=existing_dataset_infos,
            gathered_ogdch_identifiers=gathered_ogdch_identifiers,
        )
        packages_that_cannot_be_created = search_utils.get_double_packages(
            existing_dataset_infos=existing_dataset_infos,
            gathered_ogdch_identifiers=gathered_ogdch_identifiers,
        )
        for identifier, info in packages_that_cannot_be_created:
            gathered_ogdch_identifiers.remove(identifier)
            self._save_gather_error(
                'Unable to create package: %s since a package with this identifier already exists'  # noqa
                % (identifier),
                harvest_job
            )

        csw_map = csw_mapping.GeoMetadataMapping(
            organization_slug=self.config['organization'],
            geocat_perma_link=self.config['geocat_perma_link_url'],
            geocat_perma_label=self.config['geocat_perma_link_label'],
            legal_basis_url=self.config['legal_basis_url'],
            valid_identifiers=all_ogdch_identifiers,
        )

        for geocat_id in gathered_geocat_identifiers:

            ogdch_identifier = ogdch_map_utils.map_geocat_to_ogdch_identifier(
                geocat_identifier=geocat_id,
                organization_slug=self.config['organization'])
            csw_record_as_string = csw_data.get_record_by_id(geocat_id)
            dataset_dict = csw_map.get_metadata(csw_record_as_string, geocat_id)  # noqa
            try:
                harvest_obj = HarvestObject(guid=ogdch_identifier,
                                            job=harvest_job,
                                            content=json.dumps(dataset_dict))
                harvest_obj.save()
                harvest_obj_ids.append(harvest_obj.id)
            except Exception as e:
                self._save_gather_error(
                    'Error when processsing dataset: %s %r / %s'
                    % (ogdch_identifier, e, traceback.format_exc()), harvest_job)  # noqa
                return []

        log.debug('IDs: %r' % harvest_obj_ids)

        if self.config['delete_missing_datasets']:
            for package_info in packages_to_delete:
                obj = HarvestObject(
                    guid=package_info.name,
                    job=harvest_job,
                    extras=[HarvestObjectExtra(key='import_action',
                                               value='delete')])
                obj.save()
                harvest_obj_ids.append(obj.id)

        return harvest_obj_ids

    def fetch_stage(self, harvest_object):
        return True

    def import_stage(self, harvest_object):  # noqa
        log.debug('In GeocatHarvester import_stage')

        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False

        import_action = \
            search_utils.get_value_from_object_extra(harvest_object.extras,
                                                     'import_action')
        if import_action and import_action == 'delete':
            log.debug('import action: %s' % import_action)
            harvest_object.current = False
            return self._delete_dataset({'id': harvest_object.guid})

        if harvest_object.content is None:
            self._save_object_error('Empty content for object %s' %
                                    harvest_object.id,
                                    harvest_object, 'Import')
            return False

        try:
            pkg_dict = json.loads(harvest_object.content)
        except ValueError:
            self._save_object_error('Could not parse content for object {0}'
                                    .format(harvest_object.id), harvest_object, 'Import')  # noqa
            return False

        pkg_info = \
            search_utils.find_package_for_identifier(harvest_object.guid)
        context = search_utils.get_default_context()
        try:
            if pkg_info:
                # Change default schema to ignore lists of dicts, which
                # are stored in the '__junk' field
                schema = default_update_package_schema()
                context['schema'] = schema
                schema['__junk'] = [ignore]
                pkg_dict['name'] = pkg_info.name
                pkg_dict['id'] = pkg_info.package_id
                updated_pkg = \
                    tk.get_action('package_update')(context, pkg_dict)
                harvest_object.current = True
                harvest_object.package_id = updated_pkg['id']
                harvest_object.save()
                log.debug("Updated PKG: %s" % updated_pkg)
            else:
                flat_title = _derive_flat_title(pkg_dict['title'])
                if not flat_title:
                    self._save_object_error(
                        'Unable to derive name from title %s'
                        % pkg_dict['title'], harvest_object, 'Import')
                    return False
                pkg_dict['name'] = self._gen_new_name(flat_title)
                schema = default_create_package_schema()
                context['schema'] = schema
                schema['__junk'] = [ignore]
                log.debug("No package found, create a new one!")

                # generate an id to reference it in the harvest_object
                pkg_dict['id'] = unicode(uuid.uuid4())

                log.info('Package with GUID %s does not exist, '
                         'let\'s create it' % harvest_object.guid)

                harvest_object.current = True
                harvest_object.package_id = pkg_dict['id']
                harvest_object.add()

                model.Session.execute(
                    'SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
                model.Session.flush()

                created_pkg = \
                    tk.get_action('package_create')(context, pkg_dict)

                log.debug("Created PKG: %s" % created_pkg)

            Session.commit()
            return True

        except Exception as e:
            self._save_object_error(
                ('Exception in import stage: %r / %s'
                 % (e, traceback.format_exc())), harvest_object)
            return False

    def _create_new_context(self):
        # get the site user
        site_user = tk.get_action('get_site_user')(
            {'model': model, 'ignore_auth': True}, {})
        context = {
            'model': model,
            'session': Session,
            'user': site_user['name'],
        }
        return context

    def _delete_dataset(self, package_dict):
        log.debug('deleting dataset %s' % package_dict['id'])
        context = self._create_new_context()
        tk.get_action('dataset_purge')(
            context.copy(),
            package_dict
        )
        return True

    def _get_geocat_permalink_relation(self, geocat_pkg_id):
        return {'url': self.config['geocat_perma_link_url'] + geocat_pkg_id,
                'label': self.config['geocat_perma_link_label']}

class GeocatConfigError(Exception):
    pass

def _derive_flat_title(title_dict):
    """localizes language dict if no language is specified"""
    return title_dict.get('de') or title_dict.get('fr') or title_dict.get('en') or title_dict.get('it') or ""  # noqa
<commit_msg>check for geocat idetifier whether the corresponding ogdch_identifier was gathered before creating the harvest object<commit_after># -*- coding: utf-8 -*-

import traceback

from ckan.lib.helpers import json
from ckanext.harvest.model import HarvestObject, HarvestObjectExtra
from ckanext.harvest.harvesters import HarvesterBase
from ckanext.geocat.utils import search_utils, csw_processor, ogdch_map_utils, csw_mapping  # noqa
from ckan.logic.schema import default_update_package_schema,\
    default_create_package_schema
from ckan.lib.navl.validators import ignore
import ckan.plugins.toolkit as tk
from ckan import model
from ckan.model import Session
import uuid

import logging
log = logging.getLogger(__name__)

DEFAULT_PERMA_LINK_URL = 'https://www.geocat.ch/geonetwork/srv/ger/md.viewer#/full_view/'  # noqa
DEFAULT_PERMA_LINK_LABEL = 'geocat.ch Permalink'
HARVEST_USER = 'harvest'

class GeocatHarvester(HarvesterBase):
    '''
    The harvester for geocat
    '''
    def info(self):
        return {
            'name': 'geocat_harvester',
            'title': 'Geocat harvester',
            'description': (
                'Harvests metadata from geocat (CSW)'
            ),
            'form_config_interface': 'Text'
        }

    def validate_config(self, config):
        if not config:
            return config
        try:
            config_obj = json.loads(config)
            if 'delete_missing_datasets' in config_obj:
                if not isinstance(config_obj['delete_missing_datasets'], bool):
                    raise ValueError('delete_missing_dataset must be boolean')
        except Exception as e:
            raise e
        return config

    def _set_config(self, config_str, harvest_source_id):
        if config_str:
            self.config = json.loads(config_str)
        else:
            self.config = {}

        self.config['user'] = self.config.get('user', HARVEST_USER)
        self.config['delete_missing_datasets'] = \
            self.config.get('delete_missing_datasets', False)

        self.config['geocat_perma_link_label'] = \
            tk.config.get('ckanext.geocat.permalink_title',
                          DEFAULT_PERMA_LINK_LABEL)
        self.config['geocat_perma_link_url'] = \
            self.config.get('geocat_perma_link_url',
                            tk.config.get('geocat_perma_link_url',
                                          DEFAULT_PERMA_LINK_URL))

        self.config['legal_basis_url'] = \
            self.config.get('legal_basis_url', None)

        organization_slug = \
            search_utils.get_organization_slug_for_harvest_source(
                harvest_source_id)
        self.config['organization'] = organization_slug

        log.debug('Using config: %r' % self.config)

    def gather_stage(self, harvest_job):
        log.debug('In GeocatHarvester gather_stage')
        self._set_config(harvest_job.source.config, harvest_job.source.id)

        csw_url = harvest_job.source.url
        harvest_obj_ids = []

        try:
            csw_data = csw_processor.GeocatCatalogueServiceWeb(url=csw_url)
            gathered_geocat_identifiers = csw_data.get_geocat_id_from_csw()
        except Exception as e:
            self._save_gather_error(
                'Unable to get content for URL: %s: %s / %s'
                % (csw_url, str(e), traceback.format_exc()),
                harvest_job
            )
            return []

        existing_dataset_infos = \
            search_utils.get_dataset_infos_for_organization(
                organization_name=self.config['organization'],
                harvest_source_id=harvest_job.source_id,
            )

        gathered_ogdch_identifiers = \
            [ogdch_map_utils.map_geocat_to_ogdch_identifier(
                geocat_identifier=geocat_identifier,
                organization_slug=self.config['organization'])
             for geocat_identifier in gathered_geocat_identifiers]

        all_ogdch_identifiers = \
            set(gathered_ogdch_identifiers + existing_dataset_infos.keys())

        packages_to_delete = search_utils.get_packages_to_delete(
            existing_dataset_infos=existing_dataset_infos,
            gathered_ogdch_identifiers=gathered_ogdch_identifiers,
        )
        packages_that_cannot_be_created = search_utils.get_double_packages(
            existing_dataset_infos=existing_dataset_infos,
            gathered_ogdch_identifiers=gathered_ogdch_identifiers,
        )
        for identifier, info in packages_that_cannot_be_created:
            gathered_ogdch_identifiers.remove(identifier)
            self._save_gather_error(
                'Unable to create package: %s since a package with this identifier already exists'  # noqa
                % (identifier),
                harvest_job
            )

        csw_map = csw_mapping.GeoMetadataMapping(
            organization_slug=self.config['organization'],
            geocat_perma_link=self.config['geocat_perma_link_url'],
            geocat_perma_label=self.config['geocat_perma_link_label'],
            legal_basis_url=self.config['legal_basis_url'],
            valid_identifiers=all_ogdch_identifiers,
        )

        for geocat_id in gathered_geocat_identifiers:

            ogdch_identifier = ogdch_map_utils.map_geocat_to_ogdch_identifier(
                geocat_identifier=geocat_id,
                organization_slug=self.config['organization'])
            if ogdch_identifier in gathered_ogdch_identifiers:
                csw_record_as_string = csw_data.get_record_by_id(geocat_id)
                dataset_dict = csw_map.get_metadata(csw_record_as_string, geocat_id)  # noqa
                try:
                    harvest_obj = HarvestObject(guid=ogdch_identifier,
                                                job=harvest_job,
                                                content=json.dumps(dataset_dict))
                    harvest_obj.save()
                    harvest_obj_ids.append(harvest_obj.id)
                except Exception as e:
                    self._save_gather_error(
                        'Error when processsing dataset: %s %r / %s'
                        % (ogdch_identifier, e, traceback.format_exc()), harvest_job)  # noqa
                    return []

        log.debug('IDs: %r' % harvest_obj_ids)

        if self.config['delete_missing_datasets']:
            for package_info in packages_to_delete:
                obj = HarvestObject(
                    guid=package_info.name,
                    job=harvest_job,
                    extras=[HarvestObjectExtra(key='import_action',
                                               value='delete')])
                obj.save()
                harvest_obj_ids.append(obj.id)

        return harvest_obj_ids

    def fetch_stage(self, harvest_object):
        return True

    def import_stage(self, harvest_object):  # noqa
        log.debug('In GeocatHarvester import_stage')

        if not harvest_object:
            log.error('No harvest object received')
            self._save_object_error(
                'No harvest object received',
                harvest_object
            )
            return False

        import_action = \
            search_utils.get_value_from_object_extra(harvest_object.extras,
                                                     'import_action')
        if import_action and import_action == 'delete':
            log.debug('import action: %s' % import_action)
            harvest_object.current = False
            return self._delete_dataset({'id': harvest_object.guid})

        if harvest_object.content is None:
            self._save_object_error('Empty content for object %s' %
                                    harvest_object.id,
                                    harvest_object, 'Import')
            return False

        try:
            pkg_dict = json.loads(harvest_object.content)
        except ValueError:
            self._save_object_error('Could not parse content for object {0}'
                                    .format(harvest_object.id), harvest_object, 'Import')  # noqa
            return False

        pkg_info = \
            search_utils.find_package_for_identifier(harvest_object.guid)
        context = search_utils.get_default_context()
        try:
            if pkg_info:
                # Change default schema to ignore lists of dicts, which
                # are stored in the '__junk' field
                schema = default_update_package_schema()
                context['schema'] = schema
                schema['__junk'] = [ignore]
                pkg_dict['name'] = pkg_info.name
                pkg_dict['id'] = pkg_info.package_id
                updated_pkg = \
                    tk.get_action('package_update')(context, pkg_dict)
                harvest_object.current = True
                harvest_object.package_id = updated_pkg['id']
                harvest_object.save()
                log.debug("Updated PKG: %s" % updated_pkg)
            else:
                flat_title = _derive_flat_title(pkg_dict['title'])
                if not flat_title:
                    self._save_object_error(
                        'Unable to derive name from title %s'
                        % pkg_dict['title'], harvest_object, 'Import')
                    return False
                pkg_dict['name'] = self._gen_new_name(flat_title)
                schema = default_create_package_schema()
                context['schema'] = schema
                schema['__junk'] = [ignore]
                log.debug("No package found, create a new one!")

                # generate an id to reference it in the harvest_object
                pkg_dict['id'] = unicode(uuid.uuid4())

                log.info('Package with GUID %s does not exist, '
                         'let\'s create it' % harvest_object.guid)

                harvest_object.current = True
                harvest_object.package_id = pkg_dict['id']
                harvest_object.add()

                model.Session.execute(
                    'SET CONSTRAINTS harvest_object_package_id_fkey DEFERRED')
                model.Session.flush()

                created_pkg = \
                    tk.get_action('package_create')(context, pkg_dict)

                log.debug("Created PKG: %s" % created_pkg)

            Session.commit()
            return True

        except Exception as e:
            self._save_object_error(
                ('Exception in import stage: %r / %s'
                 % (e, traceback.format_exc())), harvest_object)
            return False

    def _create_new_context(self):
        # get the site user
        site_user = tk.get_action('get_site_user')(
            {'model': model, 'ignore_auth': True}, {})
        context = {
            'model': model,
            'session': Session,
            'user': site_user['name'],
        }
        return context

    def _delete_dataset(self, package_dict):
        log.debug('deleting dataset %s' % package_dict['id'])
        context = self._create_new_context()
        tk.get_action('dataset_purge')(
            context.copy(),
            package_dict
        )
        return True

    def _get_geocat_permalink_relation(self, geocat_pkg_id):
        return {'url': self.config['geocat_perma_link_url'] + geocat_pkg_id,
                'label': self.config['geocat_perma_link_label']}

class GeocatConfigError(Exception):
    pass

def _derive_flat_title(title_dict):
    """localizes language dict if no language is specified"""
    return title_dict.get('de') or title_dict.get('fr') or title_dict.get('en') or title_dict.get('it') or ""  # noqa
<|endoftext|>