<commit_before># -*- coding: utf-8 -*-
'''
Configuration of the alternatives system

Control the alternatives system

.. code-block:: yaml

  {% set my_hadoop_conf = '/opt/hadoop/conf' %}

  {{ my_hadoop_conf }}:
    file.directory

  hadoop-0.20-conf:
    alternatives.install:
      - name: hadoop-0.20-conf
      - link: /etc/hadoop-0.20/conf
      - path: {{ my_hadoop_conf }}
      - priority: 30
      - require:
        - file: {{ my_hadoop_conf }}

  hadoop-0.20-conf:
    alternatives.remove:
      - name: hadoop-0.20-conf
      - path: {{ my_hadoop_conf }}

'''
from __future__ import absolute_import, print_function, unicode_literals

# Define a function alias in order not to shadow built-in's
__func_alias__ = {
    'set_': 'set'
}

def __virtual__():
    '''
    Only load if alternatives execution module is available.
    '''
    return True if 'alternatives.auto' in __salt__ else False

def install(name, link, path, priority):
    '''
    Install new alternative for defined <name>

    name
        is the master name for this link group
        (e.g. pager)

    link
        is the symlink pointing to /etc/alternatives/<name>.
        (e.g. /usr/bin/pager)

    path
        is the location of the new alternative target.
        NB: This file / directory must already exist.
        (e.g. /usr/bin/less)

    priority
        is an integer; options with higher numbers have higher priority in
        automatic mode.
    '''
    ret = {'name': name,
           'link': link,
           'path': path,
           'priority': priority,
           'result': True,
           'changes': {},
           'comment': ''}

    isinstalled = __salt__['alternatives.check_installed'](name, path)
    if isinstalled:
        ret['comment'] = 'Alternatives for {0} is already set to {1}'.format(name, path)
    else:
        if __opts__['test']:
            ret['comment'] = (
                'Alternative will be set for {0} to {1} with priority {2}'
            ).format(name, path, priority)
            ret['result'] = None
            return ret

        out = __salt__['alternatives.install'](name, link, path, priority)
        current = __salt__['alternatives.show_current'](name)
        master_link = __salt__['alternatives.show_link'](name)
        if current == path and master_link == link:
            ret['comment'] = (
                'Alternative for {0} set to path {1} with priority {2}'
            ).format(name, current, priority)
            ret['changes'] = {'name': name,
                              'link': link,
                              'path': path,
                              'priority': priority}
        else:
            ret['result'] = False
            ret['comment'] = (
                'Alternative for {0} not installed: {1}'
            ).format(name, out)

    return ret

def remove(name, path):
    '''
    Removes installed alternative for defined <name> and <path>
    or fallback to default alternative, if some defined before.

    name
        is the master name for this link group
        (e.g. pager)

    path
        is the location of one of the alternative target files.
        (e.g. /usr/bin/less)
    '''
    ret = {'name': name,
           'path': path,
           'result': True,
           'changes': {},
           'comment': ''}

    isinstalled = __salt__['alternatives.check_exists'](name, path)
    if isinstalled:
        if __opts__['test']:
            ret['comment'] = ('Alternative for {0} will be removed'
                              .format(name))
            ret['result'] = None
            return ret
        __salt__['alternatives.remove'](name, path)
        current = __salt__['alternatives.show_current'](name)
        if current:
            ret['result'] = True
            ret['comment'] = (
                'Alternative for {0} removed. Falling back to path {1}'
            ).format(name, current)
            ret['changes'] = {'path': current}
            return ret

        ret['comment'] = 'Alternative for {0} removed'.format(name)
        ret['changes'] = {}
        return ret

    current = __salt__['alternatives.show_current'](name)
    if current:
        ret['result'] = True
        ret['comment'] = (
            'Alternative for {0} is set to it\'s default path {1}'
        ).format(name, current)
        return ret

    ret['result'] = False
    ret['comment'] = (
        'Alternative for {0} doesn\'t exist'
    ).format(name)

    return ret

def auto(name):
    '''
    .. versionadded:: 0.17.0

    Instruct alternatives to use the highest priority
    path for <name>

    name
        is the master name for this link group
        (e.g. pager)

    '''
    ret = {'name': name,
           'result': True,
           'comment': '',
           'changes': {}}

    display = __salt__['alternatives.display'](name)
    line = display.splitlines()[0]
    if line.endswith(' auto mode'):
        ret['comment'] = '{0} already in auto mode'.format(name)
        return ret

    if __opts__['test']:
        ret['comment'] = '{0} will be put in auto mode'.format(name)
        ret['result'] = None
        return ret
    ret['changes']['result'] = __salt__['alternatives.auto'](name)
    return ret

def set_(name, path):
    '''
    .. versionadded:: 0.17.0

    Sets alternative for <name> to <path>, if <path> is defined
    as an alternative for <name>.

    name
        is the master name for this link group
        (e.g. pager)

    path
        is the location of one of the alternative target files.
        (e.g. /usr/bin/less)
    '''
    ret = {'name': name,
           'path': path,
           'result': True,
           'changes': {},
           'comment': ''}

    current = __salt__['alternatives.show_current'](name)
    if current == path:
        ret['comment'] = 'Alternative for {0} already set to {1}'.format(name, path)
        return ret

    display = __salt__['alternatives.display'](name)
    isinstalled = False
    for line in display.splitlines():
        if line.startswith(path):
            isinstalled = True
            break

    if isinstalled:
        if __opts__['test']:
            ret['comment'] = (
                'Alternative for {0} will be set to path {1}'
            ).format(name, path)
            ret['result'] = None
            return ret
        __salt__['alternatives.set'](name, path)
        current = __salt__['alternatives.show_current'](name)
        if current == path:
            ret['comment'] = (
                'Alternative for {0} set to path {1}'
            ).format(name, current)
            ret['changes'] = {'path': current}
        else:
            ret['comment'] = 'Alternative for {0} not updated'.format(name)

        return ret

    else:
        ret['result'] = False
        ret['comment'] = (
            'Alternative {0} for {1} doesn\'t exist'
            ).format(path, name)

    return ret
<commit_msg>fix states.alternatives.install to really install alternative<commit_after># -*- coding: utf-8 -*-
'''
Configuration of the alternatives system

Control the alternatives system

.. code-block:: yaml

  {% set my_hadoop_conf = '/opt/hadoop/conf' %}

  {{ my_hadoop_conf }}:
    file.directory

  hadoop-0.20-conf:
    alternatives.install:
      - name: hadoop-0.20-conf
      - link: /etc/hadoop-0.20/conf
      - path: {{ my_hadoop_conf }}
      - priority: 30
      - require:
        - file: {{ my_hadoop_conf }}

  hadoop-0.20-conf:
    alternatives.remove:
      - name: hadoop-0.20-conf
      - path: {{ my_hadoop_conf }}

'''
from __future__ import absolute_import, print_function, unicode_literals

# Define a function alias in order not to shadow built-in's
__func_alias__ = {
    'set_': 'set'
}

def __virtual__():
    '''
    Only load if alternatives execution module is available.
    '''
    return True if 'alternatives.auto' in __salt__ else False

def install(name, link, path, priority):
    '''
    Install new alternative for defined <name>

    name
        is the master name for this link group
        (e.g. pager)

    link
        is the symlink pointing to /etc/alternatives/<name>.
        (e.g. /usr/bin/pager)

    path
        is the location of the new alternative target.
        NB: This file / directory must already exist.
        (e.g. /usr/bin/less)

    priority
        is an integer; options with higher numbers have higher priority in
        automatic mode.
    '''
    ret = {'name': name,
           'link': link,
           'path': path,
           'priority': priority,
           'result': True,
           'changes': {},
           'comment': ''}

    if __salt__['alternatives.check_exists'](name, path):
        ret['comment'] = 'Alternative {0} for {1} is already registered'.format(path, name)
    else:
        if __opts__['test']:
            ret['comment'] = (
                'Alternative will be set for {0} to {1} with priority {2}'
            ).format(name, path, priority)
            ret['result'] = None
            return ret

        out = __salt__['alternatives.install'](name, link, path, priority)
        if __salt__['alternatives.check_exists'](name, path):
            if __salt__['alternatives.check_installed'](name, path):
                ret['comment'] = (
                    'Alternative for {0} set to path {1} with priority {2}'
                ).format(name, current, priority)
            else:
                ret['comment'] = (
                    'Alternative {0} for {1} registered  with priority {2} and not set to default'
                ).format(path, name, priority)
            ret['changes'] = {'name': name,
                              'link': link,
                              'path': path,
                              'priority': priority}
        else:
            ret['result'] = False
            ret['comment'] = (
                'Alternative for {0} not installed: {1}'
            ).format(name, out)

    return ret

def remove(name, path):
    '''
    Removes installed alternative for defined <name> and <path>
    or fallback to default alternative, if some defined before.

    name
        is the master name for this link group
        (e.g. pager)

    path
        is the location of one of the alternative target files.
        (e.g. /usr/bin/less)
    '''
    ret = {'name': name,
           'path': path,
           'result': True,
           'changes': {},
           'comment': ''}

    isinstalled = __salt__['alternatives.check_exists'](name, path)
    if isinstalled:
        if __opts__['test']:
            ret['comment'] = ('Alternative for {0} will be removed'
                              .format(name))
            ret['result'] = None
            return ret
        __salt__['alternatives.remove'](name, path)
        current = __salt__['alternatives.show_current'](name)
        if current:
            ret['result'] = True
            ret['comment'] = (
                'Alternative for {0} removed. Falling back to path {1}'
            ).format(name, current)
            ret['changes'] = {'path': current}
            return ret

        ret['comment'] = 'Alternative for {0} removed'.format(name)
        ret['changes'] = {}
        return ret

    current = __salt__['alternatives.show_current'](name)
    if current:
        ret['result'] = True
        ret['comment'] = (
            'Alternative for {0} is set to it\'s default path {1}'
        ).format(name, current)
        return ret

    ret['result'] = False
    ret['comment'] = (
        'Alternative for {0} doesn\'t exist'
    ).format(name)

    return ret

def auto(name):
    '''
    .. versionadded:: 0.17.0

    Instruct alternatives to use the highest priority
    path for <name>

    name
        is the master name for this link group
        (e.g. pager)

    '''
    ret = {'name': name,
           'result': True,
           'comment': '',
           'changes': {}}

    display = __salt__['alternatives.display'](name)
    line = display.splitlines()[0]
    if line.endswith(' auto mode'):
        ret['comment'] = '{0} already in auto mode'.format(name)
        return ret

    if __opts__['test']:
        ret['comment'] = '{0} will be put in auto mode'.format(name)
        ret['result'] = None
        return ret
    ret['changes']['result'] = __salt__['alternatives.auto'](name)
    return ret

def set_(name, path):
    '''
    .. versionadded:: 0.17.0

    Sets alternative for <name> to <path>, if <path> is defined
    as an alternative for <name>.

    name
        is the master name for this link group
        (e.g. pager)

    path
        is the location of one of the alternative target files.
        (e.g. /usr/bin/less)
    '''
    ret = {'name': name,
           'path': path,
           'result': True,
           'changes': {},
           'comment': ''}

    current = __salt__['alternatives.show_current'](name)
    if current == path:
        ret['comment'] = 'Alternative for {0} already set to {1}'.format(name, path)
        return ret

    display = __salt__['alternatives.display'](name)
    isinstalled = False
    for line in display.splitlines():
        if line.startswith(path):
            isinstalled = True
            break

    if isinstalled:
        if __opts__['test']:
            ret['comment'] = (
                'Alternative for {0} will be set to path {1}'
            ).format(name, path)
            ret['result'] = None
            return ret
        __salt__['alternatives.set'](name, path)
        current = __salt__['alternatives.show_current'](name)
        if current == path:
            ret['comment'] = (
                'Alternative for {0} set to path {1}'
            ).format(name, current)
            ret['changes'] = {'path': current}
        else:
            ret['comment'] = 'Alternative for {0} not updated'.format(name)

        return ret

    else:
        ret['result'] = False
        ret['comment'] = (
            'Alternative {0} for {1} doesn\'t exist'
            ).format(path, name)

    return ret
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Manage Rackspace Queues
=======================

.. versionadded:: XXXXXXXX

Create and destroy Rackspace queues. Be aware that this interacts with
Rackspace's services, and so may incur charges.

This module uses ``pyrax``, which can be installed via package, or pip.
This module is greatly inspired by boto_* modules from SaltStack code source.

.. code-block:: yaml

    myqueue:
        pyrax_queues.present:
            - provider: my-pyrax

    myqueue:
        pyrax_queues.absent:
            - provider: my-pyrax
'''

import salt.utils.openstack.pyrax as suop

def __virtual__():
    '''
    Only load if pyrax is available.
    '''
    return suop.HAS_PYRAX

def present(name, provider):
    '''
    Ensure the RackSpace queue exists.

    name
        Name of the Rackspace queue.

    provider
        Salt Cloud Provider

    '''
    ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}

    is_present = __salt__['cloud.action']('queues_exists', provider=provider, name=name)[provider].values()[0]

    if not is_present:
        if __opts__['test']:
            msg = 'Rackspace queue {0} is set to be created.'.format(name)
            ret['comment'] = msg
            ret['result'] = None
            return ret
        created = __salt__['cloud.action']('queues_create', provider=provider, name=name)
        if created:
            queue = __salt__['cloud.action']('queues_show', provider=provider, name=name)
            ret['changes']['old'] = {}
            ret['changes']['new'] = {'queue': queue}
        else:
            ret['result'] = False
            ret['comment'] = 'Failed to create {0} Rackspace queue.'.format(name)
            return ret
    else:
        ret['comment'] = '{0} present.'.format(name)

    return ret

def absent(name, provider):
    '''
    Ensure the named Rackspace queue is deleted.

    name
        Name of the Rackspace queue.

    provider
        Salt Cloud provider
    '''
    ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}

    is_present = __salt__['cloud.action']('queues_exists', provider=provider, name=name)[provider].values()[0]

    if is_present:
        if __opts__['test']:
            ret['comment'] = 'Rackspace queue {0} is set to be removed.'.format(
                name)
            ret['result'] = None
            return ret
        queue = __salt__['cloud.action']('queues_show', provider=provider, name=name)
        deleted = __salt__['cloud.action']('queues_delete', provider=provider, name=name)
        if deleted:
            ret['changes']['old'] = queue
            ret['changes']['new'] = {}
        else:
            ret['result'] = False
            ret['comment'] = 'Failed to delete {0} Rackspace queue.'.format(name)
    else:
        ret['comment'] = '{0} does not exist.'.format(name)

    return ret
<commit_msg>missed one pylint<commit_after># -*- coding: utf-8 -*-
'''
Manage Rackspace Queues
=======================

.. versionadded:: XXXXXXXX

Create and destroy Rackspace queues. Be aware that this interacts with
Rackspace's services, and so may incur charges.

This module uses ``pyrax``, which can be installed via package, or pip.
This module is greatly inspired by boto_* modules from SaltStack code source.

.. code-block:: yaml

    myqueue:
        pyrax_queues.present:
            - provider: my-pyrax

    myqueue:
        pyrax_queues.absent:
            - provider: my-pyrax
'''

import salt.utils.openstack.pyrax as suop

def __virtual__():
    '''
    Only load if pyrax is available.
    '''
    return suop.HAS_PYRAX

def present(name, provider):
    '''
    Ensure the RackSpace queue exists.

    name
        Name of the Rackspace queue.

    provider
        Salt Cloud Provider
    '''
    ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}

    is_present = __salt__['cloud.action']('queues_exists', provider=provider, name=name)[provider].values()[0]

    if not is_present:
        if __opts__['test']:
            msg = 'Rackspace queue {0} is set to be created.'.format(name)
            ret['comment'] = msg
            ret['result'] = None
            return ret
        created = __salt__['cloud.action']('queues_create', provider=provider, name=name)
        if created:
            queue = __salt__['cloud.action']('queues_show', provider=provider, name=name)
            ret['changes']['old'] = {}
            ret['changes']['new'] = {'queue': queue}
        else:
            ret['result'] = False
            ret['comment'] = 'Failed to create {0} Rackspace queue.'.format(name)
            return ret
    else:
        ret['comment'] = '{0} present.'.format(name)

    return ret

def absent(name, provider):
    '''
    Ensure the named Rackspace queue is deleted.

    name
        Name of the Rackspace queue.

    provider
        Salt Cloud provider
    '''
    ret = {'name': name, 'result': True, 'comment': '', 'changes': {}}

    is_present = __salt__['cloud.action']('queues_exists', provider=provider, name=name)[provider].values()[0]

    if is_present:
        if __opts__['test']:
            ret['comment'] = 'Rackspace queue {0} is set to be removed.'.format(
                name)
            ret['result'] = None
            return ret
        queue = __salt__['cloud.action']('queues_show', provider=provider, name=name)
        deleted = __salt__['cloud.action']('queues_delete', provider=provider, name=name)
        if deleted:
            ret['changes']['old'] = queue
            ret['changes']['new'] = {}
        else:
            ret['result'] = False
            ret['comment'] = 'Failed to delete {0} Rackspace queue.'.format(name)
    else:
        ret['comment'] = '{0} does not exist.'.format(name)

    return ret
<|endoftext|>


<commit_before>
from django.db import models
import collections
import logging
from amcat.tools.amcates import ES
from amcat.tools.table import table3

log = logging.getLogger(__name__)

def _get_filter_date(cleaned_data, prop):
    if prop not in cleaned_data: return "*"
    return cleaned_data[prop].isoformat() + "Z"

FILTER_FIELDS = {"mediums" : "mediumid", "article_ids" : "ids", "articlesets" : "sets",
                 "start_date" : "start_date", "end_date": "end_date"}

def _serialize(x):
    if isinstance(x, collections.Iterable):
        return [_serialize(e) for e in x]
    elif isinstance(x, models.Model):
        return x.id
    return x

def filters_from_form(form_data):
    return {FILTER_FIELDS[k] : _serialize(v)
            for (k,v) in form_data.iteritems() if v and k in FILTER_FIELDS}

def getArticles(form):
    fields = ['mediumid', 'date', 'headline']

    sort = form.get('sortColumn', None)

    if 'keywordInContext' in form['columns']:
        raise NotImplementedError()
    if 'hits' in form['columns']:
        raise NotImplementedError()

    query = form['query']
    kargs = {}
    if form['highlight']:
        kargs["highlight" if query else "lead"] = True

    filters = filters_from_form(form)

    log.info("Query: {query!r}, with filters: {filters}".format(**locals()))

    return ES().query(query, filters=filters, fields=fields, sort=sort, **kargs)

def getTable(form):
    table = table3.DictTable(default=0)
    table.rowNamesRequired = True
    dateInterval = form['dateInterval']
    group_by = 'mediumid' if form['xAxis'] == 'medium' else form['xAxis']
    filters = filters_from_form(form)

    yAxis = form['yAxis']
    if yAxis == 'total':
        query = form['query']
        _add_column(table, 'total', query, filters, group_by, dateInterval)
    elif yAxis == 'medium':
        query = form['query']
        media = ES().list_media(query, filters)
        for mediumid in sorted(media):
            filters['mediumid'] = mediumid
            _add_column(table, str(mediumid), query, filters, group_by, dateInterval)
    elif yAxis == 'searchTerm':
        for q in form['queries']:
            _add_column(table, q.label, q.query, filters, group_by, dateInterval)
    else:
        raise Exception('yAxis {yAxis} not recognized'.format(**locals()))

    return table

def _add_column(table, column_name, query, filters, group_by, dateInterval):
    for group, n in ES().aggregate_query(query, filters, group_by, dateInterval):
        table.addValue(str(group), column_name, n)

def get_statistics(form):
    query = form['query']
    filters = filters_from_form(form)
    return ES().statistics(query, filters)

<commit_msg>Automated merge with https://code.google.com/p/amcat<commit_after>#TODO: NOTE: This module replaces the old 'solrlib', but I feel it can be removed
# entirely. The 'getTable' / 'getArticles' can move either to their respective
# webscripts, or to the REST API. The form handling should just go to the form.

from django.db import models
import collections
import logging
from amcat.tools.amcates import ES
from amcat.tools.table import table3

log = logging.getLogger(__name__)

def _get_filter_date(cleaned_data, prop):
    if prop not in cleaned_data: return "*"
    return cleaned_data[prop].isoformat() + "Z"

FILTER_FIELDS = {"mediums" : "mediumid", "article_ids" : "ids", "articlesets" : "sets",
                 "start_date" : "start_date", "end_date": "end_date"}

def _serialize(x):
    if isinstance(x, collections.Iterable):
        return [_serialize(e) for e in x]
    elif isinstance(x, models.Model):
        return x.id
    return x

def filters_from_form(form_data):
    return {FILTER_FIELDS[k] : _serialize(v)
            for (k,v) in form_data.iteritems() if v and k in FILTER_FIELDS}

def getArticles(form):
    fields = ['mediumid', 'date', 'headline']

    sort = form.get('sortColumn', None)

    if 'keywordInContext' in form['columns']:
        raise NotImplementedError()
    if 'hits' in form['columns']:
        raise NotImplementedError()

    query = form['query']
    kargs = {}
    if form['highlight']:
        kargs["highlight" if query else "lead"] = True

    filters = filters_from_form(form)

    log.info("Query: {query!r}, with filters: {filters}".format(**locals()))

    return ES().query(query, filters=filters, fields=fields, sort=sort, **kargs)

def getTable(form):
    table = table3.DictTable(default=0)
    table.rowNamesRequired = True
    dateInterval = form['dateInterval']
    group_by = 'mediumid' if form['xAxis'] == 'medium' else form['xAxis']
    filters = filters_from_form(form)

    yAxis = form['yAxis']
    if yAxis == 'total':
        query = form['query']
        _add_column(table, 'total', query, filters, group_by, dateInterval)
    elif yAxis == 'medium':
        query = form['query']
        media = ES().list_media(query, filters)
        for mediumid in sorted(media):
            filters['mediumid'] = mediumid
            _add_column(table, str(mediumid), query, filters, group_by, dateInterval)
    elif yAxis == 'searchTerm':
        for q in form['queries']:
            _add_column(table, q.label, q.query, filters, group_by, dateInterval)
    else:
        raise Exception('yAxis {yAxis} not recognized'.format(**locals()))

    return table

def _add_column(table, column_name, query, filters, group_by, dateInterval):
    for group, n in ES().aggregate_query(query, filters, group_by, dateInterval):
        table.addValue(str(group), column_name, n)

def get_statistics(form):
    query = form['query']
    filters = filters_from_form(form)
    return ES().statistics(query, filters)

<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Copy articles to a different amcat server using the API

Limitations: does not copy UUID or parent/child relations
"""

import argparse
import requests
import logging
import itertools

from amcatclient import AmcatAPI

SET_ARGS = ["name", "provenance"]
ART_ARGS = ["metastring", "byline", "uuid", "author", "headline", "text",
            "section", "url", "length", "addressee", "externalid",
            "insertdate", "date", "pagenr", "medium", "parent"]

def create_set(src_api, src_project, src_set, trg_api, trg_project):
    s = src_api.get_set(src_project, src_set)
    s = {k: v for (k, v) in s.items() if k in SET_ARGS}

    provenance = "Copied from {src_api.host} project {src_project} set {src_set}".format(**locals())
    if s.get('provenance'):
        s['provenance'] = "\n".join([s['provenance'], provenance])
    else:
        s['provenance'] = provenance

    result = trg.create_set(trg_project, s)
    logging.info("Created set {id}:{name} in project {project}"
                 .format(**result))
    return result["id"]

def copy_articles(src_api, src_project, src_set,
                  trg_api, trg_project, trg_set=None,
                  batch_size=100, from_page=1):

    srcv = src_api.get_version()
    trgv = trg_api.get_version()

    if not (srcv.major == 3 and trgv.major == 3 and srcv.minor in (4,5) and trgv.minor in (4,5)):
        raise Exception("copy_articles only possible between versions 3.4 and 3.5")

    if trg_set is None:
        trg_set = create_set(src_api, src_project, src_set, trg_api, trg_project)
    if srcv.minor == 5:
        kargs = {}
    else:
        kargs = dict(order_by='parent')
    articles = src_api.list_articles(src_project, src_set, page=from_page, page_size=batch_size, text=True, **kargs)
    for i in itertools.count(from_page):
        batch = list(itertools.islice(articles, batch_size))
        if not batch:
            logging.info("Done")
            break
        logging.info("Copying batch {i}: {n} articles"
                     .format(n=len(batch), **locals()))

        def convert(a):
            a = {k: v for (k, v) in a.items() if k in ART_ARGS and v}
            if not a.get('text'): a['text'] = "-"
            # someone decided to rename headline to title in 3.5, so check and rename as needed
            title = a.pop('headline', '-') if srcv.minor == 4 else a.pop("title", '-')
            medium = a.pop('medium', None) if srcv.minor == 4 else a.pop("publisher", None)
            if trgv.minor == 5:
                a['title'] = title
                if medium:
                    a['publisher'] = medium
            if trgv.minor == 4:
                a['headline'] = title
                a['medium'] = medium or "-"
            return a
        batch = [convert(a) for a in batch]

        trg_api.create_articles(trg_project, trg_set, batch)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(epilog=__doc__)
    parser.add_argument("source_url", help='URL of the source '
                        '(e.g. "http://amcat-dev.labs.vu.nl")')
    parser.add_argument("target_url", help='URL of the target '
                        '(e.g. "http://amcat.vu.nl")')
    parser.add_argument("source_project", help='Article set ID in the source',
                        type=int)
    parser.add_argument("source_set", help='Article set ID in the source',
                        type=int)
    parser.add_argument("target_project", help='Project ID in the target',
                        type=int)
    parser.add_argument("--target-set", "-s", help='Article set ID in the '
                        'target (if omitted, a new set will be created',
                        type=int)
    parser.add_argument("--batch-size", "-b", help='Batch size for copying',
                        type=int, default=100)
    parser.add_argument("--from-page", "-p", help='Start from page (batch)',
                        type=int, default=1)

    args = parser.parse_args()

    fmt = '[%(asctime)s %(levelname)s %(name)s] %(message)s'
    logging.basicConfig(format=fmt, level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    src = AmcatAPI(args.source_url)
    trg = AmcatAPI(args.target_url)

    copy_articles(src, args.source_project, args.source_set,
                  trg, args.target_project, args.target_set,
                  args.batch_size, args.from_page)
<commit_msg>add publisher to arguments<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Lesser General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Lesser General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
Copy articles to a different amcat server using the API

Limitations: does not copy UUID or parent/child relations
"""

import argparse
import requests
import logging
import itertools

from amcatclient import AmcatAPI

SET_ARGS = ["name", "provenance"]
ART_ARGS = ["metastring", "byline", "uuid", "author", "headline","title", "text",
            "section", "url", "length", "addressee", "externalid","insertdate",
            "date", "pagenr", "medium","publisher", "parent"]

def create_set(src_api, src_project, src_set, trg_api, trg_project):
    s = src_api.get_set(src_project, src_set)
    s = {k: v for (k, v) in s.items() if k in SET_ARGS}

    provenance = "Copied from {src_api.host} project {src_project} set {src_set}".format(**locals())
    if s.get('provenance'):
        s['provenance'] = "\n".join([s['provenance'], provenance])
    else:
        s['provenance'] = provenance

    result = trg.create_set(trg_project, s)
    logging.info("Created set {id}:{name} in project {project}"
                 .format(**result))
    return result["id"]

def copy_articles(src_api, src_project, src_set,
                  trg_api, trg_project, trg_set=None,
                  batch_size=100, from_page=1):

    srcv = src_api.get_version()
    trgv = trg_api.get_version()

    if not (srcv.major == 3 and trgv.major == 3 and srcv.minor in (4,5) and trgv.minor in (4,5)):
        raise Exception("copy_articles only possible between versions 3.4 and 3.5")

    if trg_set is None:
        trg_set = create_set(src_api, src_project, src_set, trg_api, trg_project)
    if srcv.minor == 5:
        kargs = {}
    else:
        kargs = dict(order_by='parent')
    articles = src_api.list_articles(src_project, src_set, page=from_page, page_size=batch_size, text=True, **kargs)
    for i in itertools.count(from_page):
        batch = list(itertools.islice(articles, batch_size))
        if not batch:
            logging.info("Done")
            break
        logging.info("Copying batch {i}: {n} articles"
                     .format(n=len(batch), **locals()))

        def convert(a):
            if srcv.minor == 5 and 'properties' in a:
                a.update(a.pop('properties'))
            a = {k: v for (k, v) in a.items() if k in ART_ARGS and v}
            if not a.get('text'): a['text'] = "-"
            # someone decided to rename headline to title in 3.5, so check and rename as needed
            title = a.pop('headline', '-') if srcv.minor == 4 else a.pop("title", '-')
            medium = a.pop('medium', None) if srcv.minor == 4 else a.pop("publisher", None)
            if trgv.minor == 5:
                a['title'] = title
                if medium:
                    a['publisher'] = medium
            if trgv.minor == 4:
                a['headline'] = title
                a['medium'] = medium or "-"
            return a
        batch = [convert(a) for a in batch]

        trg_api.create_articles(trg_project, trg_set, batch)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(epilog=__doc__)
    parser.add_argument("source_url", help='URL of the source '
                        '(e.g. "http://amcat-dev.labs.vu.nl")')
    parser.add_argument("target_url", help='URL of the target '
                        '(e.g. "http://amcat.vu.nl")')
    parser.add_argument("source_project", help='Article set ID in the source',
                        type=int)
    parser.add_argument("source_set", help='Article set ID in the source',
                        type=int)
    parser.add_argument("target_project", help='Project ID in the target',
                        type=int)
    parser.add_argument("--target-set", "-s", help='Article set ID in the '
                        'target (if omitted, a new set will be created',
                        type=int)
    parser.add_argument("--batch-size", "-b", help='Batch size for copying',
                        type=int, default=100)
    parser.add_argument("--from-page", "-p", help='Start from page (batch)',
                        type=int, default=1)

    args = parser.parse_args()

    fmt = '[%(asctime)s %(levelname)s %(name)s] %(message)s'
    logging.basicConfig(format=fmt, level=logging.INFO)
    logging.getLogger("requests").setLevel(logging.WARNING)

    src = AmcatAPI(args.source_url)
    trg = AmcatAPI(args.target_url)

    copy_articles(src, args.source_project, args.source_set,
                  trg, args.target_project, args.target_set,
                  args.batch_size, args.from_page)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import climate
import lmj.cubes
import numpy as np
import pandas as pd
import scipy.signal
import theano
import theano.tensor as TT

logging = climate.get_logger('fill')

# this is the set of markers that gets included in our output.
MARKERS = [
    'marker00-r-head-back',
    'marker01-r-head-front',
    'marker02-l-head-front',
    'marker03-l-head-back',
    'marker06-r-collar',
    'marker07-r-shoulder',
    'marker08-r-elbow',
    'marker09-r-wrist',
    'marker12-r-fing-middle',
    'marker13-r-fing-index',
    'marker14-r-mc-outer',
    'marker15-r-mc-inner',
    'marker16-r-thumb-base',
    'marker17-r-thumb-tip',
    'marker18-l-collar',
    'marker19-l-shoulder',
    'marker20-l-elbow',
    'marker21-l-wrist',
    'marker24-l-fing-middle',
    'marker25-l-fing-index',
    'marker26-l-mc-outer',
    'marker27-l-mc-inner',
    'marker28-l-thumb-base',
    'marker29-l-thumb-tip',
    'marker30-abdomen',
    'marker31-sternum',
    'marker32-t3',
    'marker33-t9',
    'marker34-l-ilium',
    'marker35-r-ilium',
    'marker36-r-hip',
    'marker37-r-knee',
    'marker38-r-shin',
    'marker39-r-ankle',
    'marker40-r-heel',
    'marker41-r-mt-outer',
    'marker42-r-mt-inner',
    'marker43-l-hip',
    'marker44-l-knee',
    'marker45-l-shin',
    'marker46-l-ankle',
    'marker47-l-heel',
    'marker48-l-mt-outer',
    'marker49-l-mt-inner',
]

def marker_channel_columns(df):
    '''Return the names of columns that contain marker data.'''
    return [c for c in df.columns if c[:-2] in MARKERS and c[-1] in 'xyz']

def fill(df, tol=1e-4, window=20, gamma=1):
    '''Complete missing marker data using matrix completion.

    This method alters the given `df` in-place.

    Parameters
    ----------
    df : pd.DataFrame
        Data frame for source data.
    tol : float, optional
        Continue the reconstruction process until reconstructed data is within
        this relative tolerance threshold. Defaults to 1e-4.
    window : int, optional
        Model windows of this many consecutive frames. Defaults to 20.
    gamma : float, optional
        Use this parameter to regularize the low-rank entries on the factored
        data matrix. Defaults to 1.
    '''
    cols = marker_channel_columns(df)
    data = df[cols]
    num_frames, num_channels = data.shape
    num_entries = num_frames * num_channels
    filled_ratio = data.count().sum() / num_entries
    logging.info('missing %d of (%d, %d) = %d values (%.1f%% filled)',
                 num_entries - data.count().sum(),
                 num_frames, num_channels, num_entries,
                 100 * filled_ratio)

    filled = data.fillna(0).values
    weights = (~data.isnull()).values.astype(float)

    # here we create windows of consecutive data frames, all stacked together
    # along axis 1. for example, with 10 2-dimensional frames, we can stack them
    # into windows of length 4 as follows:
    #
    #     data   windows
    # 0   A B    A B C D E F G H
    # 1   C D    C D E F G H I J
    # 2   E F    E F G H I J K L
    # 3   G H    G H I J K L M N
    # 4   I J    I J K L M N O P
    # 5   K L    K L M N O P Q R
    # 6   M N    M N O P Q R S T
    # 7   O P
    # 8   Q R
    # 9   S T
    #
    # this is more or less like a convolution with a sliding rectangular window.
    # this stacked data matrix is the one we'll want to fill in the SVT process
    # below.
    w = np.concatenate([weights[i:num_frames-(window-1-i)] for i in range(window)], axis=1).astype(np.int8)
    t = np.concatenate([filled[i:num_frames-(window-1-i)] for i in range(window)], axis=1).astype('f')
    logging.info('processing windowed data %s', t.shape)

    rank = 600
    s = scipy.linalg.svd(t, compute_uv=False)
    cdf = (s / s.sum()).cumsum()
    logging.info('reconstructing with rank %d (%d%%) %s',
                 rank, 100 * cdf[rank],
                 [(f, cdf.searchsorted(f)) for f in (0.5, 0.8, 0.9, 0.99)])

    # optimization parameters
    gamma = TT.cast(gamma, 'float32')
    learning_rate = TT.cast(0.0001, 'float32')
    momentum = TT.cast(0.94, 'float32')
    max_norm = TT.cast(100000, 'float32')
    one = TT.cast(1, 'float32')

    # symbolic variables.
    data = TT.matrix('data')
    mask = TT.bmatrix('mask')

    # model parameters.
    u = theano.shared(np.random.randn(t.shape[0], rank).astype('f'), name='u')
    v = theano.shared(np.random.randn(rank, t.shape[1]).astype('f'), name='v')

    # symbolic computations, including sgd with momentum.
    err = data - TT.dot(u, v)
    sqerr = (mask * err * err).sum()
    sqdat = (mask * data * data).sum()
    loss = sqerr + gamma * ((u * u).sum() + (v * v).sum())
    updates = []
    for param, grad in zip([u, v], TT.grad(loss, [u, v])):
        g = grad * TT.minimum(one, max_norm / TT.sqrt((grad * grad).sum()))
        v = theano.shared(np.zeros_like(param.get_value()), name='v' + param.name)
        vnew = momentum * v - learning_rate * g
        updates.append((v, vnew))
        updates.append((param, param + vnew))
    f = theano.function([data, mask], [sqerr, sqerr / sqdat], updates=updates)

    i = 0
    n = w.sum()
    rerr = tol + 1
    while rerr > tol:
        aerr, rerr = f(t, w)
        if not i % 10:
            logging.info('%d: abs error %f, relative error %f',
                         i, np.sqrt(aerr / n), rerr)
        i += 1
    del t
    del w

    def avg(xs):
        return np.mean(list(xs), axis=0)

    def idx(a, b=None):
        return slice(df.index[a], df.index[b] if b else df.index[a])

    parts = np.split(np.dot(u.get_value(), v.get_value()), window, axis=1)
    w = window - 1
    f = num_frames - 1

    # super confusing bit! above, we created <window> duplicates of our data,
    # each offset by 1 frame, and stacked (along axis 1) into a big ol matrix.
    # here we unpack these duplicates, remove their offsets, take the average of
    # the appropriate values, and put them back in the data frame.
    #
    # this is particularly tricky because the first and last <window> frames
    # only appear a small number of times in the overall matrix. but the
    # indexing shenanigans below seem to do the trick.
    df.loc[idx(w, f - w), cols] = avg(parts[j][w - j:f + 1 - w - j] for j in range(window))
    for i in range(window):
        df.loc[idx(i), cols] = avg(parts[i - j][j] for j in range(i, -1, -1))
        df.loc[idx(f - i), cols] = avg(parts[w - (i - j + 1)][-j] for j in range(i+1, 0, -1))

def lowpass(df, freq=10., order=4):
    '''Filter marker data using a butterworth low-pass filter.

    This method alters the data in `df` in-place.

    Parameters
    ----------
    freq : float, optional
        Use a butterworth filter with this cutoff frequency. Defaults to
        10Hz.
    order : int, optional
        Order of the butterworth filter. Defaults to 4.
    '''
    nyquist = 1 / (2 * pd.Series(df.index).diff().mean())
    assert 0 < freq < nyquist
    b, a = scipy.signal.butter(order, freq / nyquist)
    for c in marker_channel_columns(df):
        df.loc[:, c] = scipy.signal.filtfilt(b, a, df[c])

@climate.annotate(
    root='load data files from this directory tree',
    output='save smoothed data files to this directory tree',
    pattern=('process only trials matching this pattern', 'option'),
    frame_rate=('reindex frames to this rate', 'option', None, float),
    tol=('fill dropouts with this error tolerance', 'option', None, float),
    window=('process windows of T frames', 'option', None, int),
    freq=('lowpass filter at N Hz', 'option', None, float),
)
def main(root, output, pattern='*', frame_rate=100, tol=0.0001, window=20, freq=20):
    for t in lmj.cubes.Experiment(root).trials_matching(pattern):
        t.load()
        t.mask_dropouts()
        t.reindex(frame_rate)
        for c in t.columns:
            if c.startswith('marker') and c[:-2] not in MARKERS:
                del t.df[c]
        try:
            fill(t.df, tol=tol, window=window)
        except Exception as e:
            logging.exception('error filling dropouts!')
        lowpass(t.df, freq)
        t.save(t.root.replace(root, output))

if __name__ == '__main__':
    climate.call(main)
<commit_msg>Make lowpass filtering optional.<commit_after>#!/usr/bin/env python

import climate
import lmj.cubes
import numpy as np
import pandas as pd
import scipy.signal
import theano
import theano.tensor as TT

logging = climate.get_logger('fill')

# this is the set of markers that gets included in our output.
MARKERS = [
    'marker00-r-head-back',
    'marker01-r-head-front',
    'marker02-l-head-front',
    'marker03-l-head-back',
    'marker06-r-collar',
    'marker07-r-shoulder',
    'marker08-r-elbow',
    'marker09-r-wrist',
    'marker12-r-fing-middle',
    'marker13-r-fing-index',
    'marker14-r-mc-outer',
    'marker15-r-mc-inner',
    'marker16-r-thumb-base',
    'marker17-r-thumb-tip',
    'marker18-l-collar',
    'marker19-l-shoulder',
    'marker20-l-elbow',
    'marker21-l-wrist',
    'marker24-l-fing-middle',
    'marker25-l-fing-index',
    'marker26-l-mc-outer',
    'marker27-l-mc-inner',
    'marker28-l-thumb-base',
    'marker29-l-thumb-tip',
    'marker30-abdomen',
    'marker31-sternum',
    'marker32-t3',
    'marker33-t9',
    'marker34-l-ilium',
    'marker35-r-ilium',
    'marker36-r-hip',
    'marker37-r-knee',
    'marker38-r-shin',
    'marker39-r-ankle',
    'marker40-r-heel',
    'marker41-r-mt-outer',
    'marker42-r-mt-inner',
    'marker43-l-hip',
    'marker44-l-knee',
    'marker45-l-shin',
    'marker46-l-ankle',
    'marker47-l-heel',
    'marker48-l-mt-outer',
    'marker49-l-mt-inner',
]

def marker_channel_columns(df):
    '''Return the names of columns that contain marker data.'''
    return [c for c in df.columns if c[:-2] in MARKERS and c[-1] in 'xyz']

def fill(df, tol=1e-4, window=20, gamma=1):
    '''Complete missing marker data using matrix completion.

    This method alters the given `df` in-place.

    Parameters
    ----------
    df : pd.DataFrame
        Data frame for source data.
    tol : float, optional
        Continue the reconstruction process until reconstructed data is within
        this relative tolerance threshold. Defaults to 1e-4.
    window : int, optional
        Model windows of this many consecutive frames. Defaults to 20.
    gamma : float, optional
        Use this parameter to regularize the low-rank entries on the factored
        data matrix. Defaults to 1.
    '''
    cols = marker_channel_columns(df)
    data = df[cols]
    num_frames, num_channels = data.shape
    num_entries = num_frames * num_channels
    filled_ratio = data.count().sum() / num_entries
    logging.info('missing %d of (%d, %d) = %d values (%.1f%% filled)',
                 num_entries - data.count().sum(),
                 num_frames, num_channels, num_entries,
                 100 * filled_ratio)

    filled = data.fillna(0).values
    weights = (~data.isnull()).values.astype(float)

    # here we create windows of consecutive data frames, all stacked together
    # along axis 1. for example, with 10 2-dimensional frames, we can stack them
    # into windows of length 4 as follows:
    #
    #     data   windows
    # 0   A B    A B C D E F G H
    # 1   C D    C D E F G H I J
    # 2   E F    E F G H I J K L
    # 3   G H    G H I J K L M N
    # 4   I J    I J K L M N O P
    # 5   K L    K L M N O P Q R
    # 6   M N    M N O P Q R S T
    # 7   O P
    # 8   Q R
    # 9   S T
    #
    # this is more or less like a convolution with a sliding rectangular window.
    # this stacked data matrix is the one we'll want to fill in the SVT process
    # below.
    w = np.concatenate([weights[i:num_frames-(window-1-i)] for i in range(window)], axis=1).astype(np.int8)
    t = np.concatenate([filled[i:num_frames-(window-1-i)] for i in range(window)], axis=1).astype('f')
    logging.info('processing windowed data %s', t.shape)

    rank = 600
    s = scipy.linalg.svd(t, compute_uv=False)
    cdf = (s / s.sum()).cumsum()
    logging.info('reconstructing with rank %d (%d%%) %s',
                 rank, 100 * cdf[rank],
                 [(f, cdf.searchsorted(f)) for f in (0.5, 0.8, 0.9, 0.99)])

    # optimization parameters
    gamma = TT.cast(gamma, 'float32')
    learning_rate = TT.cast(0.0001, 'float32')
    momentum = TT.cast(0.94, 'float32')
    max_norm = TT.cast(100000, 'float32')
    one = TT.cast(1, 'float32')

    # symbolic variables.
    data = TT.matrix('data')
    mask = TT.bmatrix('mask')

    # model parameters.
    u = theano.shared(np.random.randn(t.shape[0], rank).astype('f'), name='u')
    v = theano.shared(np.random.randn(rank, t.shape[1]).astype('f'), name='v')

    # symbolic computations, including sgd with momentum.
    err = data - TT.dot(u, v)
    sqerr = (mask * err * err).sum()
    sqdat = (mask * data * data).sum()
    loss = sqerr + gamma * ((u * u).sum() + (v * v).sum())
    updates = []
    for param, grad in zip([u, v], TT.grad(loss, [u, v])):
        g = grad * TT.minimum(one, max_norm / TT.sqrt((grad * grad).sum()))
        v = theano.shared(np.zeros_like(param.get_value()), name='v' + param.name)
        vnew = momentum * v - learning_rate * g
        updates.append((v, vnew))
        updates.append((param, param + vnew))
    f = theano.function([data, mask], [sqerr, sqerr / sqdat], updates=updates)

    i = 0
    n = w.sum()
    rerr = tol + 1
    while rerr > tol:
        aerr, rerr = f(t, w)
        if not i % 10:
            logging.info('%d: abs error %f, relative error %f',
                         i, np.sqrt(aerr / n), rerr)
        i += 1
    del t
    del w

    def avg(xs):
        return np.mean(list(xs), axis=0)

    def idx(a, b=None):
        return slice(df.index[a], df.index[b] if b else df.index[a])

    parts = np.split(np.dot(u.get_value(), v.get_value()), window, axis=1)
    w = window - 1
    f = num_frames - 1

    # super confusing bit! above, we created <window> duplicates of our data,
    # each offset by 1 frame, and stacked (along axis 1) into a big ol matrix.
    # here we unpack these duplicates, remove their offsets, take the average of
    # the appropriate values, and put them back in the data frame.
    #
    # this is particularly tricky because the first and last <window> frames
    # only appear a small number of times in the overall matrix. but the
    # indexing shenanigans below seem to do the trick.
    df.loc[idx(w, f - w), cols] = avg(parts[j][w - j:f + 1 - w - j] for j in range(window))
    for i in range(window):
        df.loc[idx(i), cols] = avg(parts[i - j][j] for j in range(i, -1, -1))
        df.loc[idx(f - i), cols] = avg(parts[w - (i - j + 1)][-j] for j in range(i+1, 0, -1))

def lowpass(df, freq=10., order=4):
    '''Filter marker data using a butterworth low-pass filter.

    This method alters the data in `df` in-place.

    Parameters
    ----------
    freq : float, optional
        Use a butterworth filter with this cutoff frequency. Defaults to
        10Hz.
    order : int, optional
        Order of the butterworth filter. Defaults to 4.
    '''
    nyquist = 1 / (2 * pd.Series(df.index).diff().mean())
    assert 0 < freq < nyquist
    b, a = scipy.signal.butter(order, freq / nyquist)
    for c in marker_channel_columns(df):
        df.loc[:, c] = scipy.signal.filtfilt(b, a, df[c])

@climate.annotate(
    root='load data files from this directory tree',
    output='save smoothed data files to this directory tree',
    pattern=('process only trials matching this pattern', 'option'),
    frame_rate=('reindex frames to this rate', 'option', None, float),
    tol=('fill dropouts with this error tolerance', 'option', None, float),
    window=('process windows of T frames', 'option', None, int),
    freq=('lowpass filter at N Hz', 'option', None, float),
)
def main(root, output, pattern='*', frame_rate=100, tol=0.0001, window=20, freq=None):
    for t in lmj.cubes.Experiment(root).trials_matching(pattern):
        t.load()
        t.mask_dropouts()
        t.reindex(frame_rate)
        for c in t.columns:
            if c.startswith('marker') and c[:-2] not in MARKERS:
                del t.df[c]
        try:
            fill(t.df, tol=tol, window=window)
        except Exception as e:
            logging.exception('error filling dropouts!')
        if freq:
            lowpass(t.df, freq)
        t.save(t.root.replace(root, output))

if __name__ == '__main__':
    climate.call(main)
<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import contextlib
import itertools
import os.path
import threading
from typing import Any, Optional, Iterator

import jax.version
from jax.lib import xla_client

from jax._src import traceback_util
from jax._src.traceback_util import path_starts_with
traceback_util.register_exclusion(__file__)

Traceback = Any  # xla_client.Traceback
Frame = Any  # xla_client.Traceback::Frame

_exclude_paths = [os.path.dirname(jax.version.__file__)]

def register_exclusion(path):
  _exclude_paths.append(path)

def user_frames(source_info: Optional[Traceback]) -> Iterator[Frame]:
  """Heuristic that guesses the identity of the user's code in a stack trace."""
  # Guess the user's frame is the innermost frame not in the jax source tree
  return (x for x in (source_info.frames if source_info else [])
          if not any(path_starts_with(x.file_name, p) for p in _exclude_paths))

def user_frame(source_info: Optional[Traceback]) -> Optional[Frame]:
  return next(user_frames(source_info), None)

def summarize(source_info: Optional[Traceback], num_frames=1) -> str:
  frames = itertools.islice(user_frames(source_info), num_frames)
  frame_strs = [f"{frame.file_name}:{frame.line_num} ({frame.function_name})"
                if frame else "unknown" for frame in frames]
  return '\n'.join(reversed(frame_strs))

class _SourceInfoContext(threading.local):
  context: Optional[Traceback]

  def __init__(self):
    self.context = None

_source_info_context = _SourceInfoContext()

def current() -> Optional[Traceback]:
  return _source_info_context.context or xla_client.Traceback.get_traceback()

@contextlib.contextmanager
def user_context(c):
  prev = _source_info_context.context
  _source_info_context.context = c or _source_info_context.context
  try:
    yield
  finally:
    _source_info_context.context = prev
<commit_msg>Revert part of #5319: don't use `traceback_util.path_starts` in `source_info_util.user_frame`.<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import contextlib
import itertools
import os.path
import threading
from typing import Any, Optional, Iterator

import jax.version
from jax.lib import xla_client

from jax._src import traceback_util
traceback_util.register_exclusion(__file__)

Traceback = Any  # xla_client.Traceback
Frame = Any  # xla_client.Traceback::Frame

_exclude_paths = [os.path.dirname(jax.version.__file__)]

def register_exclusion(path):
  _exclude_paths.append(path)

def user_frames(source_info: Optional[Traceback]) -> Iterator[Frame]:
  """Heuristic that guesses the identity of the user's code in a stack trace."""
  # Guess the user's frame is the innermost frame not in the jax source tree
  # We don't use traceback_util.path_starts_with because that incurs filesystem
  # access, which may be slow; we call this function when e.g. adding source
  # provenance annotations to XLA lowerings, so we don't want to incur the cost.
  return (x for x in (source_info.frames if source_info else [])
          if not any(x.file_name.startswith(p) for p in _exclude_paths))

def user_frame(source_info: Optional[Traceback]) -> Optional[Frame]:
  return next(user_frames(source_info), None)

def summarize(source_info: Optional[Traceback], num_frames=1) -> str:
  frames = itertools.islice(user_frames(source_info), num_frames)
  frame_strs = [f"{frame.file_name}:{frame.line_num} ({frame.function_name})"
                if frame else "unknown" for frame in frames]
  return '\n'.join(reversed(frame_strs))

class _SourceInfoContext(threading.local):
  context: Optional[Traceback]

  def __init__(self):
    self.context = None

_source_info_context = _SourceInfoContext()

def current() -> Optional[Traceback]:
  return _source_info_context.context or xla_client.Traceback.get_traceback()

@contextlib.contextmanager
def user_context(c):
  prev = _source_info_context.context
  _source_info_context.context = c or _source_info_context.context
  try:
    yield
  finally:
    _source_info_context.context = prev
<|endoftext|>


<commit_before>import bleach

from rest_framework import serializers as ser
from modularodm import Q
from framework.auth.core import Auth
from framework.exceptions import PermissionsError
from website.files.models import StoredFileNode
from website.project.model import Comment, Node
from rest_framework.exceptions import ValidationError, PermissionDenied
from api.base.exceptions import InvalidModelValueError, Conflict
from api.base.utils import absolute_reverse
from api.base.settings import osf_settings
from api.base.serializers import (JSONAPISerializer,
                                  TargetField,
                                  RelationshipField,
                                  IDField, TypeField, LinksField,
                                  AuthorizedCharField)

class CommentReport(object):
    def __init__(self, user_id, category, text):
        self._id = user_id
        self.category = category
        self.text = text

class CommentSerializer(JSONAPISerializer):

    filterable_fields = frozenset([
        'deleted',
        'date_created',
        'date_modified',
        'page',
        'target'
    ])

    id = IDField(source='_id', read_only=True)
    type = TypeField()
    content = AuthorizedCharField(source='get_content', required=True, max_length=osf_settings.COMMENT_MAXLENGTH)
    page = ser.CharField(read_only=True)

    target = TargetField(link_type='related', meta={'type': 'get_target_type'})
    user = RelationshipField(related_view='users:user-detail', related_view_kwargs={'user_id': '<user._id>'})
    node = RelationshipField(related_view='nodes:node-detail', related_view_kwargs={'node_id': '<node._id>'})
    replies = RelationshipField(self_view='nodes:node-comments', self_view_kwargs={'node_id': '<node._id>'}, filter={'target': '<pk>'})
    reports = RelationshipField(related_view='comments:comment-reports', related_view_kwargs={'comment_id': '<pk>'})

    date_created = ser.DateTimeField(read_only=True)
    date_modified = ser.DateTimeField(read_only=True)
    modified = ser.BooleanField(read_only=True, default=False)
    deleted = ser.BooleanField(read_only=True, source='is_deleted', default=False)
    is_abuse = ser.SerializerMethodField(help_text='Whether the current user reported this comment.')
    has_children = ser.SerializerMethodField(help_text='Whether this comment has any replies.')
    can_edit = ser.SerializerMethodField(help_text='Whether the current user can edit this comment.')

    # LinksField.to_representation adds link to "self"
    links = LinksField({})

    class Meta:
        type_ = 'comments'

    def validate_content(self, value):
        if value is None or not value.strip():
            raise ValidationError('Comment cannot be empty.')
        return value

    def get_is_abuse(self, obj):
        user = self.context['request'].user
        if user.is_anonymous():
            return False
        return user._id in obj.reports and not obj.reports[user._id].get('retracted', True)

    def get_can_edit(self, obj):
        user = self.context['request'].user
        if user.is_anonymous():
            return False
        return obj.user._id == user._id

    def get_has_children(self, obj):
        return Comment.find(Q('target', 'eq', obj)).count() > 0

    def update(self, comment, validated_data):
        assert isinstance(comment, Comment), 'comment must be a Comment'
        auth = Auth(self.context['request'].user)
        if validated_data:
            if validated_data.get('is_deleted', None) is False and comment.is_deleted:
                try:
                    comment.undelete(auth, save=True)
                except PermissionsError:
                    raise PermissionDenied('Not authorized to undelete this comment.')
            elif 'get_content' in validated_data:
                content = validated_data.pop('get_content')
                try:
                    comment.edit(content, auth=auth, save=True)
                except PermissionsError:
                    raise PermissionDenied('Not authorized to edit this comment.')
        return comment

    def get_target_type(self, obj):
        if isinstance(obj, Node):
            return 'nodes'
        elif isinstance(obj, Comment):
            return 'comments'
        elif isinstance(obj, StoredFileNode):
            return 'files'
        else:
            raise InvalidModelValueError(
                source={'pointer': '/data/relationships/target/links/related/meta/type'},
                detail='Invalid comment target type.'
            )

    def sanitize_data(self):
        ret = super(CommentSerializer, self).sanitize_data()
        content = self.validated_data.get('get_content', None)
        if content:
            ret['get_content'] = bleach.clean(content)
        return ret

class CommentCreateSerializer(CommentSerializer):

    target_type = ser.SerializerMethodField(method_name='get_validated_target_type')

    def get_validated_target_type(self, obj):
        target = obj.target
        target_type = self.context['request'].data.get('target_type')
        expected_target_type = self.get_target_type(target)
        if target_type != expected_target_type:
            raise Conflict('Invalid target type. Expected "{0}", got "{1}."'.format(expected_target_type, target_type))
        return target_type

    def get_target(self, node_id, target_id):
        node = Node.load(target_id)
        comment = Comment.load(target_id)
        target_file = StoredFileNode.load(target_id)

        if node:
            if node_id == target_id:
                return node
            else:
                raise ValueError('Cannot post comment to another node.')
        elif comment:
            if comment.node._id == node_id:
                return comment
            else:
                raise ValueError('Cannot post reply to comment on another node.')
        elif target_file:
            if target_file.provider not in osf_settings.ADDONS_COMMENTABLE:
                raise ValueError('Comments are not supported for this file provider.')
            elif target_file.node._id != node_id:
                raise ValueError('Cannot post comment to file on another node.')
            else:
                return target_file
        else:
            raise ValueError('Invalid comment target.')

    def create(self, validated_data):
        user = validated_data['user']
        auth = Auth(user)
        node = validated_data['node']
        target_id = self.context['request'].data.get('id')

        try:
            target = self.get_target(node._id, target_id)
        except ValueError:
            raise InvalidModelValueError(
                source={'pointer': '/data/relationships/target/data/id'},
                detail='Invalid comment target \'{}\'.'.format(target_id)
            )
        validated_data['target'] = target
        validated_data['content'] = validated_data.pop('get_content')
        try:
            comment = Comment.create(auth=auth, **validated_data)
        except PermissionsError:
            raise PermissionDenied('Not authorized to comment on this project.')
        return comment

class CommentDetailSerializer(CommentSerializer):
    """
    Overrides CommentSerializer to make id required.
    """
    id = IDField(source='_id', required=True)
    deleted = ser.BooleanField(source='is_deleted', required=True)

class CommentReportSerializer(JSONAPISerializer):
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    category = ser.ChoiceField(choices=[('spam', 'Spam or advertising'),
                                        ('hate', 'Hate speech'),
                                        ('violence', 'Violence or harmful behavior')], required=True)
    message = ser.CharField(source='text', required=False, allow_blank=True)
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'comment_reports'

    def get_absolute_url(self, obj):
        comment_id = self.context['request'].parser_context['kwargs']['comment_id']
        return absolute_reverse(
            'comments:report-detail',
            kwargs={
                'comment_id': comment_id,
                'user_id': obj._id
            }
        )

    def create(self, validated_data):
        user = self.context['request'].user
        comment = self.context['view'].get_comment()
        if user._id in comment.reports and not comment.reports[user._id].get('retracted', True):
            raise ValidationError('Comment already reported.')
        try:
            comment.report_abuse(user, save=True, **validated_data)
        except ValueError:
            raise ValidationError('You cannot report your own comment.')
        return CommentReport(user._id, **validated_data)

    def update(self, comment_report, validated_data):
        user = self.context['request'].user
        comment = self.context['view'].get_comment()
        if user._id != comment_report._id:
            raise ValidationError('You cannot report a comment on behalf of another user.')
        try:
            comment.report_abuse(user, save=True, **validated_data)
        except ValueError:
            raise ValidationError('You cannot report your own comment.')
        return CommentReport(user._id, **validated_data)

class CommentReportDetailSerializer(CommentReportSerializer):
    """
    Overrides CommentReportSerializer to make id required.
    """
    id = IDField(source='_id', required=True)
<commit_msg>Fix can_edit CommentSerializer field<commit_after>import bleach

from rest_framework import serializers as ser
from modularodm import Q
from framework.auth.core import Auth
from framework.exceptions import PermissionsError
from website.files.models import StoredFileNode
from website.project.model import Comment, Node
from rest_framework.exceptions import ValidationError, PermissionDenied
from api.base.exceptions import InvalidModelValueError, Conflict
from api.base.utils import absolute_reverse
from api.base.settings import osf_settings
from api.base.serializers import (JSONAPISerializer,
                                  TargetField,
                                  RelationshipField,
                                  IDField, TypeField, LinksField,
                                  AuthorizedCharField)

class CommentReport(object):
    def __init__(self, user_id, category, text):
        self._id = user_id
        self.category = category
        self.text = text

class CommentSerializer(JSONAPISerializer):

    filterable_fields = frozenset([
        'deleted',
        'date_created',
        'date_modified',
        'page',
        'target'
    ])

    id = IDField(source='_id', read_only=True)
    type = TypeField()
    content = AuthorizedCharField(source='get_content', required=True, max_length=osf_settings.COMMENT_MAXLENGTH)
    page = ser.CharField(read_only=True)

    target = TargetField(link_type='related', meta={'type': 'get_target_type'})
    user = RelationshipField(related_view='users:user-detail', related_view_kwargs={'user_id': '<user._id>'})
    node = RelationshipField(related_view='nodes:node-detail', related_view_kwargs={'node_id': '<node._id>'})
    replies = RelationshipField(self_view='nodes:node-comments', self_view_kwargs={'node_id': '<node._id>'}, filter={'target': '<pk>'})
    reports = RelationshipField(related_view='comments:comment-reports', related_view_kwargs={'comment_id': '<pk>'})

    date_created = ser.DateTimeField(read_only=True)
    date_modified = ser.DateTimeField(read_only=True)
    modified = ser.BooleanField(read_only=True, default=False)
    deleted = ser.BooleanField(read_only=True, source='is_deleted', default=False)
    is_abuse = ser.SerializerMethodField(help_text='Whether the current user reported this comment.')
    has_children = ser.SerializerMethodField(help_text='Whether this comment has any replies.')
    can_edit = ser.SerializerMethodField(help_text='Whether the current user can edit this comment.')

    # LinksField.to_representation adds link to "self"
    links = LinksField({})

    class Meta:
        type_ = 'comments'

    def validate_content(self, value):
        if value is None or not value.strip():
            raise ValidationError('Comment cannot be empty.')
        return value

    def get_is_abuse(self, obj):
        user = self.context['request'].user
        if user.is_anonymous():
            return False
        return user._id in obj.reports and not obj.reports[user._id].get('retracted', True)

    def get_can_edit(self, obj):
        user = self.context['request'].user
        if user.is_anonymous():
            return False
        return obj.user._id == user._id and obj.node.can_comment(Auth(user))

    def get_has_children(self, obj):
        return Comment.find(Q('target', 'eq', obj)).count() > 0

    def update(self, comment, validated_data):
        assert isinstance(comment, Comment), 'comment must be a Comment'
        auth = Auth(self.context['request'].user)
        if validated_data:
            if validated_data.get('is_deleted', None) is False and comment.is_deleted:
                try:
                    comment.undelete(auth, save=True)
                except PermissionsError:
                    raise PermissionDenied('Not authorized to undelete this comment.')
            elif 'get_content' in validated_data:
                content = validated_data.pop('get_content')
                try:
                    comment.edit(content, auth=auth, save=True)
                except PermissionsError:
                    raise PermissionDenied('Not authorized to edit this comment.')
        return comment

    def get_target_type(self, obj):
        if isinstance(obj, Node):
            return 'nodes'
        elif isinstance(obj, Comment):
            return 'comments'
        elif isinstance(obj, StoredFileNode):
            return 'files'
        else:
            raise InvalidModelValueError(
                source={'pointer': '/data/relationships/target/links/related/meta/type'},
                detail='Invalid comment target type.'
            )

    def sanitize_data(self):
        ret = super(CommentSerializer, self).sanitize_data()
        content = self.validated_data.get('get_content', None)
        if content:
            ret['get_content'] = bleach.clean(content)
        return ret

class CommentCreateSerializer(CommentSerializer):

    target_type = ser.SerializerMethodField(method_name='get_validated_target_type')

    def get_validated_target_type(self, obj):
        target = obj.target
        target_type = self.context['request'].data.get('target_type')
        expected_target_type = self.get_target_type(target)
        if target_type != expected_target_type:
            raise Conflict('Invalid target type. Expected "{0}", got "{1}."'.format(expected_target_type, target_type))
        return target_type

    def get_target(self, node_id, target_id):
        node = Node.load(target_id)
        comment = Comment.load(target_id)
        target_file = StoredFileNode.load(target_id)

        if node:
            if node_id == target_id:
                return node
            else:
                raise ValueError('Cannot post comment to another node.')
        elif comment:
            if comment.node._id == node_id:
                return comment
            else:
                raise ValueError('Cannot post reply to comment on another node.')
        elif target_file:
            if target_file.provider not in osf_settings.ADDONS_COMMENTABLE:
                raise ValueError('Comments are not supported for this file provider.')
            elif target_file.node._id != node_id:
                raise ValueError('Cannot post comment to file on another node.')
            else:
                return target_file
        else:
            raise ValueError('Invalid comment target.')

    def create(self, validated_data):
        user = validated_data['user']
        auth = Auth(user)
        node = validated_data['node']
        target_id = self.context['request'].data.get('id')

        try:
            target = self.get_target(node._id, target_id)
        except ValueError:
            raise InvalidModelValueError(
                source={'pointer': '/data/relationships/target/data/id'},
                detail='Invalid comment target \'{}\'.'.format(target_id)
            )
        validated_data['target'] = target
        validated_data['content'] = validated_data.pop('get_content')
        try:
            comment = Comment.create(auth=auth, **validated_data)
        except PermissionsError:
            raise PermissionDenied('Not authorized to comment on this project.')
        return comment

class CommentDetailSerializer(CommentSerializer):
    """
    Overrides CommentSerializer to make id required.
    """
    id = IDField(source='_id', required=True)
    deleted = ser.BooleanField(source='is_deleted', required=True)

class CommentReportSerializer(JSONAPISerializer):
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    category = ser.ChoiceField(choices=[('spam', 'Spam or advertising'),
                                        ('hate', 'Hate speech'),
                                        ('violence', 'Violence or harmful behavior')], required=True)
    message = ser.CharField(source='text', required=False, allow_blank=True)
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'comment_reports'

    def get_absolute_url(self, obj):
        comment_id = self.context['request'].parser_context['kwargs']['comment_id']
        return absolute_reverse(
            'comments:report-detail',
            kwargs={
                'comment_id': comment_id,
                'user_id': obj._id
            }
        )

    def create(self, validated_data):
        user = self.context['request'].user
        comment = self.context['view'].get_comment()
        if user._id in comment.reports and not comment.reports[user._id].get('retracted', True):
            raise ValidationError('Comment already reported.')
        try:
            comment.report_abuse(user, save=True, **validated_data)
        except ValueError:
            raise ValidationError('You cannot report your own comment.')
        return CommentReport(user._id, **validated_data)

    def update(self, comment_report, validated_data):
        user = self.context['request'].user
        comment = self.context['view'].get_comment()
        if user._id != comment_report._id:
            raise ValidationError('You cannot report a comment on behalf of another user.')
        try:
            comment.report_abuse(user, save=True, **validated_data)
        except ValueError:
            raise ValidationError('You cannot report your own comment.')
        return CommentReport(user._id, **validated_data)

class CommentReportDetailSerializer(CommentReportSerializer):
    """
    Overrides CommentReportSerializer to make id required.
    """
    id = IDField(source='_id', required=True)
<|endoftext|>


<commit_before>from rest_framework import serializers as ser

from api.base.serializers import (
    JSONAPISerializer, LinksField, IDField, TypeField
)
from api.base.utils import absolute_reverse

class LicenseSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'name',
        'id',
    ])
    non_anonymized_fields = ['type']
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    name = ser.CharField(required=True, help_text='License name')
    text = ser.CharField(required=True, help_text='Full text of the license')
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'licenses'

    def get_absolute_url(self, obj):
        return absolute_reverse('licenses:license-detail', kwargs={
            'license_id': obj._id,
            'version': self.context['request'].parser_context['kwargs']['version']
        })
<commit_msg>Add required_properties to LicenseSerializer. [skip ci]<commit_after>from rest_framework import serializers as ser

from api.base.serializers import (
    JSONAPISerializer, LinksField, IDField, TypeField
)
from api.base.utils import absolute_reverse

class LicenseSerializer(JSONAPISerializer):
    filterable_fields = frozenset([
        'name',
        'id',
    ])
    non_anonymized_fields = ['type']
    id = IDField(source='_id', read_only=True)
    type = TypeField()
    name = ser.CharField(required=True, help_text='License name')
    text = ser.CharField(required=True, help_text='Full text of the license')
    required_properties = ser.ListField(source='properties', read_only=True, help_text='Required attributes for the license')
    links = LinksField({'self': 'get_absolute_url'})

    class Meta:
        type_ = 'licenses'

    def get_absolute_url(self, obj):
        return absolute_reverse('licenses:license-detail', kwargs={
            'license_id': obj._id,
            'version': self.context['request'].parser_context['kwargs']['version']
        })
<|endoftext|>


<commit_before># Copyright (C) 2020 Open Source Integrators
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
{
    "name": "Animal Owner",
    "version": "14.0.1.1.1",
    "license": "AGPL-3",
    "summary": "Add owner to the animal",
    "author": "Open Source Integrators, Odoo Community Association (OCA)",
    "maintainer": "Open Source Integrators",
    "website": "https://github.com/OCA/partner-contact",
    "depends": ["animal"],
    "data": ["views/animal.xml", "views/res_partner.xml"],
    "development_status": "Beta",
    "maintainers": ["max3903"],
}
<commit_msg>animal_owner 14.0.1.2.0<commit_after># Copyright (C) 2020 Open Source Integrators
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
{
    "name": "Animal Owner",
    "version": "14.0.1.2.0",
    "license": "AGPL-3",
    "summary": "Add owner to the animal",
    "author": "Open Source Integrators, Odoo Community Association (OCA)",
    "maintainer": "Open Source Integrators",
    "website": "https://github.com/OCA/partner-contact",
    "depends": ["animal"],
    "data": ["views/animal.xml", "views/res_partner.xml"],
    "development_status": "Beta",
    "maintainers": ["max3903"],
}
<|endoftext|>


<commit_before>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
API Viewsets for dealing with NLP (pre)processing via nlpipe
"""

from __future__ import unicode_literals, print_function, absolute_import

from collections import namedtuple
import itertools
import json
import tempfile
import logging

from rest_framework.response import Response
from rest_framework.exceptions import APIException, ValidationError
from rest_framework.decorators import api_view
from rest_framework.status import HTTP_200_OK
from rest_framework import serializers
from rest_framework.mixins import ListModelMixin, CreateModelMixin
from rest_framework.viewsets import ModelViewSet, ViewSet, GenericViewSet

from amcat.models import Article, ArticleSet

from api.rest.viewsets.articleset import ArticleSetViewSetMixin
from api.rest.viewsets.project import ProjectViewSetMixin
from api.rest.viewsets.article import ArticleViewSetMixin
from api.rest.mixins import DatatablesMixin

from nlpipe.pipeline import get_results
from nlpipe.celery import app

from KafNafParserPy import KafNafParser
from io import BytesIO

def _get_tokens(term_vector, pos_inc=0, offset_inc=0):
    for term, info in term_vector.items():
        for token in info['tokens']:
            token['position'] += pos_inc
            token['start_offset'] += offset_inc
            token['end_offset'] += offset_inc
            token['term'] = term
            yield token

def _termvector(aid):
    from amcat.tools import amcates
    import collections
    fields = ["headline", "text"]
    res =  amcates.ES().term_vector(aid, fields=fields)
    pos_inc, offset_inc = 0, 0
    for field in fields:
        tokens = _get_tokens(res['term_vectors'][field]['terms'],
                             pos_inc=pos_inc, offset_inc=offset_inc)
        tokens = sorted(tokens, key = lambda t:t['position'])
        pos_inc = (tokens[-1]['position'] + 1) if tokens else 0
        offset_inc = (tokens[-1]['end_offset'] + 1) if tokens else 0
        for token in tokens:
            token['aid'] = aid
            yield token

def _termvectors(aids):
    return {aid: list(_termvector(aid)) for aid in aids}

class NLPipeLemmataSerializer(serializers.Serializer):

    class Meta:
        class list_serializer_class(serializers.ListSerializer):
            def to_representation(self, data):

                only_cached = self.context['request'].GET.get('only_cached', 'N')
                only_cached = only_cached[0].lower() in ['1', 'y']
                aids = [a.pk for a in data]
                if self.context['request'].GET.get('module') == "elastic":
                    self.child._cache = _termvectors(aids)
                else:
                    self.child._cache = get_results(aids, self.child.module, only_cached=only_cached)
                if only_cached:
                    data = [a for a in data if a.pk in self.child._cache]
                result = serializers.ListSerializer.to_representation(self, data)
                # flatten list of lists
                result = itertools.chain(*result)
                return result

    @property
    def module(self):
        module = self.context['request'].GET.get('module')
        if not module:
            raise ValidationError("Please specify the NLP module to use with a module= GET parameter")
        from nlpipe import tasks
        if not hasattr(tasks, module):
            raise ValidationError("Module {module} not known".format(**locals()))

        return getattr(tasks, module)

    def to_representation(self, article):
        result = self._cache[article.pk]
        if isinstance(result, list):
            return result
        else:
            return self.from_naf(result.input)

    def from_naf(self, naf):
        naf = KafNafParser(BytesIO(naf.encode("utf-8")))
        tokendict = {token.get_id(): token for token in naf.get_tokens()}

        for term in naf.get_terms():
            tokens = [tokendict[id] for id in term.get_span().get_span_ids()]
            for token in tokens:
                yield {"aid": article.pk,
                       "token_id": token.get_id(),
                       "offset": token.get_offset(),
                       "sentence": token.get_sent(),
                       "para": token.get_para(),
                       "word": token.get_text(),
                       "term_id": term.get_id(),
                       "lemma": term.get_lemma(),
                       "pos": term.get_pos()}

class NLPipeLemmataViewSet(ProjectViewSetMixin, ArticleSetViewSetMixin, DatatablesMixin, ModelViewSet):
    model_key = "token"
    model = Article
    queryset = Article.objects.all()
    serializer_class = NLPipeLemmataSerializer

    def filter_queryset(self, queryset):
        queryset = super(NLPipeLemmataViewSet, self).filter_queryset(queryset)
        # only(.) would be better on serializer, but meh
        try:
            queryset = queryset.filter(articlesets_set=self.articleset).only("pk")
        except ArticleSet.DoesNotExist:
            from django.http import Http404
            raise Http404("Articleset does not exist")
        return queryset

    def get_renderer_context(self):
        context = super(NLPipeLemmataViewSet, self).get_renderer_context()
        context['fast_csv'] = True
        return context

ModuleCount = namedtuple("ModuleCount", ["module", "n"])

class PreprocessViewSet(ProjectViewSetMixin, ArticleSetViewSetMixin, DatatablesMixin,
                        ListModelMixin, GenericViewSet):
    model_key = "preproces"
    model = None
    base_name = "preprocess"

    class serializer_class(serializers.Serializer):
        module = serializers.CharField()
        n = serializers.IntegerField()

    def filter_queryset(self, queryset):
        return queryset

    def get_queryset(self):
        ids = list(self.articleset.get_article_ids_from_elastic())
        result = [ModuleCount("Total #articles", len(ids))]
        from nlpipe.document import count_cached
        for module, n in count_cached(ids):
            result.append(ModuleCount(module, n))

        return result

    def get_filter_fields(self):
        return []
<commit_msg>Make tokens module default to elastic<commit_after>###########################################################################
#          (C) Vrije Universiteit, Amsterdam (the Netherlands)            #
#                                                                         #
# This file is part of AmCAT - The Amsterdam Content Analysis Toolkit     #
#                                                                         #
# AmCAT is free software: you can redistribute it and/or modify it under  #
# the terms of the GNU Affero General Public License as published by the  #
# Free Software Foundation, either version 3 of the License, or (at your  #
# option) any later version.                                              #
#                                                                         #
# AmCAT is distributed in the hope that it will be useful, but WITHOUT    #
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or   #
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public     #
# License for more details.                                               #
#                                                                         #
# You should have received a copy of the GNU Affero General Public        #
# License along with AmCAT.  If not, see <http://www.gnu.org/licenses/>.  #
###########################################################################

"""
API Viewsets for dealing with NLP (pre)processing via nlpipe
"""

from __future__ import unicode_literals, print_function, absolute_import

from collections import namedtuple
import itertools
import json
import tempfile
import logging

from rest_framework.response import Response
from rest_framework.exceptions import APIException, ValidationError
from rest_framework.decorators import api_view
from rest_framework.status import HTTP_200_OK
from rest_framework import serializers
from rest_framework.mixins import ListModelMixin, CreateModelMixin
from rest_framework.viewsets import ModelViewSet, ViewSet, GenericViewSet

from amcat.models import Article, ArticleSet

from api.rest.viewsets.articleset import ArticleSetViewSetMixin
from api.rest.viewsets.project import ProjectViewSetMixin
from api.rest.viewsets.article import ArticleViewSetMixin
from api.rest.mixins import DatatablesMixin

from nlpipe.pipeline import get_results
from nlpipe.celery import app

from KafNafParserPy import KafNafParser
from io import BytesIO

def _get_tokens(term_vector, pos_inc=0, offset_inc=0):
    for term, info in term_vector.items():
        for token in info['tokens']:
            token['position'] += pos_inc
            token['start_offset'] += offset_inc
            token['end_offset'] += offset_inc
            token['term'] = term
            yield token

def _termvector(aid):
    from amcat.tools import amcates
    import collections
    fields = ["headline", "text"]
    res =  amcates.ES().term_vector(aid, fields=fields)
    pos_inc, offset_inc = 0, 0
    for field in fields:
        tokens = _get_tokens(res['term_vectors'][field]['terms'],
                             pos_inc=pos_inc, offset_inc=offset_inc)
        tokens = sorted(tokens, key = lambda t:t['position'])
        pos_inc = (tokens[-1]['position'] + 1) if tokens else 0
        offset_inc = (tokens[-1]['end_offset'] + 1) if tokens else 0
        for token in tokens:
            token['aid'] = aid
            yield token

def _termvectors(aids):
    return {aid: list(_termvector(aid)) for aid in aids}

class NLPipeLemmataSerializer(serializers.Serializer):

    class Meta:
        class list_serializer_class(serializers.ListSerializer):
            def to_representation(self, data):

                only_cached = self.context['request'].GET.get('only_cached', 'N')
                only_cached = only_cached[0].lower() in ['1', 'y']
                aids = [a.pk for a in data]
                if self.context['request'].GET.get('module', 'elastic') == "elastic":
                    self.child._cache = _termvectors(aids)
                else:
                    self.child._cache = get_results(aids, self.child.module, only_cached=only_cached)
                if only_cached:
                    data = [a for a in data if a.pk in self.child._cache]
                result = serializers.ListSerializer.to_representation(self, data)
                # flatten list of lists
                result = itertools.chain(*result)
                return result

    @property
    def module(self):
        module = self.context['request'].GET.get('module')
        if not module:
            raise ValidationError("Please specify the NLP module to use with a module= GET parameter")
        from nlpipe import tasks
        if not hasattr(tasks, module):
            raise ValidationError("Module {module} not known".format(**locals()))

        return getattr(tasks, module)

    def to_representation(self, article):
        result = self._cache[article.pk]
        if isinstance(result, list):
            return result
        else:
            return self.from_naf(result.input)

    def from_naf(self, naf):
        naf = KafNafParser(BytesIO(naf.encode("utf-8")))
        tokendict = {token.get_id(): token for token in naf.get_tokens()}

        for term in naf.get_terms():
            tokens = [tokendict[id] for id in term.get_span().get_span_ids()]
            for token in tokens:
                yield {"aid": article.pk,
                       "token_id": token.get_id(),
                       "offset": token.get_offset(),
                       "sentence": token.get_sent(),
                       "para": token.get_para(),
                       "word": token.get_text(),
                       "term_id": term.get_id(),
                       "lemma": term.get_lemma(),
                       "pos": term.get_pos()}

class NLPipeLemmataViewSet(ProjectViewSetMixin, ArticleSetViewSetMixin, DatatablesMixin, ModelViewSet):
    model_key = "token"
    model = Article
    queryset = Article.objects.all()
    serializer_class = NLPipeLemmataSerializer

    def filter_queryset(self, queryset):
        queryset = super(NLPipeLemmataViewSet, self).filter_queryset(queryset)
        # only(.) would be better on serializer, but meh
        try:
            queryset = queryset.filter(articlesets_set=self.articleset).only("pk")
        except ArticleSet.DoesNotExist:
            from django.http import Http404
            raise Http404("Articleset does not exist")
        return queryset

    def get_renderer_context(self):
        context = super(NLPipeLemmataViewSet, self).get_renderer_context()
        context['fast_csv'] = True
        return context

ModuleCount = namedtuple("ModuleCount", ["module", "n"])

class PreprocessViewSet(ProjectViewSetMixin, ArticleSetViewSetMixin, DatatablesMixin,
                        ListModelMixin, GenericViewSet):
    model_key = "preproces"
    model = None
    base_name = "preprocess"

    class serializer_class(serializers.Serializer):
        module = serializers.CharField()
        n = serializers.IntegerField()

    def filter_queryset(self, queryset):
        return queryset

    def get_queryset(self):
        ids = list(self.articleset.get_article_ids_from_elastic())
        result = [ModuleCount("Total #articles", len(ids))]
        from nlpipe.document import count_cached
        for module, n in count_cached(ids):
            result.append(ModuleCount(module, n))

        return result

    def get_filter_fields(self):
        return []
<|endoftext|>