<commit_before># ----------------------------------------------------------------------------
# pyglet
# Copyright (c) 2006-2008 Alex Holkner
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of pyglet nor the names of its
#    contributors may be used to endorse or promote products
#    derived from this software without specific prior written
#    permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: pil.py 163 2006-11-13 04:15:46Z Alex.Holkner $'

import sys

from ctypes import *

from pyglet.gl import *
from pyglet.image import *
from pyglet.image.codecs import *

from pyglet.window.carbon import carbon, quicktime
from pyglet.window.carbon.constants import _name
from pyglet.window.carbon.types import *

Handle = POINTER(POINTER(c_byte))
GWorldPtr = c_void_p
carbon.NewHandle.restype = Handle
HandleDataHandlerSubType = _name('hndl')
ComponentInstance = c_void_p

k1MonochromePixelFormat       = 0x00000001
k2IndexedPixelFormat          = 0x00000002
k4IndexedPixelFormat          = 0x00000004
k8IndexedPixelFormat          = 0x00000008
k16BE555PixelFormat           = 0x00000010
k24RGBPixelFormat             = 0x00000018
k32ARGBPixelFormat            = 0x00000020
k32BGRAPixelFormat            = _name('BGRA')
k1IndexedGrayPixelFormat      = 0x00000021
k2IndexedGrayPixelFormat      = 0x00000022
k4IndexedGrayPixelFormat      = 0x00000024
k8IndexedGrayPixelFormat      = 0x00000028
kNativeEndianPixMap           = 1 << 8

class QuickTimeImageDecoder(ImageDecoder):
    def get_file_extensions(self):
        # Only most common ones shown here
        return ['.bmp', '.cur', '.gif', '.ico', '.jpg', '.jpeg', '.pcx', '.png',
                '.tga', '.tif', '.tiff', '.xbm', '.xpm']

    def decode(self, file, filename):
        data = file.read()
        handle = Handle()
        dataref = Handle()
        carbon.PtrToHand(data, byref(handle), len(data))
        carbon.PtrToHand(byref(handle), byref(dataref), sizeof(Handle))
        importer = ComponentInstance()
        quicktime.GetGraphicsImporterForDataRef(dataref,
            HandleDataHandlerSubType, byref(importer))

        rect = Rect()
        quicktime.GraphicsImportGetNaturalBounds(importer, byref(rect))
        width = rect.right
        height = rect.bottom

        # TODO choose 24 bit where appropriate.
        if sys.byteorder == 'big':
            format = 'ARGB'
            qtformat = k32ARGBPixelFormat
        else:
            format = 'BGRA'
            qtformat = k32BGRAPixelFormat

        buffer = (c_byte * (width * height * len(format)))()
        world = GWorldPtr()
        quicktime.QTNewGWorldFromPtr(byref(world), qtformat,
            byref(rect), c_void_p(), c_void_p(), 0, buffer,
            len(format) * width)

        quicktime.GraphicsImportSetGWorld(importer, world, c_void_p())
        quicktime.GraphicsImportDraw(importer)
        quicktime.DisposeGWorld(world)

        pitch = len(format) * width

        return ImageData(width, height, format, buffer, -pitch)

def get_decoders():
    return [QuickTimeImageDecoder()]

def get_encoders():
    return []
<commit_msg>Quicktime animation.<commit_after># ----------------------------------------------------------------------------
# pyglet
# Copyright (c) 2006-2008 Alex Holkner
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
#  * Neither the name of pyglet nor the names of its
#    contributors may be used to endorse or promote products
#    derived from this software without specific prior written
#    permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# ----------------------------------------------------------------------------

'''
'''

__docformat__ = 'restructuredtext'
__version__ = '$Id: pil.py 163 2006-11-13 04:15:46Z Alex.Holkner $'

import sys

from ctypes import *

from pyglet.gl import *
from pyglet.image import *
from pyglet.image.codecs import *

from pyglet.window.carbon import carbon, quicktime, _oscheck
from pyglet.window.carbon.constants import _name
from pyglet.window.carbon.types import *

Handle = POINTER(POINTER(c_byte))
GWorldPtr = c_void_p
carbon.NewHandle.restype = Handle
HandleDataHandlerSubType = _name('hndl')
ComponentInstance = c_void_p

k1MonochromePixelFormat       = 0x00000001
k2IndexedPixelFormat          = 0x00000002
k4IndexedPixelFormat          = 0x00000004
k8IndexedPixelFormat          = 0x00000008
k16BE555PixelFormat           = 0x00000010
k24RGBPixelFormat             = 0x00000018
k32ARGBPixelFormat            = 0x00000020
k32BGRAPixelFormat            = _name('BGRA')
k1IndexedGrayPixelFormat      = 0x00000021
k2IndexedGrayPixelFormat      = 0x00000022
k4IndexedGrayPixelFormat      = 0x00000024
k8IndexedGrayPixelFormat      = 0x00000028
kNativeEndianPixMap           = 1 << 8

newMovieActive                = 1
noErr                         = 0
movieTrackMediaType = 1 << 0
movieTrackCharacteristic = 1 << 1
movieTrackEnabledOnly = 1 << 2
VisualMediaCharacteristic = _name('eyes')
nextTimeMediaSample           = 1

def Str255(value):
    return chr(len(value)) + value

class QuickTimeImageDecoder(ImageDecoder):
    def get_file_extensions(self):
        # Only most common ones shown here
        return ['.bmp', '.cur', '.gif', '.ico', '.jpg', '.jpeg', '.pcx', '.png',
                '.tga', '.tif', '.tiff', '.xbm', '.xpm']

    def get_animation_file_extensions(self):
        return ['.gif']

    def _get_data_ref(self, file, filename):
        data = file.read()
        handle = Handle()
        dataref = Handle()
        carbon.PtrToHand(data, byref(handle), len(data))
        carbon.PtrToHand(byref(handle), byref(dataref), sizeof(Handle))

        self.filename = filename =Str255(filename)
        carbon.PtrAndHand(filename, dataref, len(filename))

        return dataref

    def _get_formats(self):
        # TODO choose 24 bit where appropriate.
        if sys.byteorder == 'big':
            format = 'ARGB'
            qtformat = k32ARGBPixelFormat
        else:
            format = 'BGRA'
            qtformat = k32BGRAPixelFormat
        return format, qtformat

    def decode(self, file, filename):
        dataref = self._get_data_ref(file, filename)
        importer = ComponentInstance()
        quicktime.GetGraphicsImporterForDataRef(dataref,
            HandleDataHandlerSubType, byref(importer))

        rect = Rect()
        quicktime.GraphicsImportGetNaturalBounds(importer, byref(rect))
        width = rect.right
        height = rect.bottom

        format, qtformat = self._get_formats()

        buffer = (c_byte * (width * height * len(format)))()
        world = GWorldPtr()
        quicktime.QTNewGWorldFromPtr(byref(world), qtformat,
            byref(rect), c_void_p(), c_void_p(), 0, buffer,
            len(format) * width)

        quicktime.GraphicsImportSetGWorld(importer, world, c_void_p())
        quicktime.GraphicsImportDraw(importer)
        quicktime.DisposeGWorld(world)

        pitch = len(format) * width

        return ImageData(width, height, format, buffer, -pitch)

    def decode_animation(self, file, filename):
        # TODO: Stop playing chicken with the GC
        # TODO: Cleanup in errors

        quicktime.EnterMovies()

        data_ref = self._get_data_ref(file, filename)
        if not data_ref:
            raise ImageDecodeException(filename or file)

        movie = c_void_p()
        id = c_short()
        result = quicktime.NewMovieFromDataRef(byref(movie),
                                      newMovieActive,
                                      0,
                                      data_ref,
                                      HandleDataHandlerSubType)

        if not movie:
            #_oscheck(result)
            raise ImageDecodeException(filename or file)
        quicktime.GoToBeginningOfMovie(movie)

        time_scale = float(quicktime.GetMovieTimeScale(movie))

        format, qtformat = self._get_formats()

        # Get movie width and height
        rect = Rect()
        quicktime.GetMovieBox(movie, byref(rect))
        width = rect.right
        height = rect.bottom
        pitch = len(format) * width

        # Set gworld
        buffer = (c_byte * (width * height * len(format)))()
        world = GWorldPtr()
        quicktime.QTNewGWorldFromPtr(byref(world), qtformat,
            byref(rect), c_void_p(), c_void_p(), 0, buffer,
            len(format) * width)
        quicktime.SetGWorld(world, 0)
        quicktime.SetMovieGWorld(movie, world, 0)

        visual = quicktime.GetMovieIndTrackType(movie, 1,
                                                VisualMediaCharacteristic,
                                                movieTrackCharacteristic)
        if not visual:
            raise ImageDecodeException('No video track')

        animation = Animation([])

        time = 0

        interesting_time = c_int()
        quicktime.GetTrackNextInterestingTime(
            visual,
            nextTimeMediaSample,
            time,
            1,
            byref(interesting_time),
            None)
        duration = interesting_time.value / time_scale

        while time >= 0:
            result = quicktime.GetMoviesError()
            if result == noErr:
                # force redraw
                result = quicktime.UpdateMovie(movie)
            if result == noErr:
                # process movie
                quicktime.MoviesTask(movie, 0)
                result = quicktime.GetMoviesError()
            _oscheck(result)

            buffer_copy = (c_byte * len(buffer))()
            memmove(buffer_copy, buffer, len(buffer))
            image = ImageData(width, height, format, buffer_copy, -pitch)
            animation.frames.append(AnimationFrame(image, duration))

            interesting_time = c_int()
            duration = c_int()
            quicktime.GetTrackNextInterestingTime(
                visual,
                nextTimeMediaSample,
                time,
                1,
                byref(interesting_time),
                byref(duration))

            quicktime.SetMovieTimeValue(movie, interesting_time)
            time = interesting_time.value
            duration = duration.value / time_scale

        quicktime.DisposeMovie(movie)
        carbon.DisposeHandle(data_ref)

        quicktime.ExitMovies()

        return animation

def get_decoders():
    return [QuickTimeImageDecoder()]

def get_encoders():
    return []

<|endoftext|>


<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import operator
from collections import OrderedDict

from flask import request
from MaKaC.accessControl import AccessWrapper

from indico.core.db import db
from indico.modules.users import User, UserAffiliation, UserEmail
from indico.util.event import truncate_path
from indico.util.redis import write_client as redis_write_client
from indico.util.redis import suggestions
from MaKaC.conference import CategoryManager

def get_related_categories(user):
    """Gets the related categories of a user for the dashboard"""
    favorites = user.favorite_categories
    managed = user.get_linked_objects('category', 'manager')
    res = {}
    for categ in favorites | managed:
        res[(categ.getTitle(), categ.getId())] = {
            'categ': categ,
            'favorite': categ in favorites,
            'managed': categ in managed,
            'path': truncate_path(categ.getCategoryPathTitles(), 30, False)
        }
    return OrderedDict(sorted(res.items(), key=operator.itemgetter(0)))

def get_suggested_categories(user):
    """Gets the suggested categories of a user for the dashboard"""
    if not redis_write_client:
        return []
    related = user.favorite_categories | user.get_linked_objects('category', 'manager')
    res = []
    for id_, score in suggestions.get_suggestions(user, 'category').iteritems():
        try:
            categ = CategoryManager().getById(id_)
        except KeyError:
            suggestions.unsuggest(user, 'category', id_)
            continue
        if not categ or categ.isSuggestionsDisabled() or categ in related:
            continue
        if any(p.isSuggestionsDisabled() for p in categ.iterParents()):
            continue
        if not categ.canAccess(AccessWrapper(user.as_avatar, request.remote_addr)):
            continue
        res.append({
            'score': score,
            'categ': categ,
            'path': truncate_path(categ.getCategoryPathTitles(), 30, False)
        })
    return res

def serialize_user(user):
    """Serialize user to JSON-like object"""
    return {
        'id': user.id,
        'identifier': 'User:{}'.format(user.id),
        'name': user.full_name,
        'familyName': user.last_name,
        'firstName': user.first_name,
        'affiliation': user.affiliation,
        'email': user.email,
        '_type': 'Avatar'
    }

def _build_match(column, value, exact):
    value = value.replace('%', '')
    if exact:
        return column.ilike(value)
    else:
        return column.ilike("%{}%".format(value))

def search_users(exact=False, include_deleted=False, include_pending=False, external=False, **criteria):
    unspecified = object()
    query = db.session.query(User)
    original_criteria = dict(criteria)

    if not include_pending:
        query = query.filter(~User.is_pending)
    if not include_deleted:
        query = query.filter(~User.is_deleted)

    organisation = criteria.pop('affiliation', unspecified)
    if organisation is not unspecified:
        query = query.join(UserAffiliation).filter(_build_match(UserAffiliation.name, organisation, exact))

    email = criteria.pop('email', unspecified)
    if email is not unspecified:
        query = query.join(UserEmail).filter(_build_match(UserEmail.email, email, exact))

    for k, v in criteria.iteritems():
        query = query.filter(_build_match(getattr(User, k), v, exact))

    found_emails = {}
    found_identities = {}
    for user in query:
        for identity in user.identities:
            found_identities[(identity.provider, identity.identifier)] = user
        for email in user.all_emails:
            found_emails[email.lower()] = user

    # external user providers
    if external:
        from indico.modules.auth import multipass
        identities = multipass.search_identities(
            exact=exact,
            **{c: v for c, v in original_criteria.iteritems()})

        for ident in identities:
            if ((ident.provider.name, ident.identifier) not in found_identities and
                    ident.data['email'].lower() not in found_emails):
                found_emails[ident.data['email'].lower()] = ident
                found_identities[(ident.provider, ident.identifier)] = ident

    return set(found_emails.values())
<commit_msg>Return empty when no criteria<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2015 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

import operator
from collections import OrderedDict

from flask import request
from MaKaC.accessControl import AccessWrapper

from indico.core.db import db
from indico.modules.users import User, UserAffiliation, UserEmail
from indico.util.event import truncate_path
from indico.util.redis import write_client as redis_write_client
from indico.util.redis import suggestions
from MaKaC.conference import CategoryManager

def get_related_categories(user):
    """Gets the related categories of a user for the dashboard"""
    favorites = user.favorite_categories
    managed = user.get_linked_objects('category', 'manager')
    res = {}
    for categ in favorites | managed:
        res[(categ.getTitle(), categ.getId())] = {
            'categ': categ,
            'favorite': categ in favorites,
            'managed': categ in managed,
            'path': truncate_path(categ.getCategoryPathTitles(), 30, False)
        }
    return OrderedDict(sorted(res.items(), key=operator.itemgetter(0)))

def get_suggested_categories(user):
    """Gets the suggested categories of a user for the dashboard"""
    if not redis_write_client:
        return []
    related = user.favorite_categories | user.get_linked_objects('category', 'manager')
    res = []
    for id_, score in suggestions.get_suggestions(user, 'category').iteritems():
        try:
            categ = CategoryManager().getById(id_)
        except KeyError:
            suggestions.unsuggest(user, 'category', id_)
            continue
        if not categ or categ.isSuggestionsDisabled() or categ in related:
            continue
        if any(p.isSuggestionsDisabled() for p in categ.iterParents()):
            continue
        if not categ.canAccess(AccessWrapper(user.as_avatar, request.remote_addr)):
            continue
        res.append({
            'score': score,
            'categ': categ,
            'path': truncate_path(categ.getCategoryPathTitles(), 30, False)
        })
    return res

def serialize_user(user):
    """Serialize user to JSON-like object"""
    return {
        'id': user.id,
        'identifier': 'User:{}'.format(user.id),
        'name': user.full_name,
        'familyName': user.last_name,
        'firstName': user.first_name,
        'affiliation': user.affiliation,
        'email': user.email,
        '_type': 'Avatar'
    }

def _build_match(column, value, exact):
    value = value.replace('%', '')
    if exact:
        return column.ilike(value)
    else:
        return column.ilike("%{}%".format(value))

def search_users(exact=False, include_deleted=False, include_pending=False, external=False, **criteria):
    unspecified = object()
    query = db.session.query(User)
    original_criteria = dict(criteria)

    if not criteria:
        return set()

    if not include_pending:
        query = query.filter(~User.is_pending)
    if not include_deleted:
        query = query.filter(~User.is_deleted)

    organisation = criteria.pop('affiliation', unspecified)
    if organisation is not unspecified:
        query = query.join(UserAffiliation).filter(_build_match(UserAffiliation.name, organisation, exact))

    email = criteria.pop('email', unspecified)
    if email is not unspecified:
        query = query.join(UserEmail).filter(_build_match(UserEmail.email, email, exact))

    for k, v in criteria.iteritems():
        query = query.filter(_build_match(getattr(User, k), v, exact))

    found_emails = {}
    found_identities = {}
    for user in query:
        for identity in user.identities:
            found_identities[(identity.provider, identity.identifier)] = user
        for email in user.all_emails:
            found_emails[email.lower()] = user

    # external user providers
    if external:
        from indico.modules.auth import multipass
        identities = multipass.search_identities(
            exact=exact,
            **{c: v for c, v in original_criteria.iteritems()})

        for ident in identities:
            if ((ident.provider.name, ident.identifier) not in found_identities and
                    ident.data['email'].lower() not in found_emails):
                found_emails[ident.data['email'].lower()] = ident
                found_identities[(ident.provider, ident.identifier)] = ident

    return set(found_emails.values())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""Twisted utility functions"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import itertools as it

from functools import partial
from twisted.internet import defer
from twisted.internet.defer import (
    inlineCallbacks, maybeDeferred, gatherResults, returnValue)
from twisted.internet.task import coiterate, cooperate
from functools import partial
from itertools import ifilter, imap, starmap, repeat
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.reactor import spawnProcess
from pipe2py.lib import utils

WORKERS = 50

asyncNone = defer.succeed(None)
asyncReturn = partial(defer.succeed)

def _get_work(asyncCallable, callback, map_func, *iterables):
    func = lambda *args: asyncCallable(*args).addCallback(callback)
    return map_func(func, *iterables)

def _parallel(work, asyncCallable):
    deferreds = it.repeat(coiterate(work), WORKERS)
    return gatherResults(deferreds, consumeErrors=True)

# helper functions
# def coop(asyncCallable, callback, *iterables):
#     work = _get_work(asyncCallable, callback, it.imap, *iterables)
#     return coiterate(work)

# def asyncParallel(asyncCallable, callback, *iterables):
#     work = _get_work(asyncCallable, callback, it.imap, *iterables)
#     return _parallel(work, asyncCallable)

def coopStar(asyncCallable, callback, iterable):
    work = _get_work(asyncCallable, callback, it.starmap, *[iterable])
    return coiterate(work)

def asyncStarParallel(asyncCallable, callback, iterable):
    work = _get_work(asyncCallable, callback, it.starmap, *[iterable])
    return _parallel(work, asyncCallable)

def trueDeferreds(sources, filter_func=None):
    return it.imap(partial(maybeDeferred, it.ifilter, filter_func), sources)

@inlineCallbacks
def coopReduce(func, iterable, initializer=None):
    it = iter(iterable)
    x = initializer or next(it)

    def cooperator(func, it, x):
        for y in it:
            x = func(x, y)
            yield

        returnValue(x)

    task = cooperate(cooperator(func, it, x))
    result = yield task.whenDone()
    returnValue(result)

def asyncReduce(asyncCallable, iterable, initializer=None):
    it = iter(iterable)
    x = initializer or next(it)

    @inlineCallbacks
    def work(asyncCallable, it, x):
        for y in it:
            x = yield asyncCallable(x, y)

        returnValue(x)

    return work(asyncCallable, it, x)

def asyncImap(asyncCallable, *iterables):
    """itertools.imap for deferred callables
    """
    deferreds = it.imap(asyncCallable, *iterables)
    return gatherResults(deferreds, consumeErrors=True)

@inlineCallbacks
def asyncStarCmap(asyncCallable, iterable):
    """itertools.starmap for deferred callables using cooperative multitasking
    """
    results = []
    yield coopStar(asyncCallable, results.append, iterable)
    returnValue(results)

@inlineCallbacks
def asyncStarPmap(asyncCallable, iterable):
    """itertools.starmap for deferred callables using parallel cooperative
    multitasking
    """
    results = []
    yield asyncStarParallel(asyncCallable, results.append, iterable)
    returnValue(results)

def asyncStarMap(asyncCallable, iterable):
    """itertools.starmap for deferred callables
    """
    deferreds = it.starmap(asyncCallable, iterable)
    return gatherResults(deferreds, consumeErrors=True)

# Internal functions
_apply_func = partial(utils._apply_func, map_func=asyncStarMap)
_map_func = asyncImap

def asyncBroadcast(_INPUT, *asyncCallables):
    kwargs = {'map_func': _map_func, 'apply_func': _apply_func}
    return utils.broadcast(_INPUT, *asyncCallables, **kwargs)

def asyncDispatch(splits, *asyncCallables):
    kwargs = {'map_func': _map_func, 'apply_func': _apply_func}
    return utils.dispatch(splits, *asyncCallables, **kwargs)
<commit_msg>Add documentation<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""Twisted utility functions"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import itertools as it

from functools import partial
from twisted.internet import defer
from twisted.internet.defer import (
    inlineCallbacks, maybeDeferred, gatherResults, returnValue)
from twisted.internet.task import coiterate, cooperate
from functools import partial
from itertools import ifilter, imap, starmap, repeat
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.reactor import spawnProcess
from pipe2py.lib import utils

WORKERS = 50

asyncNone = defer.succeed(None)
asyncReturn = partial(defer.succeed)

def _get_work(asyncCallable, callback, map_func, *iterables):
    func = lambda *args: asyncCallable(*args).addCallback(callback)
    return map_func(func, *iterables)

def _parallel(work, asyncCallable):
    deferreds = it.repeat(coiterate(work), WORKERS)
    return gatherResults(deferreds, consumeErrors=True)

# helper functions
# def coop(asyncCallable, callback, *iterables):
#     work = _get_work(asyncCallable, callback, it.imap, *iterables)
#     return coiterate(work)

# def asyncParallel(asyncCallable, callback, *iterables):
#     work = _get_work(asyncCallable, callback, it.imap, *iterables)
#     return _parallel(work, asyncCallable)

def coopStar(asyncCallable, callback, iterable):
    work = _get_work(asyncCallable, callback, it.starmap, *[iterable])
    return coiterate(work)

def asyncStarParallel(asyncCallable, callback, iterable):
    work = _get_work(asyncCallable, callback, it.starmap, *[iterable])
    return _parallel(work, asyncCallable)

def trueDeferreds(sources, filter_func=None):
    return it.imap(partial(maybeDeferred, it.ifilter, filter_func), sources)

@inlineCallbacks
def coopReduce(func, iterable, initializer=None):
    it = iter(iterable)
    x = initializer or next(it)

    def cooperator(func, it, x):
        for y in it:
            x = func(x, y)
            yield

        returnValue(x)

    task = cooperate(cooperator(func, it, x))
    result = yield task.whenDone()
    returnValue(result)

def asyncReduce(asyncCallable, iterable, initializer=None):
    it = iter(iterable)
    x = initializer or next(it)

    @inlineCallbacks
    def work(asyncCallable, it, x):
        for y in it:
            x = yield asyncCallable(x, y)

        returnValue(x)

    return work(asyncCallable, it, x)

def asyncImap(asyncCallable, *iterables):
    """itertools.imap for deferred callables
    """
    deferreds = it.imap(asyncCallable, *iterables)
    return gatherResults(deferreds, consumeErrors=True)

@inlineCallbacks
def asyncStarCmap(asyncCallable, iterable):
    """itertools.starmap for deferred callables using cooperative multitasking
    """
    results = []
    yield coopStar(asyncCallable, results.append, iterable)
    returnValue(results)

@inlineCallbacks
def asyncStarPmap(asyncCallable, iterable):
    """itertools.starmap for deferred callables using parallel cooperative
    multitasking
    """
    results = []
    yield asyncStarParallel(asyncCallable, results.append, iterable)
    returnValue(results)

def asyncStarMap(asyncCallable, iterable):
    """itertools.starmap for deferred callables
    """
    deferreds = it.starmap(asyncCallable, iterable)
    return gatherResults(deferreds, consumeErrors=True)

# Internal functions
_apply_func = partial(utils._apply_func, map_func=asyncStarMap)
_map_func = asyncImap

def asyncBroadcast(_INPUT, *asyncCallables):
    """copies a source and delivers the items to multiple functions

    _INPUT = it.repeat({'title': 'foo'}, 3)

           /--> foo2bar(_INPUT) --> _OUTPUT1 == it.repeat('bar', 3)
          /
    _INPUT ---> foo2baz(_INPUT) --> _OUTPUT2 == it.repeat('baz', 3)
          \
           \--> foo2qux(_INPUT) --> _OUTPUT3 == it.repeat('quz', 3)

    The way you would construct such a flow in code would be::

        succeed = twisted.internet.defer.succeed
        foo2bar = lambda item: succeed(item['title'].replace('foo', 'bar'))
        foo2baz = lambda item: succeed(item['title'].replace('foo', 'baz'))
        foo2qux = lambda item: succeed(item['title'].replace('foo', 'quz'))
        asyncBroadcast(_INPUT, foo2bar, foo2baz, foo2qux)
    """
    kwargs = {'map_func': _map_func, 'apply_func': _apply_func}
    return utils.broadcast(_INPUT, *asyncCallables, **kwargs)

def asyncDispatch(splits, *asyncCallables):
    """takes multiple sources (returned by asyncDispatch or asyncBroadcast)
    and delivers the items to multiple functions

    _INPUT1 = it.repeat('bar', 3)
    _INPUT2 = it.repeat('baz', 3)
    _INPUT3 = it.repeat('qux', 3)

    _INPUT1 --> double(_INPUT) --> _OUTPUT1 == it.repeat('barbar', 3)

    _INPUT2 --> triple(_INPUT) --> _OUTPUT2 == it.repeat('bazbazbaz', 3)

    _INPUT3 --> quadruple(_INPUT) --> _OUTPUT3 == it.repeat('quxquxquxqux', 3)

    The way you would construct such a flow in code would be::

        succeed = twisted.internet.defer.succeed
        _INPUT = it.repeat({'title': 'foo'}, 3)
        splits = asyncBroadcast(_INPUT, foo2bar, foo2baz, foo2qux)
        double = lambda item: succeed(item * 2)
        triple = lambda item: succeed(item * 3)
        quadruple = lambda item: succeed(item * 4)
        asyncBroadcast(splits, double, triple, quadruple)
    """
    kwargs = {'map_func': _map_func, 'apply_func': _apply_func}
    return utils.dispatch(splits, *asyncCallables, **kwargs)
<|endoftext|>


<commit_before>import re
from collections import OrderedDict

class JATSContribException(Exception):
    pass

class JATSContribs(object):

    def __init__(self, soup=None):
        self.regex_spcom = re.compile(r'\s+,')
        self.regex_multisp = re.compile(r'\s+')
        self.regex_email = re.compile(r'^[a-zA-Z0-9+_.-]+@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+')
        self.soup = soup
        self.collab = {}
        self.auth_list = []
        self.xref_dict = OrderedDict()
        self.email_xref = OrderedDict()
        self.output = []
        pass

    def _name_to_ads(self, name):
        '''
        conv a flded name (surname/given) to ADS text name ('surname, given')
        '''
        try:
            # NOTE: using 'contents[0]' will strip out any markup in the object
            # surname = name.find('surname').contents[0]
            # given = name.find('given-names').contents[0]
            try:
                surname = name.find('surname').text
            except:
                surname = ''
            try:
                given = name.find('given-names').text
            except:
                given = ''
            if surname and given:
                ads_name = str(surname) + ', ' + str(given)
            elif surname:
                ads_name = str(surname)
            elif given:
                ads_name = str(given)
            else:
                ads_name = None
            return ads_name
        except Exception as noop:
            # print('this is a problem, add logging/exception')
            return

    def _match_xref(self):
        for a in self.auth_list:
            try:
                for item in a['xaff']:
                    item = re.sub(r'\s+',',',item)
                    xi = re.split(',',item)
                    for x in xi:
                        try:
                            a['aff'].append(self.xref_dict[x])
                        except Exception as err:
                            # print('missing key in xaff!', err)
                            pass

                    # if you found any emails in an affstring, add them
                    # to the email field
                    if item in self.email_xref:
                        a['email'].append(self.email_xref[item])

                # Check for 'ALLAUTH' affils (global affils without a key),
                # and assign them to all authors
                if 'ALLAUTH' in self.xref_dict:
                    a['aff'].append(self.xref_dict['ALLAUTH'])
            except Exception as noop:
                pass

            try:
                for item in a['xemail']:
                    try:
                        a['email'].append(self.xref_dict[item])
                    except Exception as err:
                        # print('missing key in xemail!',err)
                        pass
            except Exception as err:
                pass

    def _fix_email(self, email):
        email_new = []
        for em in email:
            if ' ' in em:
                for e in em.strip().split():
                    if '@' in e:
                        email_new.append(e.strip())
            else:
                if '@' in em:
                    email_new.append(em.strip())
        return list(dict.fromkeys(email_new))

    def _fix_orcid(self, orcid):
        orcid_new = []
        for orc in orcid:
            osplit = orc.strip().split()
            for o in osplit:
                o = o.rstrip('/').split('/')[-1]
                orcid_new.append(o)
        return list(dict.fromkeys(orcid_new))

    def _fix_affil(self,affstring):
        aff_list = affstring.split(';')
        new_aff = []
        emails = []
        for a in aff_list:
            a = a.strip()
            if self.regex_email.match(a):
                emails.append(a)
            else:
                # check for empty strings with commas
                checka = a.replace(',',' ')
                if checka.replace(' ',''):
                    new_aff.append(a)

        newaffstr = '; '.join(new_aff)
        return (newaffstr, emails)

    def _refield(self):
        out_auth = []
        out_aff = []
        email = None
        orcid = None
        for a in self.auth_list:
            try:
                name = a['name']
            except Exception as err:
                # this shouldn't happen, add an exception
                pass
            else:
                try:
                    aff = a['aff']
                    email = self._fix_email(a['email']) or None
                    orcid = self._fix_orcid(a['orcid']) or None
                    aff_new = '; '.join(aff)
                    if orcid:
                        orcid = '; '.join(orcid).strip()
                        orcid = '; <ID system="ORCID">' + orcid + '</ID>'
                        aff_new = aff_new + orcid
                    if email:
                        email = '; '.join(email).strip()
                        email = '<EMAIL>' + email + '</EMAIL>'
                        aff_new = aff_new + ' ' + email
                    aff_new = self.regex_spcom.sub(',',aff_new)
                    aff_new = self.regex_multisp.sub(' ',aff_new)
                except Exception as noop:
                    aff_new = ''
                out_auth.append(name.strip())
                out_aff.append(aff_new)

        # sanity check:
        if len(out_auth) != len(out_aff):
            # this should be a try/except...
            print('length mismatch in auth-affil data!')
        self.output = {'authors': out_auth, 'affiliations': out_aff}

    def _decompose(self, soup=None, tag=None):
        try:
            for element in soup(tag):
                element.decompose()
        except Exception as noop:
            pass
        return soup

    def parse(self):
        try:
            # BEGIN
            art_meta = self.soup

            # De-label everything:
            try:
                art_meta = self._decompose(soup=art_meta, tag='label')
            except Exception as noop:
                pass

            # JATS puts author data in <contrib-group>, giving individual
            # authors in each <contrib>
            try:
                art_contrib_group = art_meta.find('contrib-group').extract()
            except Exception as noop:
                # print('If you see this, you have a problem...', noop)
                pass
            else:
                contrib_raw = art_contrib_group.find_all('contrib')

                for a in contrib_raw:
                    try:
                        subcontrib = a.find_all('contrib')
                    except Exception as noop:
                        pass
                    auth = {}
                    l_correspondent = False

                    # note: IOP, APS get affil data within each contrib block,
                    #       OUP, AIP, Springer, etc get them via xrefs.
                    try:
                        if a['contrib-type'] == 'collab':
                            collab = a.find('collab')
                            try:
                                # collab_name = collab.contents[0]
                                collab_name = collab.text
                            except Exception as err:
                                pass
                            else:
                                self.collab = {'name': collab_name,
                                               'aff': [],
                                               'xaff': [],
                                               'xemail': [],
                                               'email': [],
                                               'corresp': False
                                              }
                        elif a['contrib-type'] == 'author':
                            if a.find('collab') is not None:
                                collab = a.find('collab')
                                try:
                                    collab_name = collab.contents[0]
                                except Exception as noop:
                                    pass
                                else:
                                    self.collab = {'name': collab_name,
                                                   'aff': [],
                                                   'xaff': [],
                                                   'xemail': [],
                                                   'email': [],
                                                   'corresp': False
                                                  }
                                    try:
                                        collab_affil = collab.find('address').text
                                    except Exception as noop:
                                        pass
                                    else:
                                        self.collab['aff'] = collab_affil
                            else:
                                # corresponding author?
                                try:
                                    if a['corresp'] == 'yes':
                                        l_correspondent = True
                                except Exception as noop:
                                    pass

                                # get author's name...
                                try:
                                    n1 = a.find('string-name')
                                    n1.name = 'name'
                                except Exception as noop:
                                    pass
                                name = self._name_to_ads(a.find('name').extract())

                                # get named affiliations within the contrib block
                                affs = a.find_all('aff')
                                aff_text = []
                                for i in affs:
                                    # special case: some pubs label affils with
                                    # label, strip them
                                    try:
                                        # NOTE: institution-id is actually useful, but at
                                        # at the moment, strip it
                                        a = self._decompose(soup=a, tag='institution-id')
                                        i = self._decompose(soup=i, tag='sup')
                                    except Exception as noop:
                                        pass
                                    affstr = (i.get_text(separator=' ').strip())
                                    (affstr, email_list) = self._fix_affil(affstr)
                                    aff_text.append(affstr)
                                    i.decompose()

                                # get xrefs...
                                xrefs = a.find_all('xref')
                                xref_aff = []
                                xref_email = []
                                for x in xrefs:
                                    try:
                                        if x['ref-type'] == 'aff':
                                            xref_aff.append(x['rid'])
                                        elif x['ref-type'] == 'corresp':
                                            xref_email.append(x['rid'])
                                    except Exception as noop:
                                        pass
                                    x.decompose()

                                # get orcid
                                contrib_id = a.find_all('contrib-id')
                                orcid = []
                                for c in contrib_id:
                                    try:
                                        if c['contrib-id-type'] == 'orcid':
                                            orcid.append(c.get_text(separator=' ').strip())
                                    except Exception as noop:
                                        pass
                                    c.decompose()

                                # get email(s)...
                                emails = []
                                # first, add any emails found by stripping raw emails out of affil strings above...
                                try:
                                    for e in email_list:
                                        emails.append(e)
                                except Exception as noop:
                                    pass
                                else:
                                    email_list = []
                                try:
                                    email = a.find_all('email')
                                    for e in email:
                                        try:
                                            emails.append(e.get_text(separator=' ').strip())
                                        except Exception as noop:
                                            pass
                                        e.decompose()
                                except Exception as noop:
                                    pass

                                # double-check for other things...
                                extlinks = a.find_all('ext-link')
                                for e in extlinks:
                                    # orcid
                                    try:
                                        if e['ext-link-type'] == 'orcid':
                                            orcid.append(e.get_text(separator=' ').strip())
                                    except Exception as noop:
                                        pass
                                    e.decompose()

                                # create the author dict
                                auth.update(corresp=l_correspondent)
                                auth.update(name=name)
                                auth.update(aff=aff_text)
                                auth.update(xaff=xref_aff, xemail=xref_email)
                                auth.update(orcid=orcid)
                                auth.update(email=emails)
                                a.decompose()
                                emails = None
                                orcid = None

                        # this is a list of author dicts
                        if auth:
                            self.auth_list.append(auth)
                    except Exception as err:
                        pass
                if self.collab:
                    self.auth_list.append(self.collab)

                # special case: affs defined in contrib-group, but
                #               not in individual contrib
                contrib_aff = art_contrib_group.find_all('aff')
                for a in contrib_aff:
                    # check and see if the publisher defined an email tag
                    # inside an affil (like IOP does...)
                    nested_email_list = a.find_all('ext-link')
                    if nested_email_list:
                         for e in nested_email_list:
                             try:
                                 key = e['id']
                                 value = e.text
                                 self.email_xref[key] = value
                                 e.decompose()
                             except Exception as noop:
                                 pass
                    try:
                        key = a['id']
                    except:
                        key = 'ALLAUTH'
                    try:
                        # special case: get rid of ...
                        try:
                            a = self._decompose(soup=a, tag='sup')
                            a = self._decompose(soup=a, tag='institution-id')
                            # getting rid of ext-link eliminates *all* emails,
                            # so this is not the place to fix the iop thing
                            # a = self._decompose(soup=a, tag='ext-link')
                        except Exception as noop:
                            pass
                        affstr = a.get_text(separator=' ').strip()
                        (affstr, email_list) = self._fix_affil(affstr)
                        if email_list:
                            self.email_xref[key] = email_list
                            email_list = []
                        self.xref_dict[key] = affstr
                    except Exception as err:
                        pass

            # now get the xref keys outside of contrib-group:

            # aff xrefs...
            try:
                aff_glob = art_meta.find_all('aff')
            except Exception as noop:
                aff_glob = None
            else:
                try:
                    for a in aff_glob:
                        key = a['id']
                        # special case: get rid of ...
                        try:
                            a = self._decompose(soup=a, tag='sup')
                        except Exception as noop:
                            pass
                        try:
                            # NOTE: institution-id is actually useful, but at
                            # at the moment, strip it
                            a = self._decompose(soup=a, tag='institution-id')
                        except Exception as noop:
                            pass
                        affstr = a.get_text(separator=' ').strip()
                        (affstr, email_list) = self._fix_affil(affstr)
                        self.xref_dict[key] = affstr
                        a.decompose()
                except Exception as err:
                    print('no aff id key!',a)
                    pass

            # author-notes xrefs...
            try:
                authnote_glob = art_meta.find_all('author-notes')
            except Exception as noop:
                authnote_glob = None
            else:
                try:
                    for a in authnote_glob:
                        # emails...
                        cor = a.find_all('corresp')
                        for c in cor:
                            try:
                                c = self._decompose(soup=c, tag='sup')
                                key = c['id']
                                val = c.get_text(separator=' ').strip()
                                self.xref_dict[key] = val
                                c.decompose()
                            except Exception as noop:
                                pass
                except Exception as err:
                    print('no authnote id key!',a)
                    pass

            # finishing up
            self._match_xref()
            self._refield()
            return
        except Exception as err:
            raise JATSContribException(err)
<commit_msg> 	modified:   pyingest/parsers/jats_contrib.py<commit_after>import re
from collections import OrderedDict

class JATSContribException(Exception):
    pass

class JATSContribs(object):

    def __init__(self, soup=None):
        self.regex_spcom = re.compile(r'\s+,')
        self.regex_multisp = re.compile(r'\s+')
        self.regex_email = re.compile(r'^[a-zA-Z0-9+_.-]+@[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+')
        self.soup = soup
        self.collab = {}
        self.auth_list = []
        self.xref_dict = OrderedDict()
        self.email_xref = OrderedDict()
        self.output = []
        pass

    def _name_to_ads(self, name):
        '''
        conv a flded name (surname/given) to ADS text name ('surname, given')
        '''
        try:
            # NOTE: using 'contents[0]' will strip out any markup in the object
            # surname = name.find('surname').contents[0]
            # given = name.find('given-names').contents[0]
            try:
                surname = name.find('surname').text
            except:
                surname = ''
            try:
                given = name.find('given-names').text
            except:
                given = ''
            if surname and given:
                ads_name = str(surname) + ', ' + str(given)
            elif surname:
                ads_name = str(surname)
            elif given:
                ads_name = str(given)
            else:
                ads_name = None
            return ads_name
        except Exception as noop:
            # print('this is a problem, add logging/exception')
            return

    def _match_xref(self):
        for a in self.auth_list:
            try:
                for item in a['xaff']:
                    item = re.sub(r'\s+',',',item)
                    xi = re.split(',',item)
                    for x in xi:
                        try:
                            a['aff'].append(self.xref_dict[x])
                        except Exception as err:
                            # print('missing key in xaff!', err)
                            pass

                    # if you found any emails in an affstring, add them
                    # to the email field
                    if item in self.email_xref:
                        a['email'].append(self.email_xref[item])

                # Check for 'ALLAUTH' affils (global affils without a key),
                # and assign them to all authors
                if 'ALLAUTH' in self.xref_dict:
                    a['aff'].append(self.xref_dict['ALLAUTH'])
            except Exception as noop:
                pass

            try:
                for item in a['xemail']:
                    try:
                        a['email'].append(self.xref_dict[item])
                    except Exception as err:
                        # print('missing key in xemail!',err)
                        pass
            except Exception as err:
                pass

    def _fix_email(self, email):
        email_new = []
        for em in email:
            if ' ' in em:
                for e in em.strip().split():
                    if '@' in e:
                        email_new.append(e.strip())
            else:
                if '@' in em:
                    email_new.append(em.strip())
        return list(dict.fromkeys(email_new))

    def _fix_orcid(self, orcid):
        orcid_new = []
        for orc in orcid:
            osplit = orc.strip().split()
            for o in osplit:
                o = o.rstrip('/').split('/')[-1]
                orcid_new.append(o)
        return list(dict.fromkeys(orcid_new))

    def _fix_affil(self,affstring):
        aff_list = affstring.split(';')
        new_aff = []
        emails = []
        for a in aff_list:
            a = a.strip()
            if self.regex_email.match(a):
                emails.append(a)
            else:
                # check for empty strings with commas
                checka = a.replace(',',' ')
                if checka.replace(' ',''):
                    new_aff.append(a)

        newaffstr = '; '.join(new_aff)
        return (newaffstr, emails)

    def _refield(self):
        out_auth = []
        out_aff = []
        email = None
        orcid = None
        for a in self.auth_list:
            try:
                name = a['name']
            except Exception as err:
                # this shouldn't happen, add an exception
                pass
            else:
                try:
                    aff = a['aff']
                    email = self._fix_email(a['email']) or None
                    orcid = self._fix_orcid(a['orcid']) or None
                    aff_new = '; '.join(aff)
                    if orcid:
                        orcid = '; '.join(orcid).strip()
                        orcid = '; <ID system="ORCID">' + orcid + '</ID>'
                        aff_new = aff_new + orcid
                    if email:
                        email = '; '.join(email).strip()
                        email = '<EMAIL>' + email + '</EMAIL>'
                        aff_new = aff_new + ' ' + email
                    aff_new = self.regex_spcom.sub(',',aff_new)
                    aff_new = self.regex_multisp.sub(' ',aff_new)
                except Exception as noop:
                    aff_new = ''
                out_auth.append(name.strip())
                out_aff.append(aff_new)

        # sanity check:
        if len(out_auth) != len(out_aff):
            # this should be a try/except...
            print('length mismatch in auth-affil data!')
        self.output = {'authors': out_auth, 'affiliations': out_aff}

    def _decompose(self, soup=None, tag=None):
        try:
            for element in soup(tag):
                element.decompose()
        except Exception as noop:
            pass
        return soup

    def parse(self):
        try:
            # BEGIN
            art_meta = self.soup

            # De-label everything:
            try:
                art_meta = self._decompose(soup=art_meta, tag='label')
            except Exception as noop:
                pass

            # JATS puts author data in <contrib-group>, giving individual
            # authors in each <contrib>
            try:
                art_contrib_group = art_meta.find('contrib-group').extract()
            except Exception as noop:
                # print('If you see this, you have a problem...', noop)
                pass
            else:
                contrib_raw = art_contrib_group.find_all('contrib')

                for a in contrib_raw:
                    try:
                        subcontrib = a.find_all('contrib')
                    except Exception as noop:
                        pass
                    auth = {}
                    l_correspondent = False

                    # note: IOP, APS get affil data within each contrib block,
                    #       OUP, AIP, Springer, etc get them via xrefs.
                    try:
                        if a['contrib-type'] == 'collab':
                            collab = a.find('collab')
                            try:
                                # collab_name = collab.contents[0]
                                collab_name = collab.text
                            except Exception as err:
                                pass
                            else:
                                self.collab = {'name': collab_name,
                                               'aff': [],
                                               'xaff': [],
                                               'xemail': [],
                                               'email': [],
                                               'corresp': False
                                              }
                        elif a['contrib-type'] == 'author':
                            if a.find('collab') is not None:
                                collab = a.find('collab')
                                try:
                                    collab_name = collab.contents[0]
                                except Exception as noop:
                                    pass
                                else:
                                    self.collab = {'name': collab_name,
                                                   'aff': [],
                                                   'xaff': [],
                                                   'xemail': [],
                                                   'email': [],
                                                   'corresp': False
                                                  }
                                    try:
                                        collab_affil = collab.find('address').text
                                    except Exception as noop:
                                        pass
                                    else:
                                        self.collab['aff'] = collab_affil
                            else:
                                # corresponding author?
                                try:
                                    if a['corresp'] == 'yes':
                                        l_correspondent = True
                                except Exception as noop:
                                    pass

                                # get author's name...
                                try:
                                    n1 = a.find('string-name')
                                    n1.name = 'name'
                                except Exception as noop:
                                    pass
                                name = self._name_to_ads(a.find('name').extract())

                                # get named affiliations within the contrib block
                                affs = a.find_all('aff')
                                aff_text = []
                                for i in affs:
                                    # special case: some pubs label affils with
                                    # label, strip them
                                    try:
                                        # NOTE: institution-id is actually useful, but at
                                        # at the moment, strip it
                                        a = self._decompose(soup=a, tag='institution-id')
                                        i = self._decompose(soup=i, tag='sup')
                                    except Exception as noop:
                                        pass
                                    affstr = (i.get_text(separator=' ').strip())
                                    (affstr, email_list) = self._fix_affil(affstr)
                                    aff_text.append(affstr)
                                    i.decompose()

                                # get xrefs...
                                xrefs = a.find_all('xref')
                                xref_aff = []
                                xref_email = []
                                for x in xrefs:
                                    try:
                                        if x['ref-type'] == 'aff':
                                            xref_aff.append(x['rid'])
                                        elif x['ref-type'] == 'corresp':
                                            xref_email.append(x['rid'])
                                    except Exception as noop:
                                        pass
                                    x.decompose()

                                # get orcid
                                contrib_id = a.find_all('contrib-id')
                                orcid = []
                                for c in contrib_id:
                                    try:
                                        if c['contrib-id-type'] == 'orcid':
                                            orcid.append(c.get_text(separator=' ').strip())
                                    except Exception as noop:
                                        pass
                                    c.decompose()

                                # get email(s)...
                                emails = []
                                # first, add any emails found by stripping raw emails out of affil strings above...
                                try:
                                    for e in email_list:
                                        emails.append(e)
                                except Exception as noop:
                                    pass
                                else:
                                    email_list = []
                                try:
                                    email = a.find_all('email')
                                    for e in email:
                                        try:
                                            emails.append(e.get_text(separator=' ').strip())
                                        except Exception as noop:
                                            pass
                                        e.decompose()
                                except Exception as noop:
                                    pass

                                # double-check for other things...
                                extlinks = a.find_all('ext-link')
                                for e in extlinks:
                                    # orcid
                                    try:
                                        if e['ext-link-type'] == 'orcid':
                                            orcid.append(e.get_text(separator=' ').strip())
                                    except Exception as noop:
                                        pass
                                    e.decompose()

                                # create the author dict
                                auth.update(corresp=l_correspondent)
                                auth.update(name=name)
                                auth.update(aff=aff_text)
                                auth.update(xaff=xref_aff, xemail=xref_email)
                                auth.update(orcid=orcid)
                                auth.update(email=emails)
                                a.decompose()
                                emails = None
                                orcid = None

                        # this is a list of author dicts
                        if auth:
                            self.auth_list.append(auth)
                    except Exception as err:
                        pass
                if self.collab:
                    self.auth_list.append(self.collab)

                # special case: affs defined in contrib-group, but
                #               not in individual contrib
                contrib_aff = art_contrib_group.find_all('aff')
                for a in contrib_aff:
                    # check and see if the publisher defined an email tag
                    # inside an affil (like IOP does...)
                    nested_email_list = a.find_all('ext-link')
                    if nested_email_list:
                         for e in nested_email_list:
                             try:
                                 key = e['id']
                                 value = e.text
                                 self.email_xref[key] = value
                                 e.decompose()
                             except Exception as noop:
                                 pass
                    try:
                        key = a['id']
                    except:
                        key = 'ALLAUTH'
                    try:
                        # special case: get rid of ...
                        try:
                            a = self._decompose(soup=a, tag='sup')
                            a = self._decompose(soup=a, tag='institution-id')
                            # getting rid of ext-link eliminates *all* emails,
                            # so this is not the place to fix the iop thing
                            # a = self._decompose(soup=a, tag='ext-link')
                        except Exception as noop:
                            pass
                        affstr = a.get_text(separator=' ').strip()
                        (affstr, email_list) = self._fix_affil(affstr)
                        if email_list:
                            self.email_xref[key] = email_list
                            email_list = []
                        self.xref_dict[key] = affstr
                    except Exception as err:
                        pass

            # now get the xref keys outside of contrib-group:

            # aff xrefs...
            try:
                aff_glob = art_meta.find_all('aff')
            except Exception as noop:
                aff_glob = None
            else:
                for a in aff_glob:
                    try:
                        key = a['id']
                    except:
                        key = 'ALLAUTH'
                    # special case: get rid of ...
                    try:
                        a = self._decompose(soup=a, tag='sup')
                    except Exception as noop:
                        pass
                    try:
                        # NOTE: institution-id is actually useful, but at
                        # at the moment, strip it
                        a = self._decompose(soup=a, tag='institution-id')
                    except Exception as noop:
                        pass
                    affstr = a.get_text(separator=' ').strip()
                    (affstr, email_list) = self._fix_affil(affstr)
                    self.xref_dict[key] = affstr
                    a.decompose()

            # author-notes xrefs...
            try:
                authnote_glob = art_meta.find_all('author-notes')
            except Exception as noop:
                authnote_glob = None
            else:
                try:
                    for a in authnote_glob:
                        # emails...
                        cor = a.find_all('corresp')
                        for c in cor:
                            try:
                                c = self._decompose(soup=c, tag='sup')
                                key = c['id']
                                val = c.get_text(separator=' ').strip()
                                self.xref_dict[key] = val
                                c.decompose()
                            except Exception as noop:
                                pass
                except Exception as err:
                    print('no authnote id key!',a)
                    pass

            # finishing up
            self._match_xref()
            self._refield()
            return
        except Exception as err:
            raise JATSContribException(err)
<|endoftext|>


<commit_before>"""
Helper methods for generating gw input / and work flows.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "May 2014"

import time
import os
import io
import ast
import copy
import math
import shutil
import numpy as np
from pymatgen.core.units import Ha_to_eV, eV_to_Ha
from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.transformations.standard_transformations import OxidationStateRemovalTransformation, \
    PrimitiveCellTransformation, SupercellTransformation

def now():
    """
    helper to return a time string
    """
    return time.strftime("%H:%M:%S %d/%m/%Y")

def read_extra_abivars():
    ea = {}
    if os.path.isfile('extra_abivars'):
        f = open('extra_abivars')
        ea = ast.literal_eval(f.read())
        if not is_converged(ea, dict):
            raise RuntimeError
    return ea

def refine_structure(structure, symprec=1e-3):
    remove_ox = OxidationStateRemovalTransformation()
    structure = remove_ox.apply_transformation(structure)
    sym_finder = SymmetryFinder(structure=structure, symprec=symprec)
    structure = sym_finder.get_refined_structure()
    get_prim = PrimitiveCellTransformation()
    structure = get_prim.apply_transformation(structure)
    m = structure.lattice.matrix
    x_prod = np.dot(np.cross(m[0], m[1]), m[2])
    if x_prod < 0:
        print x_prod
        trans = SupercellTransformation(((1, 0, 0), (0, 0, 1), (0, 1, 0)))
        structure = trans.apply_transformation(structure)
        m = structure.lattice.matrix
        x_prod = np.dot(np.cross(m[0], m[1]), m[2])
        print x_prod
        if x_prod < 0:
            raise RuntimeError
    return structure

def s_name(structure):
    if os.path.isfile('old'):
        name_ = str(structure.composition.reduced_formula)
    else:
        name_ = str(structure.composition.reduced_formula) + '_' + str(structure.item)
    return name_

def clean(some_string, uppercase=False):
    """
    helper to clean up an input string
    """
    if uppercase:
        return some_string.strip().upper()
    else:
        return some_string.strip().lower()

def expand_tests(tests, level):
    from pymatgen.io.gwwrapper.codeinterfaces import get_all_ecuteps, get_all_nbands
    new_tests = copy.deepcopy(tests)
    for test in tests.keys():
        if test in get_all_ecuteps():
            ec = str(test)
            ec_range = tests[ec]['test_range']
            ec_step = ec_range[-1] - ec_range[-2]
            if int(level / 2) == level / 2:
                print 'new ec wedge'
                # even level of grid extension > new ec wedge
                new_ec_range = (ec_range[-1] + int(level / 2 * ec_step),)
            else:
                print 'new nb wedge'
                # odd level of grid extension > new nb wedge
                extension = tuple(range(ec_range[-1] + ec_step, ec_range[-1] + (1 + int((level - 1) / 2)) * ec_step, ec_step))
                new_ec_range = ec_range + extension
            new_tests[ec].update({'test_range': new_ec_range})
        if test in get_all_nbands():
            nb = str(test)
            nb_range = tests[nb]['test_range']
            nb_step = nb_range[-1] - nb_range[-2]
            print nb_step
            if int(level / 2) == level / 2:
                # even level of grid extension > new ec wedge
                extension = tuple(range(nb_range[-1] + nb_step, nb_range[-1] + (1 + int(level / 2)) * nb_step, nb_step))
                new_nb_range = nb_range + extension
            else:
                # odd level of grid extension > new nb wedge
                new_nb_range = (nb_range[-1] + int((level + 1) / 2 * nb_step),)
            new_tests[nb].update({'test_range': new_nb_range})
    print new_tests
    return new_tests

def print_gnuplot_header(filename, title='', mode='convplot', filetype='jpeg'):
    xl = 'set xlabel "nbands"\n'
    yl = 'set ylabel "ecuteps (eV)"\n'
    zl = 'set zlabel "gap (eV)"\n'
    if mode == 'convplot':
        f = open(filename, mode='a')
        f.write('set terminal '+filetype+'\n')
        f.write('set title "'+title+'"\n')
        f.write(xl)
        f.write(yl)
        f.write(zl)
        f.close()

def read_grid_from_file(filename):
    """
    Read the results of a full set of calculations from file
    """
    try:
        f = open(filename, mode='r')
        full_res = ast.literal_eval(f.read())
        f.close()
    except SyntaxError:
        print 'Problems reading ', filename
        full_res = {'grid': 0, 'all_done': False}
    except (OSError, IOError):
        full_res = {'grid': 0, 'all_done': False}
    return full_res

def is_converged(hartree_parameters, structure, return_values=False):
    filename = s_name(structure) + ".conv_res"
    to_return = {}
    try:
        f = open(filename, mode='r')
        conv_res = ast.literal_eval(f.read())
        f.close()
        converged = True if True in conv_res['control'].values() else False
    except (IOError, OSError, ValueError):
        if return_values:
            print 'Inputfile ', filename, ' not found, the convergence calculation did not finish properly' \
                                          ' or was not parsed ...'
        converged = False
        return converged
    if return_values and converged:
        if hartree_parameters:
            try:
                conv_res['values']['ecut'] = 4 * math.ceil(conv_res['values']['ecut'] * eV_to_Ha / 4)
            except (KeyError, ArithmeticError, FloatingPointError, SyntaxError):
                pass
            try:
                conv_res['values']['ecuteps'] = 4 * math.ceil(conv_res['values']['ecuteps'] * eV_to_Ha / 4)
            except (KeyError, ArithmeticError, FloatingPointError, SyntaxError):
                pass
        for k in conv_res['values'].keys():
            if conv_res['values'][k] != 0 and conv_res['values'][k] != np.inf:
                to_return.update({k: conv_res['values'][k]})
        return to_return
    else:
        return converged

def store_conv_results(name, folder):
    print "| Storing results for %s" % name
    if not os.path.isdir(folder):
        os.mkdir(folder)
    shutil.copy(name+'.full_res', os.path.join(folder, name+'.full_res'))
    for data_file in ['conv_res', 'log', 'conv.log', 'str', 'fitdat', 'convdat', 'data']:
        try:
            os.rename(name+'.'+data_file, os.path.join(folder, name+'.'+data_file))
        except OSError:
            pass

def add_gg_gap(structure):
    structure.vbm_l = "G"
    structure.cbm_l = "G"
    structure.cbm = (0.0, 0.0, 0.0)
    structure.vbm = (0.0, 0.0, 0.0)
    return structure

<commit_msg>read extraabivars<commit_after>"""
Helper methods for generating gw input / and work flows.
"""

from __future__ import division

__author__ = "Michiel van Setten"
__copyright__ = " "
__version__ = "0.9"
__maintainer__ = "Michiel van Setten"
__email__ = "mjvansetten@gmail.com"
__date__ = "May 2014"

import time
import os
import io
import ast
import copy
import math
import shutil
import numpy as np
from pymatgen.core.units import Ha_to_eV, eV_to_Ha
from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.transformations.standard_transformations import OxidationStateRemovalTransformation, \
    PrimitiveCellTransformation, SupercellTransformation

def now():
    """
    helper to return a time string
    """
    return time.strftime("%H:%M:%S %d/%m/%Y")

def read_extra_abivars():
    ea = {}
    if os.path.isfile('extra_abivars'):
        f = open('extra_abivars')
        ea = ast.literal_eval(f.read())
        if not isinstance(ea, dict):
            raise RuntimeError
    return ea

def refine_structure(structure, symprec=1e-3):
    remove_ox = OxidationStateRemovalTransformation()
    structure = remove_ox.apply_transformation(structure)
    sym_finder = SymmetryFinder(structure=structure, symprec=symprec)
    structure = sym_finder.get_refined_structure()
    get_prim = PrimitiveCellTransformation()
    structure = get_prim.apply_transformation(structure)
    m = structure.lattice.matrix
    x_prod = np.dot(np.cross(m[0], m[1]), m[2])
    if x_prod < 0:
        print x_prod
        trans = SupercellTransformation(((1, 0, 0), (0, 0, 1), (0, 1, 0)))
        structure = trans.apply_transformation(structure)
        m = structure.lattice.matrix
        x_prod = np.dot(np.cross(m[0], m[1]), m[2])
        print x_prod
        if x_prod < 0:
            raise RuntimeError
    return structure

def s_name(structure):
    if os.path.isfile('old'):
        name_ = str(structure.composition.reduced_formula)
    else:
        name_ = str(structure.composition.reduced_formula) + '_' + str(structure.item)
    return name_

def clean(some_string, uppercase=False):
    """
    helper to clean up an input string
    """
    if uppercase:
        return some_string.strip().upper()
    else:
        return some_string.strip().lower()

def expand_tests(tests, level):
    from pymatgen.io.gwwrapper.codeinterfaces import get_all_ecuteps, get_all_nbands
    new_tests = copy.deepcopy(tests)
    for test in tests.keys():
        if test in get_all_ecuteps():
            ec = str(test)
            ec_range = tests[ec]['test_range']
            ec_step = ec_range[-1] - ec_range[-2]
            if int(level / 2) == level / 2:
                print 'new ec wedge'
                # even level of grid extension > new ec wedge
                new_ec_range = (ec_range[-1] + int(level / 2 * ec_step),)
            else:
                print 'new nb wedge'
                # odd level of grid extension > new nb wedge
                extension = tuple(range(ec_range[-1] + ec_step, ec_range[-1] + (1 + int((level - 1) / 2)) * ec_step, ec_step))
                new_ec_range = ec_range + extension
            new_tests[ec].update({'test_range': new_ec_range})
        if test in get_all_nbands():
            nb = str(test)
            nb_range = tests[nb]['test_range']
            nb_step = nb_range[-1] - nb_range[-2]
            print nb_step
            if int(level / 2) == level / 2:
                # even level of grid extension > new ec wedge
                extension = tuple(range(nb_range[-1] + nb_step, nb_range[-1] + (1 + int(level / 2)) * nb_step, nb_step))
                new_nb_range = nb_range + extension
            else:
                # odd level of grid extension > new nb wedge
                new_nb_range = (nb_range[-1] + int((level + 1) / 2 * nb_step),)
            new_tests[nb].update({'test_range': new_nb_range})
    print new_tests
    return new_tests

def print_gnuplot_header(filename, title='', mode='convplot', filetype='jpeg'):
    xl = 'set xlabel "nbands"\n'
    yl = 'set ylabel "ecuteps (eV)"\n'
    zl = 'set zlabel "gap (eV)"\n'
    if mode == 'convplot':
        f = open(filename, mode='a')
        f.write('set terminal '+filetype+'\n')
        f.write('set title "'+title+'"\n')
        f.write(xl)
        f.write(yl)
        f.write(zl)
        f.close()

def read_grid_from_file(filename):
    """
    Read the results of a full set of calculations from file
    """
    try:
        f = open(filename, mode='r')
        full_res = ast.literal_eval(f.read())
        f.close()
    except SyntaxError:
        print 'Problems reading ', filename
        full_res = {'grid': 0, 'all_done': False}
    except (OSError, IOError):
        full_res = {'grid': 0, 'all_done': False}
    return full_res

def is_converged(hartree_parameters, structure, return_values=False):
    filename = s_name(structure) + ".conv_res"
    to_return = {}
    try:
        f = open(filename, mode='r')
        conv_res = ast.literal_eval(f.read())
        f.close()
        converged = True if True in conv_res['control'].values() else False
    except (IOError, OSError, ValueError):
        if return_values:
            print 'Inputfile ', filename, ' not found, the convergence calculation did not finish properly' \
                                          ' or was not parsed ...'
        converged = False
        return converged
    if return_values and converged:
        if hartree_parameters:
            try:
                conv_res['values']['ecut'] = 4 * math.ceil(conv_res['values']['ecut'] * eV_to_Ha / 4)
            except (KeyError, ArithmeticError, FloatingPointError, SyntaxError):
                pass
            try:
                conv_res['values']['ecuteps'] = 4 * math.ceil(conv_res['values']['ecuteps'] * eV_to_Ha / 4)
            except (KeyError, ArithmeticError, FloatingPointError, SyntaxError):
                pass
        for k in conv_res['values'].keys():
            if conv_res['values'][k] != 0 and conv_res['values'][k] != np.inf:
                to_return.update({k: conv_res['values'][k]})
        return to_return
    else:
        return converged

def store_conv_results(name, folder):
    print "| Storing results for %s" % name
    if not os.path.isdir(folder):
        os.mkdir(folder)
    shutil.copy(name+'.full_res', os.path.join(folder, name+'.full_res'))
    for data_file in ['conv_res', 'log', 'conv.log', 'str', 'fitdat', 'convdat', 'data']:
        try:
            os.rename(name+'.'+data_file, os.path.join(folder, name+'.'+data_file))
        except OSError:
            pass

def add_gg_gap(structure):
    structure.vbm_l = "G"
    structure.cbm_l = "G"
    structure.cbm = (0.0, 0.0, 0.0)
    structure.vbm = (0.0, 0.0, 0.0)
    return structure

<|endoftext|>