<commit_before>import collections
from dataclasses import dataclass
from typing import Dict, Any

from siebenapp.domain import Command, Graph

@dataclass(frozen=True)
class ToggleSwitchableView(Command):
    """Switch between "only switchable goals" and "all goals" views"""

class SwitchableView(Graph):
    """Non-persistent layer that allows to
    show only switchable and/or selected goals"""

    def __init__(self, goaltree: Graph):
        super().__init__(goaltree)
        self._only_switchable: bool = False

    def verify(self) -> bool:
        return self.goaltree.verify()

    def accept_ToggleSwitchableView(self, command: ToggleSwitchableView):
        self._only_switchable = not self._only_switchable

    def events(self) -> collections.deque:
        return self.goaltree.events()

    def settings(self, key: str) -> int:
        if key == "filter_switchable":
            return self._only_switchable
        return self.goaltree.settings(key)

    def q(self, keys: str = "name") -> Dict[int, Any]:
        if skip_switchable := "switchable" not in keys:
            keys = ",".join([keys, "switchable"])
        pass_through = [
            k
            for k in [
                self.goaltree.settings("selection"),
                self.goaltree.settings("previous_selection"),
            ]
            if k is not None
        ]
        goals = self.goaltree.q(keys)
        if self._only_switchable:
            goals = {
                k: v for k, v in goals.items() if v["switchable"] or k in pass_through
            }
            for v in goals.values():
                if "edge" in v:
                    v["edge"] = []
        if skip_switchable:
            for v in goals.values():
                v.pop("switchable")
        return goals
<commit_msg>Use @with_key in SwitchableView (2/n)<commit_after>import collections
from dataclasses import dataclass
from typing import Dict, Any

from siebenapp.domain import Command, Graph, with_key

@dataclass(frozen=True)
class ToggleSwitchableView(Command):
    """Switch between "only switchable goals" and "all goals" views"""

class SwitchableView(Graph):
    """Non-persistent layer that allows to
    show only switchable and/or selected goals"""

    def __init__(self, goaltree: Graph):
        super().__init__(goaltree)
        self._only_switchable: bool = False

    def verify(self) -> bool:
        return self.goaltree.verify()

    def accept_ToggleSwitchableView(self, command: ToggleSwitchableView):
        self._only_switchable = not self._only_switchable

    def events(self) -> collections.deque:
        return self.goaltree.events()

    def settings(self, key: str) -> int:
        if key == "filter_switchable":
            return self._only_switchable
        return self.goaltree.settings(key)

    @with_key("switchable")
    def q(self, keys: str = "name") -> Dict[int, Any]:
        pass_through = [
            k
            for k in [
                self.goaltree.settings("selection"),
                self.goaltree.settings("previous_selection"),
            ]
            if k is not None
        ]
        goals = self.goaltree.q(keys)
        if self._only_switchable:
            goals = {
                k: v for k, v in goals.items() if v["switchable"] or k in pass_through
            }
            for v in goals.values():
                if "edge" in v:
                    v["edge"] = []
        return goals
<|endoftext|>


<commit_before>from django.db import transaction
from django.core.exceptions import ObjectDoesNotExist
from rest_framework import validators, decorators, status
from rest_framework.authentication import TokenAuthentication
from rest_framework.response import Response
from rest_framework.serializers import ModelSerializer, Field, CharField
from rest_framework.viewsets import ModelViewSet
from utils.rest import JSONStringField
from .models import Attempt
from django.core import signing
from rest_framework.exceptions import PermissionDenied

def update_fields(obj, new_values):
    changed_fields = []
    for field, value in list(new_values.items()):
        if value != getattr(obj, field):
            setattr(obj, field, value)
            changed_fields.append(field)
    return changed_fields

class WritableJSONField(Field):
    def to_internal_value(self, data):
        return data

class AttemptSerializer(ModelSerializer):
    """
    Serialize an Attempt object.
    """
    solution = CharField(allow_blank=True, trim_whitespace=False)
    secret = WritableJSONField(write_only=True, required=False)
    token = CharField(write_only=True, required=False)
    feedback = JSONStringField()

    class Meta:
        model = Attempt
        fields = '__all__'

    @staticmethod
    def check_secret(validated_data):
        # Check and remove secret from the validated_data dictionary
        user_secret = validated_data.pop('secret', '[]')
        secret_matches, wrong_index = validated_data['part'].check_secret(user_secret)
        if not secret_matches:
            validated_data['valid'] = False
            return wrong_index

    @staticmethod
    def check_token(validated_data, user):
        if not validated_data['part'].problem.verify_attempt_tokens:
            validated_data.pop('token', None)
            return
        data = signing.loads(validated_data.pop('token'))
        if data['user'] != user.pk or data['part'] != validated_data['part'].pk:
            raise PermissionDenied()

    def create(self, validated_data):
        self.check_secret(validated_data)
        return super(AttemptSerializer, self).create(validated_data)

    def update(self, instance, validated_data):
        self.check_secret(validated_data)
        return super(AttemptSerializer, self).update(instance, validated_data)

class AttemptViewSet(ModelViewSet):
    """
    A viewset for viewing and editing Attempt instances.
    """
    serializer_class = AttemptSerializer
    queryset = Attempt.objects.all()

    @decorators.list_route(methods=['post'], authentication_classes=[TokenAuthentication])
    @transaction.atomic
    def submit(self, request):
        serializer = AttemptSerializer(data=request.data, many=True, partial=True)

        def _f(validator):
            return not isinstance(validator, validators.UniqueTogetherValidator)
        serializer.child.validators = list(filter(_f, serializer.child.validators))

        if serializer.is_valid():
            attempts = []
            wrong_indices = {}
            for attempt_data in serializer.validated_data:
                AttemptSerializer.check_token(attempt_data, request.user)
                wrong_index = AttemptSerializer.check_secret(attempt_data)
                wrong_indices[attempt_data['part'].pk] = wrong_index
                updated_fields = None
                try:
                    attempt = Attempt.objects.get(user=request.user,
                                                  part=attempt_data['part'])
                    updated_fields = update_fields(attempt, attempt_data)
                except ObjectDoesNotExist:
                    attempt = Attempt(user=request.user, **attempt_data)
                finally:
                    attempt.save(update_fields=updated_fields)
                    attempts.append(attempt)
            data = {
                'attempts': AttemptSerializer(attempts, many=True).data,
                'wrong_indices': wrong_indices
            }
            return Response(data, status=status.HTTP_200_OK)
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
<commit_msg>Refactor AttemptSerializer.check_token<commit_after>from django.db import transaction
from django.core.exceptions import ObjectDoesNotExist
from rest_framework import validators, decorators, status
from rest_framework.authentication import TokenAuthentication
from rest_framework.response import Response
from rest_framework.serializers import ModelSerializer, Field, CharField
from rest_framework.viewsets import ModelViewSet
from utils.rest import JSONStringField
from .models import Attempt
from django.core import signing
from rest_framework.exceptions import PermissionDenied

def update_fields(obj, new_values):
    changed_fields = []
    for field, value in list(new_values.items()):
        if value != getattr(obj, field):
            setattr(obj, field, value)
            changed_fields.append(field)
    return changed_fields

class WritableJSONField(Field):
    def to_internal_value(self, data):
        return data

class AttemptSerializer(ModelSerializer):
    """
    Serialize an Attempt object.
    """
    solution = CharField(allow_blank=True, trim_whitespace=False)
    secret = WritableJSONField(write_only=True, required=False)
    token = CharField(write_only=True, required=False)
    feedback = JSONStringField()

    class Meta:
        model = Attempt
        fields = '__all__'

    @staticmethod
    def check_secret(validated_data):
        # Check and remove secret from the validated_data dictionary
        user_secret = validated_data.pop('secret', '[]')
        secret_matches, wrong_index = validated_data['part'].check_secret(user_secret)
        if not secret_matches:
            validated_data['valid'] = False
            return wrong_index

    @staticmethod
    def check_token(validated_data, user):
        if not validated_data['part'].problem.verify_attempt_tokens:
            validated_data.pop('token', None)
            return True
        data = signing.loads(validated_data.pop('token'))
        return data['user'] == user.pk and data['part'] == validated_data['part'].pk

    def create(self, validated_data):
        self.check_secret(validated_data)
        return super(AttemptSerializer, self).create(validated_data)

    def update(self, instance, validated_data):
        self.check_secret(validated_data)
        return super(AttemptSerializer, self).update(instance, validated_data)

class AttemptViewSet(ModelViewSet):
    """
    A viewset for viewing and editing Attempt instances.
    """
    serializer_class = AttemptSerializer
    queryset = Attempt.objects.all()

    @decorators.list_route(methods=['post'], authentication_classes=[TokenAuthentication])
    @transaction.atomic
    def submit(self, request):
        serializer = AttemptSerializer(data=request.data, many=True, partial=True)

        def _f(validator):
            return not isinstance(validator, validators.UniqueTogetherValidator)
        serializer.child.validators = list(filter(_f, serializer.child.validators))

        if serializer.is_valid():
            attempts = []
            wrong_indices = {}
            for attempt_data in serializer.validated_data:
                if not AttemptSerializer.check_token(attempt_data, request.user):
                    raise PermissionDenied()
                wrong_index = AttemptSerializer.check_secret(attempt_data)
                wrong_indices[attempt_data['part'].pk] = wrong_index
                updated_fields = None
                try:
                    attempt = Attempt.objects.get(user=request.user,
                                                  part=attempt_data['part'])
                    updated_fields = update_fields(attempt, attempt_data)
                except ObjectDoesNotExist:
                    attempt = Attempt(user=request.user, **attempt_data)
                finally:
                    attempt.save(update_fields=updated_fields)
                    attempts.append(attempt)
            data = {
                'attempts': AttemptSerializer(attempts, many=True).data,
                'wrong_indices': wrong_indices
            }
            return Response(data, status=status.HTTP_200_OK)
        else:
            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
<|endoftext|>


<commit_before>from willie.module import commands

import urllib

@commands('cs')
def chatsounds(bot, trigger):
    bot.say('http://cs.3kelv.in/?s=%s' % (urllib.quote_plus(' '.join(trigger.groups))))<commit_msg>OK does this work<commit_after>from willie.module import commands

import urllib

@commands('cs')
def chatsounds(bot, trigger):
    bot.say('http://cs.3kelv.in/?s=%s' % (urllib.quote_plus(' '.join(trigger.args[3:]))))<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import multiprocessing as mp
import os
import json
import uuid
import sys
from concurrent import futures
from collections import defaultdict
from functools import partial

from bs4 import BeautifulSoup
from markdown import markdown
import requests
import urllib3

# Ignore security hazard since certs SHOULD be trusted (https)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Avoid rate limiting (tcp)
URL_BOT_ID = f'Bot {str(uuid.uuid4())}'

def extract_urls_from_html(content):
    soup = BeautifulSoup(content, 'html.parser')
    html_urls = set()
    for a in soup.find_all('a', href=True):
        url = a['href']
        if url.startswith('http'):
            html_urls.add(url)
    return html_urls

def extract_urls(discover_path):
    exclude = ['.git', '.vscode']
    all_urls = defaultdict(list)
    max_strlen = -1
    for root, dirs, files in os.walk(discover_path, topdown=True):
        dirs[:] = [d for d in dirs if d not in exclude]
        short_root = root.lstrip(discover_path)
        for file in files:
            output = f'Currently checking: file={file}'
            file_path = os.path.join(root, file)
            if max_strlen < len(output):
                max_strlen = len(output)
            print(output.ljust(max_strlen), end='\r')
            if file_path.endswith('.html'):
                content = open(file_path)
                extract_urls_from_html(content)
            elif file_path.endswith('.markdown'):
                content = markdown(open(file_path).read())
            else:
                continue
            html_urls = extract_urls_from_html(content)
            for url in html_urls:
                all_urls[url].append(os.path.join(short_root, file))
    return all_urls

def run_workers(work, data, threads, **kwargs):
    work_partial = partial(work, **kwargs)
    with futures.ThreadPoolExecutor(max_workers=threads) as executor:
        future_to_result = {
            executor.submit(work_partial, arg): arg
            for arg in data
        }
        for future in futures.as_completed(future_to_result):
            yield future.result()

def get_url_status(url, timeout, retries):
    for local in ('localhost', '127.0.0.1', 'app_server'):
        if url.startswith('http://' + local):
            return (url, 0)
    clean_url = url.strip('?.')
    try:
        with requests.Session() as session:
            adapter = requests.adapters.HTTPAdapter(max_retries=retries)
            session.mount('http://', adapter)
            session.mount('https://', adapter)
            response = session.get(
                clean_url, verify=False, timeout=timeout,
                headers={'User-Agent': URL_BOT_ID})
            return (clean_url, response.status_code)
    except requests.exceptions.Timeout:
        return (clean_url, 504)
    except requests.exceptions.ConnectionError:
        return (clean_url, -1)
    except requests.exceptions.TooManyRedirects:
        return (clean_url, -1)

def bad_url(url_status):
    if url_status == -1:
        return True
    elif url_status == 401 or url_status == 403:
        return False
    elif url_status == 503:
        return False
    elif url_status >= 400:
        return True
    return False

def parse_args(argv):
    parser = argparse.ArgumentParser(
        description='Check correctness of url links.',
        add_help=True)
    parser.add_argument(
        '--url-timeout',
        default=10.0,
        dest='timeout',
        help='Timeout in seconds to wait for url')
    parser.add_argument(
        '--url-retries',
        default=10,
        dest='retries',
        help='Number of url retries')
    parser.add_argument(
        '--num-threads',
        default=mp.cpu_count()*4,
        dest='threads',
        help='Number of threads to run with')
    return parser.parse_args(argv)

def main():
    args = parse_args(sys.argv[1:])
    print('Extract urls...')
    all_urls = extract_urls(os.getcwd() + os.path.sep + 'content')
    print('\nCheck urls...')
    bad_url_status = {}
    url_id = 1
    max_strlen = -1
    for url_path, url_status in run_workers(
            get_url_status, all_urls.keys(),
            threads=args.threads, timeout=args.timeout, retries=args.retries):
        output = f'Currently checking: id={url_id} host={urllib3.util.parse_url(url_path).host}'
        if max_strlen < len(output):
            max_strlen = len(output)
        print(output.ljust(max_strlen), end='\r')
        if bad_url(url_status) is True:
            bad_url_status[url_path] = url_status
        url_id += 1
    bad_url_location = {
        bad_url: all_urls[bad_url]
        for bad_url in bad_url_status
    }
    status_content = json.dumps(bad_url_status, indent=4)
    location_content = json.dumps(bad_url_location, indent=4)
    print(f'\nBad url status: {status_content}')
    print(f'\nBad url locations: {location_content}')

if __name__ == '__main__':
    main()
<commit_msg>Optimize imports in check_urls<commit_after>#!/usr/bin/env python
import os
from argparse import ArgumentParser
from concurrent import futures
from collections import defaultdict
from functools import partial
from json import dumps
from multiprocessing import cpu_count
from sys import argv
from uuid import uuid4

import requests
import urllib3
from bs4 import BeautifulSoup
from markdown import markdown

# Ignore security hazard since certs SHOULD be trusted (https)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# Avoid rate limiting (tcp)
URL_BOT_ID = f'Bot {str(uuid4())}'

def extract_urls_from_html(content):
    soup = BeautifulSoup(content, 'html.parser')
    html_urls = set()
    for a in soup.find_all('a', href=True):
        url = a['href']
        if url.startswith('http'):
            html_urls.add(url)
    return html_urls

def extract_urls(discover_path):
    exclude = ['.git', '.vscode']
    all_urls = defaultdict(list)
    max_strlen = -1
    for root, dirs, files in os.walk(discover_path, topdown=True):
        dirs[:] = [d for d in dirs if d not in exclude]
        short_root = root.lstrip(discover_path)
        for file in files:
            output = f'Currently checking: file={file}'
            file_path = os.path.join(root, file)
            if max_strlen < len(output):
                max_strlen = len(output)
            print(output.ljust(max_strlen), end='\r')
            if file_path.endswith('.html'):
                content = open(file_path)
                extract_urls_from_html(content)
            elif file_path.endswith('.markdown'):
                content = markdown(open(file_path).read())
            else:
                continue
            html_urls = extract_urls_from_html(content)
            for url in html_urls:
                all_urls[url].append(os.path.join(short_root, file))
    return all_urls

def run_workers(work, data, threads, **kwargs):
    work_partial = partial(work, **kwargs)
    with futures.ThreadPoolExecutor(max_workers=threads) as executor:
        future_to_result = {
            executor.submit(work_partial, arg): arg
            for arg in data
        }
        for future in futures.as_completed(future_to_result):
            yield future.result()

def get_url_status(url, timeout, retries):
    for local in ('localhost', '127.0.0.1', 'app_server'):
        if url.startswith('http://' + local):
            return (url, 0)
    clean_url = url.strip('?.')
    try:
        with requests.Session() as session:
            adapter = requests.adapters.HTTPAdapter(max_retries=retries)
            session.mount('http://', adapter)
            session.mount('https://', adapter)
            response = session.get(
                clean_url, verify=False, timeout=timeout,
                headers={'User-Agent': URL_BOT_ID})
            return (clean_url, response.status_code)
    except requests.exceptions.Timeout:
        return (clean_url, 504)
    except requests.exceptions.ConnectionError:
        return (clean_url, -1)
    except requests.exceptions.TooManyRedirects:
        return (clean_url, -1)

def bad_url(url_status):
    if url_status == -1:
        return True
    elif url_status == 401 or url_status == 403:
        return False
    elif url_status == 503:
        return False
    elif url_status >= 400:
        return True
    return False

def parse_args(argv):
    parser = ArgumentParser(
        description='Check correctness of url links.',
        add_help=True)
    parser.add_argument(
        '--url-timeout',
        default=10.0,
        dest='timeout',
        help='Timeout in seconds to wait for url')
    parser.add_argument(
        '--url-retries',
        default=10,
        dest='retries',
        help='Number of url retries')
    parser.add_argument(
        '--num-threads',
        default=cpu_count()*4,
        dest='threads',
        help='Number of threads to run with')
    return parser.parse_args(argv)

def main():
    args = parse_args(argv[1:])
    print('Extract urls...')
    all_urls = extract_urls(os.getcwd() + os.path.sep + 'content')
    print('\nCheck urls...')
    bad_url_status = {}
    url_id = 1
    max_strlen = -1
    for url_path, url_status in run_workers(
            get_url_status, all_urls.keys(),
            threads=args.threads, timeout=args.timeout, retries=args.retries):
        output = f'Currently checking: id={url_id} host={urllib3.util.parse_url(url_path).host}'
        if max_strlen < len(output):
            max_strlen = len(output)
        print(output.ljust(max_strlen), end='\r')
        if bad_url(url_status) is True:
            bad_url_status[url_path] = url_status
        url_id += 1
    bad_url_location = {
        bad_url: all_urls[bad_url]
        for bad_url in bad_url_status
    }
    status_content = dumps(bad_url_status, indent=4)
    location_content = dumps(bad_url_location, indent=4)
    print(f'\nBad url status: {status_content}')
    print(f'\nBad url locations: {location_content}')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#
# Copyright (C) 2012-2013 Craig Hobbs
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

from .compat import basestring_, StringIO, wsgistr_, wsgistr_new, wsgistr_str
from .doc import DocAction
from .model import jsonDefault, Struct
from .request import Request
from .resource.base import ResourceCollection
from .spec import SpecParser

from collections import namedtuple
import imp
import json
import logging
import os
import re
import sys
import threading

# Application configuration file specification
ConfigSpec = """\
# Log level
enum LogLevel
    Debug
    Info
    Warning
    Error

# Resource description
struct Resource

    # Resource name
    string name

    # Resource type
    string type

    # Resource string (e.g. connection string)
    string resourceString

# Application configuration file
struct ApplicationConfig

    # Specification directories (top-level only)
    string[] specPaths

    # Module directories (top-level only)
    string[] modulePaths

    # Resources
    [optional] Resource[] resources

    # Configuration key/value string pairs
    [optional] string{} config

    # Logger output level (default is Warning)
    [optional] LogLevel logLevel

    # Pretty JSON output (default is false)
    [optional] bool prettyOutput

    # Re-load specs and scripts on every requests (default is false)
    [optional] bool alwaysReload

    # Disable documentation (default is false)
    [optional] bool disableDocs

    # External CSS for generated documenation HTML (default is None)
    [optional] string docCssUri
"""

configComment = re.compile("^\s*#.*$", flags = re.MULTILINE)

# Application configuration file model
_configParser = SpecParser()
_configParser.parseString(ConfigSpec)
configModel = _configParser.types["ApplicationConfig"]

# Top-level WSGI application class
class Application(object):

    ENVIRON_APP = "chisel.app"

    ThreadState = namedtuple("ThreadState", "environ, start_response, log")

    def __init__(self, configPath, wrapApplication = None, resourceTypes = None, logStream = sys.stderr):

        self._relPathBase = os.path.dirname(sys.modules[self.__module__].__file__)
        if isinstance(configPath, basestring_):
            self._configPath = configPath if os.path.isabs(configPath) else os.path.join(self._relPathBase, configPath)
            self._configString = None
        else:
            self._configPath = None
            self._configString = configPath.read()
        self._wrapApplication = wrapApplication
        self._resourceTypes = dict((x.name, x) for x in resourceTypes) if resourceTypes else {}
        self._logStream = logStream
        self._threadStates = {}
        self.init()
        self._initLock = threading.Lock()

    def init(self):

        # Read config file
        if self._configPath:
            with open(self._configPath, "r") as fConfig:
                self._configString = fConfig.read()
        self._config = Struct(configModel.validate(json.loads(configComment.sub("", self._configString))))

        # Load specs
        self._specParser = SpecParser()
        for specPath in self._config.specPaths:
            if not os.path.isabs(specPath):
                specPath = os.path.join(self._relPathBase, specPath)
            self.loadSpecs(specPath)

        # Load modules
        self._requests = {}
        self._requestUrls = {}
        for modulePath in self._config.modulePaths:
            if not os.path.isabs(modulePath):
                modulePath = os.path.join(self._relPathBase, modulePath)
            self.loadModules(modulePath)

        # Add the doc request
        if not self._config.disableDocs:
            self.addRequest(DocAction())

        # Create resources collection
        self._resources = ResourceCollection()
        if self._config.resources:
            for resource in self._config.resources:
                if resource.type not in self._resourceTypes:
                    raise Exception("Unknown resource type '%s'" % (resource.type,))
                self._resources.add(resource.name, self._resourceTypes[resource.type], resource.resourceString)

        # Default thread state
        self._threadStateDefault = self.ThreadState(None, None, self._createLogger(self._logStream))

    # Recursively load all specs in a directory
    def loadSpecs(self, specPath, specExt = ".chsl", finalize = True):

        # Does the path exist?
        if not os.path.isdir(specPath):
            raise IOError("%r not found or is not a directory" % (specPath,))

        # Resursively find spec files
        for dirpath, dirnames, filenames in os.walk(specPath):
            for filename in filenames:
                (base, ext) = os.path.splitext(filename)
                if ext == specExt:
                    self._specParser.parse(os.path.join(dirpath, filename), finalize = False)
        if finalize:
            self._specParser.finalize()

    # Load a spec string
    def loadSpecString(self, spec, fileName = "", finalize = True):
        self._specParser.parseString(spec, fileName = fileName, finalize = finalize)

    # Add a request handler (Request-wrapped WSGI application)
    def addRequest(self, request):

        # Wrap bare functions in a request decorator
        if not isinstance(request, Request):
            request = Request(request)

        # Duplicate request name?
        if request.name in self._requests:
            raise Exception("Redefinition of request '%s'" % (request.name,))
        self._requests[request.name] = request

        # Add the request URLs
        for url in request.urls:
            if url in self._requestUrls:
                raise Exception("Redefinition of request URL '%s'" % (url,))
            self._requestUrls[url] = request

        # Make the request app-aware at load-time
        request.onload(self)

    # Recursively load all modules in a directory
    def loadModules(self, moduleDir, moduleExt = ".py"):

        # Does the path exist?
        if not os.path.isdir(moduleDir):
            raise IOError("%r not found or is not a directory" % (moduleDir,))

        # Recursively find module files
        moduleDirNorm = os.path.normpath(moduleDir)
        modulePathParent = os.path.dirname(moduleDirNorm)
        modulePathBase = os.path.join(modulePathParent, "") if modulePathParent else modulePathParent
        for dirpath, dirnames, filenames in os.walk(moduleDirNorm):
            for filename in filenames:
                (base, ext) = os.path.splitext(filename)
                if ext == moduleExt:

                    # Load the module
                    module = None
                    moduleParts = []
                    for modulePart in os.path.join(dirpath, base)[len(modulePathBase):].split(os.sep):
                        moduleParts.append(modulePart)
                        moduleFp, modulePath, moduleDesc = \
                            imp.find_module(modulePart, module.__path__ if module else [modulePathParent])
                        try:
                            module = imp.load_module(".".join(moduleParts), moduleFp, modulePath, moduleDesc)
                        finally:
                            if moduleFp:
                                moduleFp.close()

                    # Add the module's requests
                    for moduleAttr in dir(module):
                        request = getattr(module, moduleAttr)
                        if isinstance(request, Request):
                            self.addRequest(request)

    # WSGI entry point
    def __call__(self, environ, start_response):

        # Set the app environment item
        environ[self.ENVIRON_APP] = self

        # Add the thread state
        threadKey = threading.current_thread().ident
        threadState = self.ThreadState(environ, start_response, self._createLogger(environ["wsgi.errors"]))
        self._threadStates[threadKey] = threadState

        # Handle the request
        try:
            # Reload, if requested
            if self._config.alwaysReload:
                with self._initLock:
                    self.log.info("Reloading application config...")
                    self.init()
                    return self.call(environ, start_response)
            else:
                return self.call(environ, start_response)
        finally:
            # Remove the thread state
            del self._threadStates[threadKey]

    # Overridable WSGI entry point
    def call(self, environ, start_response):

        # Find the matching request app and call it
        request = self._requestUrls.get(environ["PATH_INFO"])
        if request is not None:
            return request(environ, start_response)
        elif self._wrapApplication:
            return self._wrapApplication(environ, start_response)
        else:
            return self.response("404 Not Found", "text/plain", "Not Found")

    # Create a request logger
    def _createLogger(self, logStream):
        logger = logging.getLoggerClass()("")
        if logStream:
            logger.addHandler(logging.StreamHandler(logStream))
        if self._config.logLevel == "Debug":
            logger.setLevel(logging.DEBUG)
        elif self._config.logLevel == "Info":
            logger.setLevel(logging.INFO)
        elif self._config.logLevel == "Error":
            logger.setLevel(logging.ERROR)
        else:
            logger.setLevel(logging.WARNING)
        return logger

    # Retreive the WSGI thread state
    def _getThreadState(self):
        threadKey = threading.current_thread().ident
        return self._threadStates.get(threadKey, self._threadStateDefault)

    # Send an HTTP response
    def response(self, status, contentType, content, headers = None):

        # Ensure proper WSGI response data type
        if not isinstance(content, (list, tuple)):
            content = [content]
        content = [(wsgistr_new(s) if not isinstance(s, wsgistr_) else s) for s in content]

        # Build the headers array
        _headers = [
            ("Content-Type", contentType),
            ("Content-Length", str(sum([len(s) for s in content])))
            ]
        if headers:
            _headers.extend(headers)

        # Return the response
        self._getThreadState().start_response(status, _headers)
        return content

    # Serialize an object to JSON
    def serializeJSON(self, o):
        return json.dumps(o, sort_keys = True, default = jsonDefault,
                          indent = 2 if self._config.prettyOutput else None,
                          separators = (", ", ": ") if self._config.prettyOutput else (",", ":"))

    # Spec parser
    @property
    def specs(self):
        return self._specParser

    # Requests
    @property
    def requests(self):
        return self._requests

    # Resources collection
    @property
    def resources(self):
        return self._resources

    # Application-specific configuration values
    @property
    def config(self):
        return self._config.config

    # WSGI request environ
    @property
    def environ(self):
        return self._getThreadState().environ

    # WSGI request start_response
    @property
    def start_response(self):
        return self._getThreadState().start_response

    # Logger object
    @property
    def log(self):
        return self._getThreadState().log

    # Make an HTTP request on this application
    def request(self, requestMethod, pathInfo, queryString = None, wsgiInput = None, environ = None):

        # WSGI environment - used passed-in environ if its complete
        _environ = dict(environ) if environ else {}
        _environ["REQUEST_METHOD"] = requestMethod
        _environ["PATH_INFO"] = pathInfo
        if "SCRIPT_NAME" not in _environ:
            _environ["SCRIPT_NAME"] = ""
        if "QUERY_STRING" not in _environ:
            _environ["QUERY_STRING"] = queryString if queryString else ""
        if "wsgi.input" not in _environ:
            _environ["wsgi.input"] = StringIO(wsgiInput if wsgiInput else "")
            if "CONTENT_LENGTH" not in _environ:
                _environ["CONTENT_LENGTH"] = str(len(wsgiInput)) if wsgiInput else "0"
        if "wsgi.errors" not in _environ:
            _environ["wsgi.errors"] = StringIO()

        # Make the request
        startResponseArgs = {}
        def startResponse(status, responseHeaders):
            startResponseArgs["status"] = status
            startResponseArgs["responseHeaders"] = responseHeaders
        responseParts = self(_environ, startResponse)
        responseString = wsgistr_str(wsgistr_new("").join(responseParts))

        return (startResponseArgs["status"],
                startResponseArgs["responseHeaders"],
                responseString)

    # Run as stand-alone server
    @classmethod
    def serve(cls, application = None, configPath = None):

        import optparse
        import wsgiref.util
        import wsgiref.simple_server

        # Command line options
        optParser = optparse.OptionParser()
        optParser.add_option("-c", dest = "configPath", metavar = "PATH", default = configPath,
                             help = "Path to configuration file")
        optParser.add_option("-p", dest = "port", type = "int", default = 8080,
                             help = "Server port (default is 8080)")
        optParser.add_option("--config-spec", dest = "configSpec", action = "store_true",
                             help = "Dump configuration file specification")
        (opts, args) = optParser.parse_args()

        # Dump configuration specification
        if opts.configSpec:
            print(cls.configSpec)
            return

        # Create the application instance
        if not application:
            if not opts.configPath:
                optParser.error("Configuration file path required")
            application = cls(os.path.abspath(opts.configPath))
        elif opts.configPath:
            application._configPath = opts.configPath
            application._configString = None
            application.init()

        # Stand-alone server WSGI entry point
        def application_simple_server(environ, start_response):
            wsgiref.util.setup_testing_defaults(environ)
            return application(environ, start_response)

        # Start the stand-alone server
        print("Serving on port %d..." % (opts.port,))
        httpd = wsgiref.simple_server.make_server("", opts.port, application_simple_server)
        httpd.serve_forever()
<commit_msg>Remove wrapped application.<commit_after>#
# Copyright (C) 2012-2013 Craig Hobbs
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

from .compat import basestring_, StringIO, wsgistr_, wsgistr_new, wsgistr_str
from .doc import DocAction
from .model import jsonDefault, Struct
from .request import Request
from .resource.base import ResourceCollection
from .spec import SpecParser

from collections import namedtuple
import imp
import json
import logging
import os
import re
import sys
import threading

# Application configuration file specification
ConfigSpec = """\
# Log level
enum LogLevel
    Debug
    Info
    Warning
    Error

# Resource description
struct Resource

    # Resource name
    string name

    # Resource type
    string type

    # Resource string (e.g. connection string)
    string resourceString

# Application configuration file
struct ApplicationConfig

    # Specification directories (top-level only)
    string[] specPaths

    # Module directories (top-level only)
    string[] modulePaths

    # Resources
    [optional] Resource[] resources

    # Configuration key/value string pairs
    [optional] string{} config

    # Logger output level (default is Warning)
    [optional] LogLevel logLevel

    # Pretty JSON output (default is false)
    [optional] bool prettyOutput

    # Re-load specs and scripts on every requests (default is false)
    [optional] bool alwaysReload

    # Disable documentation (default is false)
    [optional] bool disableDocs

    # External CSS for generated documenation HTML (default is None)
    [optional] string docCssUri
"""

configComment = re.compile("^\s*#.*$", flags = re.MULTILINE)

# Application configuration file model
_configParser = SpecParser()
_configParser.parseString(ConfigSpec)
configModel = _configParser.types["ApplicationConfig"]

# Top-level WSGI application class
class Application(object):

    ENVIRON_APP = "chisel.app"

    ThreadState = namedtuple("ThreadState", "environ, start_response, log")

    def __init__(self, configPath, resourceTypes = None, logStream = sys.stderr):

        self._relPathBase = os.path.dirname(sys.modules[self.__module__].__file__)
        if isinstance(configPath, basestring_):
            self._configPath = configPath if os.path.isabs(configPath) else os.path.join(self._relPathBase, configPath)
            self._configString = None
        else:
            self._configPath = None
            self._configString = configPath.read()
        self._resourceTypes = dict((x.name, x) for x in resourceTypes) if resourceTypes else {}
        self._logStream = logStream
        self._threadStates = {}
        self.init()
        self._initLock = threading.Lock()

    def init(self):

        # Read config file
        if self._configPath:
            with open(self._configPath, "r") as fConfig:
                self._configString = fConfig.read()
        self._config = Struct(configModel.validate(json.loads(configComment.sub("", self._configString))))

        # Load specs
        self._specParser = SpecParser()
        for specPath in self._config.specPaths:
            if not os.path.isabs(specPath):
                specPath = os.path.join(self._relPathBase, specPath)
            self.loadSpecs(specPath)

        # Load modules
        self._requests = {}
        self._requestUrls = {}
        for modulePath in self._config.modulePaths:
            if not os.path.isabs(modulePath):
                modulePath = os.path.join(self._relPathBase, modulePath)
            self.loadModules(modulePath)

        # Add the doc request
        if not self._config.disableDocs:
            self.addRequest(DocAction())

        # Create resources collection
        self._resources = ResourceCollection()
        if self._config.resources:
            for resource in self._config.resources:
                if resource.type not in self._resourceTypes:
                    raise Exception("Unknown resource type '%s'" % (resource.type,))
                self._resources.add(resource.name, self._resourceTypes[resource.type], resource.resourceString)

        # Default thread state
        self._threadStateDefault = self.ThreadState(None, None, self._createLogger(self._logStream))

    # Recursively load all specs in a directory
    def loadSpecs(self, specPath, specExt = ".chsl", finalize = True):

        # Does the path exist?
        if not os.path.isdir(specPath):
            raise IOError("%r not found or is not a directory" % (specPath,))

        # Resursively find spec files
        for dirpath, dirnames, filenames in os.walk(specPath):
            for filename in filenames:
                (base, ext) = os.path.splitext(filename)
                if ext == specExt:
                    self._specParser.parse(os.path.join(dirpath, filename), finalize = False)
        if finalize:
            self._specParser.finalize()

    # Load a spec string
    def loadSpecString(self, spec, fileName = "", finalize = True):
        self._specParser.parseString(spec, fileName = fileName, finalize = finalize)

    # Add a request handler (Request-wrapped WSGI application)
    def addRequest(self, request):

        # Wrap bare functions in a request decorator
        if not isinstance(request, Request):
            request = Request(request)

        # Duplicate request name?
        if request.name in self._requests:
            raise Exception("Redefinition of request '%s'" % (request.name,))
        self._requests[request.name] = request

        # Add the request URLs
        for url in request.urls:
            if url in self._requestUrls:
                raise Exception("Redefinition of request URL '%s'" % (url,))
            self._requestUrls[url] = request

        # Make the request app-aware at load-time
        request.onload(self)

    # Recursively load all modules in a directory
    def loadModules(self, moduleDir, moduleExt = ".py"):

        # Does the path exist?
        if not os.path.isdir(moduleDir):
            raise IOError("%r not found or is not a directory" % (moduleDir,))

        # Recursively find module files
        moduleDirNorm = os.path.normpath(moduleDir)
        modulePathParent = os.path.dirname(moduleDirNorm)
        modulePathBase = os.path.join(modulePathParent, "") if modulePathParent else modulePathParent
        for dirpath, dirnames, filenames in os.walk(moduleDirNorm):
            for filename in filenames:
                (base, ext) = os.path.splitext(filename)
                if ext == moduleExt:

                    # Load the module
                    module = None
                    moduleParts = []
                    for modulePart in os.path.join(dirpath, base)[len(modulePathBase):].split(os.sep):
                        moduleParts.append(modulePart)
                        moduleFp, modulePath, moduleDesc = \
                            imp.find_module(modulePart, module.__path__ if module else [modulePathParent])
                        try:
                            module = imp.load_module(".".join(moduleParts), moduleFp, modulePath, moduleDesc)
                        finally:
                            if moduleFp:
                                moduleFp.close()

                    # Add the module's requests
                    for moduleAttr in dir(module):
                        request = getattr(module, moduleAttr)
                        if isinstance(request, Request):
                            self.addRequest(request)

    # WSGI entry point
    def __call__(self, environ, start_response):

        # Set the app environment item
        environ[self.ENVIRON_APP] = self

        # Add the thread state
        threadKey = threading.current_thread().ident
        threadState = self.ThreadState(environ, start_response, self._createLogger(environ["wsgi.errors"]))
        self._threadStates[threadKey] = threadState

        # Handle the request
        try:
            # Reload, if requested
            if self._config.alwaysReload:
                with self._initLock:
                    self.log.info("Reloading application config...")
                    self.init()
                    return self.call(environ, start_response)
            else:
                return self.call(environ, start_response)
        finally:
            # Remove the thread state
            del self._threadStates[threadKey]

    # Overridable WSGI entry point
    def call(self, environ, start_response):

        # Find the matching request app and call it
        request = self._requestUrls.get(environ["PATH_INFO"])
        if request is not None:
            return request(environ, start_response)
        else:
            return self.response("404 Not Found", "text/plain", "Not Found")

    # Create a request logger
    def _createLogger(self, logStream):
        logger = logging.getLoggerClass()("")
        if logStream:
            logger.addHandler(logging.StreamHandler(logStream))
        if self._config.logLevel == "Debug":
            logger.setLevel(logging.DEBUG)
        elif self._config.logLevel == "Info":
            logger.setLevel(logging.INFO)
        elif self._config.logLevel == "Error":
            logger.setLevel(logging.ERROR)
        else:
            logger.setLevel(logging.WARNING)
        return logger

    # Retreive the WSGI thread state
    def _getThreadState(self):
        threadKey = threading.current_thread().ident
        return self._threadStates.get(threadKey, self._threadStateDefault)

    # Send an HTTP response
    def response(self, status, contentType, content, headers = None):

        # Ensure proper WSGI response data type
        if not isinstance(content, (list, tuple)):
            content = [content]
        content = [(wsgistr_new(s) if not isinstance(s, wsgistr_) else s) for s in content]

        # Build the headers array
        _headers = [
            ("Content-Type", contentType),
            ("Content-Length", str(sum([len(s) for s in content])))
            ]
        if headers:
            _headers.extend(headers)

        # Return the response
        self._getThreadState().start_response(status, _headers)
        return content

    # Serialize an object to JSON
    def serializeJSON(self, o):
        return json.dumps(o, sort_keys = True, default = jsonDefault,
                          indent = 2 if self._config.prettyOutput else None,
                          separators = (", ", ": ") if self._config.prettyOutput else (",", ":"))

    # Spec parser
    @property
    def specs(self):
        return self._specParser

    # Requests
    @property
    def requests(self):
        return self._requests

    # Resources collection
    @property
    def resources(self):
        return self._resources

    # Application-specific configuration values
    @property
    def config(self):
        return self._config.config

    # WSGI request environ
    @property
    def environ(self):
        return self._getThreadState().environ

    # WSGI request start_response
    @property
    def start_response(self):
        return self._getThreadState().start_response

    # Logger object
    @property
    def log(self):
        return self._getThreadState().log

    # Make an HTTP request on this application
    def request(self, requestMethod, pathInfo, queryString = None, wsgiInput = None, environ = None):

        # WSGI environment - used passed-in environ if its complete
        _environ = dict(environ) if environ else {}
        _environ["REQUEST_METHOD"] = requestMethod
        _environ["PATH_INFO"] = pathInfo
        if "SCRIPT_NAME" not in _environ:
            _environ["SCRIPT_NAME"] = ""
        if "QUERY_STRING" not in _environ:
            _environ["QUERY_STRING"] = queryString if queryString else ""
        if "wsgi.input" not in _environ:
            _environ["wsgi.input"] = StringIO(wsgiInput if wsgiInput else "")
            if "CONTENT_LENGTH" not in _environ:
                _environ["CONTENT_LENGTH"] = str(len(wsgiInput)) if wsgiInput else "0"
        if "wsgi.errors" not in _environ:
            _environ["wsgi.errors"] = StringIO()

        # Make the request
        startResponseArgs = {}
        def startResponse(status, responseHeaders):
            startResponseArgs["status"] = status
            startResponseArgs["responseHeaders"] = responseHeaders
        responseParts = self(_environ, startResponse)
        responseString = wsgistr_str(wsgistr_new("").join(responseParts))

        return (startResponseArgs["status"],
                startResponseArgs["responseHeaders"],
                responseString)

    # Run as stand-alone server
    @classmethod
    def serve(cls, application = None, configPath = None):

        import optparse
        import wsgiref.util
        import wsgiref.simple_server

        # Command line options
        optParser = optparse.OptionParser()
        optParser.add_option("-c", dest = "configPath", metavar = "PATH", default = configPath,
                             help = "Path to configuration file")
        optParser.add_option("-p", dest = "port", type = "int", default = 8080,
                             help = "Server port (default is 8080)")
        optParser.add_option("--config-spec", dest = "configSpec", action = "store_true",
                             help = "Dump configuration file specification")
        (opts, args) = optParser.parse_args()

        # Dump configuration specification
        if opts.configSpec:
            print(cls.configSpec)
            return

        # Create the application instance
        if not application:
            if not opts.configPath:
                optParser.error("Configuration file path required")
            application = cls(os.path.abspath(opts.configPath))
        elif opts.configPath:
            application._configPath = opts.configPath
            application._configString = None
            application.init()

        # Stand-alone server WSGI entry point
        def application_simple_server(environ, start_response):
            wsgiref.util.setup_testing_defaults(environ)
            return application(environ, start_response)

        # Start the stand-alone server
        print("Serving on port %d..." % (opts.port,))
        httpd = wsgiref.simple_server.make_server("", opts.port, application_simple_server)
        httpd.serve_forever()
<|endoftext|>


<commit_before>#!/usr/bin/python2

# Copyright 2015 Charles University in Prague
# Copyright 2015 Vojtech Horky
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyzotero import zotero
from pprint import pprint
from ConfigParser import SafeConfigParser
import sys
import os
import argparse
import pickle

# http://stackoverflow.com/a/518232/1360886
def strip_accents(s):
    import unicodedata
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if unicodedata.category(c) != 'Mn')

def latex_escape(s):
    return s.replace(u'\u2013', '--').replace(u'&', '\\&')

def parse_date_guessing(datestr):
    import datetime
    for fmt in [ "%B %d %Y", "%B %d, %Y", "%B %Y", "%Y", "%Y-%m-%d" ]:
        try:
            return datetime.datetime.strptime(datestr, fmt)
        except ValueError as e:
            pass
    return datetime.datetime.today()

def skip_useless_words(where):
    useless_words = 'a the on'.split()
    idx = 0
    while where[idx].lower() in useless_words:
        idx = idx + 1
    return where[idx]

def get_first_author(item):
    author = item['data']['creators'][0]
    if 'lastName' in author:
        return author['lastName']
    else:
        return author['name']

def make_bibtex_key(item):
    if 'extra' in item['data']:
        import re
        lines = item['data']['extra'].split('\n')
        pat = re.compile('bibtex:[ \t]*(.*)')
        for l in lines:
            m = pat.match(l)
            if m:
                return m.group(1)
    author = filter(unicode.isalpha, strip_accents(get_first_author(item)).lower())
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = filter(unicode.isalnum, skip_useless_words(title_words).lower())
    return "%s_%s_%s" % (author, title_start, year)

def make_sort_key(item):
    if item['data']['itemType'] == 'attachment':
        return 'xxx'
    author = strip_accents(get_first_author(item)).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s %s %s" % (year, author, title_start)

def item_to_bibtex(item):
    def shall_skip(item):
        if item['data']['itemType'] == 'attachment':
            return True
        else:
            return False

    def bib_type(item):
        if item['data']['itemType'] == 'journalArticle':
            return 'article'
        elif item['data']['itemType'] == 'conferencePaper':
            return 'inproceedings'
        elif item['data']['itemType'] == 'bookSection':
            return 'incollection'
        else:
            return 'misc'

    def make_author_list(creators, creator_type = None):
        # By default, we try to collect only authors.
        # If there is no author explicitly specify, we take everybody
        # (probably not the best thing as different types has different
        # names for creators, e.g. presenter but it is better than to
        # mix authors with editors for book chapters).
        if creator_type is None:
            result = make_author_list(creators, 'author')
            if result != '':
                return result
        names = []
        for c in creators:
            if (not creator_type is None) and ('creatorType' in c) and (c['creatorType'] != creator_type):
                continue
            if 'lastName' in c:
                names.append('%s, %s' % (c['lastName'], c['firstName']))
            else:
                names.append('{%s}' % (c['name']))
        return ' and '.join(names)

    def print_key(key, value, print_empty = True):
        if (not print_empty) and (value == ''):
            return
        print('    %s = {%s},' % (key, value.encode('utf-8')))

    def has_field(zoterokey, item):
        return zoterokey in item['data'] and item['data'][zoterokey] != ''

    def try_field(bibtexkey, zoterokeys, item, escape=True, protect=False, conversion=None):
        if not type(zoterokeys) is list:
            zoterokeys = [ zoterokeys ]
        for key in zoterokeys:
            if (key in item['data']) and (item['data'][key] != ''):
                value = item['data'][key]
                if not conversion is None:
                    value = conversion(value)
                if escape:
                    value = latex_escape(value)
                if protect:
                    value = "{%s}" % value
                print_key(bibtexkey, value)
                # Exit after first match
                return

    def get_doi(item):
        if ('DOI' in item['data']) and (item['data']['DOI'] != ''):
            return item['data']['DOI']
        else:
            import re
            lines = item['data']['extra'].split('\n')
            pat = re.compile('[dD][oO][iI]:[ \t]*(.*)')
            for l in lines:
                m = pat.match(l)
                if m:
                    return m.group(1)
            return ''

    if shall_skip(item):
        return

    print('@%s{%s,' % (bib_type(item), make_bibtex_key(item)))

    try_field('title', 'title', item, protect=True)
    print_key('author', make_author_list(item['data']['creators']))

    print_key('year', '%d' % parse_date_guessing(item['data']['date']).year)

    # Not so traditional types are distinguished by howpublished field for now
    if item['data']['itemType'] in [ 'blogPost', 'webpage', 'computerProgram' ]:
        try_field('howpublished', 'url', item, conversion=lambda x : '\\url{%s}' % x)
    if item['data']['itemType'] == 'presentation':
        if has_field('meetingName', item):
            s = 'Presentation at {%s}' % item['data']['meetingName']
            if has_field('url', item):
                s = '%s, \\url{%s}' % (s, item['data']['url'])
            print_key('howpublished', s)

    try_field('booktitle', [ 'proceedingsTitle', 'bookTitle' ], item, protect=True)
    try_field('journal', 'publicationTitle', item, protect=True)
    print_key('editor', make_author_list(item['data']['creators'], 'editor'), False)
    try_field('publisher', 'publisher', item)
    try_field('series', 'series', item, protect=True)
    try_field('number', [ 'seriesNumber', 'issue' ], item)

    try_field('location', 'place', item)

    item_doi = get_doi(item)
    if item_doi != '':
        print_key('doi', item_doi)
    try_field('isbn', 'ISBN', item)
    try_field('issn', 'ISSN', item)

    try_field('pages', 'pages', item, conversion=lambda x : x.replace('-', '--'))

    try_field('url', 'url', item)
    try_field('volume', 'volume', item)
    try_field('shorttitle', 'shortTitle', item)
    try_field('abstract', 'abstractNote', item)

    print('}\n')

class MyConfigParser(SafeConfigParser):
    def __init__(self):
        SafeConfigParser.__init__(self)

    def get_with_default(self, section, option, default_value=None):
        if self.has_option(section, option):
            return self.get(section, option)
        else:
            return default_value

cfgfile = MyConfigParser()
cfgfile.read(os.path.expanduser('~/.config/cli-zotero.conf'))

parser = argparse.ArgumentParser(description='Command-line client for Zotero')

parser.add_argument('--key',
        dest='key',
        required=not cfgfile.has_option('core', 'key'),
        default=cfgfile.get_with_default('core', 'key'),
        metavar='API-KEY',
        help='Zotero API key (https://www.zotero.org/settings/keys)\nOr specify in [core] of configuration file.')

identity_opts = parser.add_mutually_exclusive_group(required=True)
identity_opts.add_argument('--group',
        dest='group',
        metavar='ID',
        type=int,
        help='Group ID (https://www.zotero.org/groups/)')
identity_opts.add_argument('--user',
        dest='user',
        metavar='ID',
        type=int,
        help='User ID (https://www.zotero.org/settings/keys)')
identity_opts.add_argument('--id',
        dest='identity',
        metavar='NAME',
        help='Identity specified in [identities] in configuration file.')

action_args = parser.add_mutually_exclusive_group(required=True)
action_args.add_argument('--list-collections',
        dest='collection_filter',
        nargs='?',
        const='',
        metavar='TITLE',
        help='List your collections (title partial match)')
action_args.add_argument('--collection-to-bibtex',
        dest='collection_to_bibtex',
        metavar='COLLECTION-ID',
        help='Export given collection to BibTeX')

parser.add_argument('--dump',
        dest='dump_file',
        metavar='FILENAME',
        help='Dump retrieved data through pprint to FILENAME')

parser.add_argument('--limit',
        dest='limit',
        type=int,
        default=30,
        metavar='N',
        help='Set internal limit of the queries')

cfg = parser.parse_args()

zot = None
if cfg.group:
    zot = zotero.Zotero(cfg.group, 'group', cfg.key)
elif cfg.user:
    zot = zotero.Zotero(cfg.user, 'user', cfg.key)
else:
    id_line = cfgfile.get_with_default('identities', cfg.identity)
    if id_line is None:
        sys.exit('Unknown identity "%s".' % cfg.identity)
    id_parts = id_line.split()
    if len(id_parts) != 2 or (id_parts[0] not in ['user', 'group' ]):
        sys.exit('Wrong identity configuration "%s".' % id_line)
    zot = zotero.Zotero(id_parts[1], id_parts[0], cfg.key)

if not cfg.collection_filter is None:
    all_collections = zot.collections(q=cfg.collection_filter, limit=cfg.limit)
    if cfg.dump_file:
        with open(cfg.dump_file, "w") as f:
            pprint(all_collections, f)
    for col in all_collections:
        print("%s - %s" % (col['key'], col['data']['name']))
    sys.exit()

if cfg.collection_to_bibtex:
    items = []
    start_index = 0
    while True:
        next_items = zot.collection_items(cfg.collection_to_bibtex, limit=cfg.limit, start=start_index)
        if len(next_items) == 0:
            break
        items.extend(next_items)
        start_index = start_index + len(next_items)
    if cfg.dump_file:
        with open(cfg.dump_file, "w") as f:
            pprint(items, f)
    items = sorted(items, key=make_sort_key)
    for item in items:
        item_to_bibtex(item)
    sys.exit()

parser.print_help()
<commit_msg>Handle `thesis' type<commit_after>#!/usr/bin/python2

# Copyright 2015 Charles University in Prague
# Copyright 2015 Vojtech Horky
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyzotero import zotero
from pprint import pprint
from ConfigParser import SafeConfigParser
import sys
import os
import argparse
import pickle

# http://stackoverflow.com/a/518232/1360886
def strip_accents(s):
    import unicodedata
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if unicodedata.category(c) != 'Mn')

def latex_escape(s):
    return s.replace(u'\u2013', '--').replace(u'&', '\\&')

def parse_date_guessing(datestr):
    import datetime
    for fmt in [ "%B %d %Y", "%B %d, %Y", "%B %Y", "%Y", "%Y-%m-%d" ]:
        try:
            return datetime.datetime.strptime(datestr, fmt)
        except ValueError as e:
            pass
    return datetime.datetime.today()

def skip_useless_words(where):
    useless_words = 'a the on'.split()
    idx = 0
    while where[idx].lower() in useless_words:
        idx = idx + 1
    return where[idx]

def get_first_author(item):
    author = item['data']['creators'][0]
    if 'lastName' in author:
        return author['lastName']
    else:
        return author['name']

def make_bibtex_key(item):
    if 'extra' in item['data']:
        import re
        lines = item['data']['extra'].split('\n')
        pat = re.compile('bibtex:[ \t]*(.*)')
        for l in lines:
            m = pat.match(l)
            if m:
                return m.group(1)
    author = filter(unicode.isalpha, strip_accents(get_first_author(item)).lower())
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = filter(unicode.isalnum, skip_useless_words(title_words).lower())
    return "%s_%s_%s" % (author, title_start, year)

def make_sort_key(item):
    if item['data']['itemType'] == 'attachment':
        return 'xxx'
    author = strip_accents(get_first_author(item)).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s %s %s" % (year, author, title_start)

def item_to_bibtex(item):
    def shall_skip(item):
        if item['data']['itemType'] == 'attachment':
            return True
        else:
            return False

    def bib_type(item):
        if item['data']['itemType'] == 'journalArticle':
            return 'article'
        elif item['data']['itemType'] == 'conferencePaper':
            return 'inproceedings'
        elif item['data']['itemType'] == 'bookSection':
            return 'incollection'
        elif item['data']['itemType'] == 'thesis':
            return 'mastersthesis'
        else:
            return 'misc'

    def make_author_list(creators, creator_type = None):
        # By default, we try to collect only authors.
        # If there is no author explicitly specify, we take everybody
        # (probably not the best thing as different types has different
        # names for creators, e.g. presenter but it is better than to
        # mix authors with editors for book chapters).
        if creator_type is None:
            result = make_author_list(creators, 'author')
            if result != '':
                return result
        names = []
        for c in creators:
            if (not creator_type is None) and ('creatorType' in c) and (c['creatorType'] != creator_type):
                continue
            if 'lastName' in c:
                names.append('%s, %s' % (c['lastName'], c['firstName']))
            else:
                names.append('{%s}' % (c['name']))
        return ' and '.join(names)

    def print_key(key, value, print_empty = True):
        if (not print_empty) and (value == ''):
            return
        print('    %s = {%s},' % (key, value.encode('utf-8')))

    def has_field(zoterokey, item):
        return zoterokey in item['data'] and item['data'][zoterokey] != ''

    def try_field(bibtexkey, zoterokeys, item, escape=True, protect=False, conversion=None):
        if not type(zoterokeys) is list:
            zoterokeys = [ zoterokeys ]
        for key in zoterokeys:
            if (key in item['data']) and (item['data'][key] != ''):
                value = item['data'][key]
                if not conversion is None:
                    value = conversion(value)
                if escape:
                    value = latex_escape(value)
                if protect:
                    value = "{%s}" % value
                print_key(bibtexkey, value)
                # Exit after first match
                return

    def get_doi(item):
        if ('DOI' in item['data']) and (item['data']['DOI'] != ''):
            return item['data']['DOI']
        else:
            import re
            lines = item['data']['extra'].split('\n')
            pat = re.compile('[dD][oO][iI]:[ \t]*(.*)')
            for l in lines:
                m = pat.match(l)
                if m:
                    return m.group(1)
            return ''

    if shall_skip(item):
        return

    print('@%s{%s,' % (bib_type(item), make_bibtex_key(item)))

    try_field('title', 'title', item, protect=True)
    print_key('author', make_author_list(item['data']['creators']))

    print_key('year', '%d' % parse_date_guessing(item['data']['date']).year)

    # Not so traditional types are distinguished by howpublished field for now
    if item['data']['itemType'] in [ 'blogPost', 'webpage', 'computerProgram' ]:
        try_field('howpublished', 'url', item, conversion=lambda x : '\\url{%s}' % x)
    if item['data']['itemType'] == 'presentation':
        if has_field('meetingName', item):
            s = 'Presentation at {%s}' % item['data']['meetingName']
            if has_field('url', item):
                s = '%s, \\url{%s}' % (s, item['data']['url'])
            print_key('howpublished', s)

    try_field('booktitle', [ 'proceedingsTitle', 'bookTitle' ], item, protect=True)
    try_field('journal', 'publicationTitle', item, protect=True)
    print_key('editor', make_author_list(item['data']['creators'], 'editor'), False)
    try_field('publisher', 'publisher', item)
    try_field('series', 'series', item, protect=True)
    try_field('number', [ 'seriesNumber', 'issue' ], item)

    try_field('type', 'thesisType', item)
    try_field('school', 'university', item)
    if item['data']['itemType'] == 'thesis':
        try_field('address', 'place', item)
    else:
        try_field('location', 'place', item)

    item_doi = get_doi(item)
    if item_doi != '':
        print_key('doi', item_doi)
    try_field('isbn', 'ISBN', item)
    try_field('issn', 'ISSN', item)

    try_field('pages', 'pages', item, conversion=lambda x : x.replace('-', '--'))

    try_field('url', 'url', item)
    try_field('volume', 'volume', item)
    try_field('shorttitle', 'shortTitle', item)
    try_field('abstract', 'abstractNote', item)

    print('}\n')

class MyConfigParser(SafeConfigParser):
    def __init__(self):
        SafeConfigParser.__init__(self)

    def get_with_default(self, section, option, default_value=None):
        if self.has_option(section, option):
            return self.get(section, option)
        else:
            return default_value

cfgfile = MyConfigParser()
cfgfile.read(os.path.expanduser('~/.config/cli-zotero.conf'))

parser = argparse.ArgumentParser(description='Command-line client for Zotero')

parser.add_argument('--key',
        dest='key',
        required=not cfgfile.has_option('core', 'key'),
        default=cfgfile.get_with_default('core', 'key'),
        metavar='API-KEY',
        help='Zotero API key (https://www.zotero.org/settings/keys)\nOr specify in [core] of configuration file.')

identity_opts = parser.add_mutually_exclusive_group(required=True)
identity_opts.add_argument('--group',
        dest='group',
        metavar='ID',
        type=int,
        help='Group ID (https://www.zotero.org/groups/)')
identity_opts.add_argument('--user',
        dest='user',
        metavar='ID',
        type=int,
        help='User ID (https://www.zotero.org/settings/keys)')
identity_opts.add_argument('--id',
        dest='identity',
        metavar='NAME',
        help='Identity specified in [identities] in configuration file.')

action_args = parser.add_mutually_exclusive_group(required=True)
action_args.add_argument('--list-collections',
        dest='collection_filter',
        nargs='?',
        const='',
        metavar='TITLE',
        help='List your collections (title partial match)')
action_args.add_argument('--collection-to-bibtex',
        dest='collection_to_bibtex',
        metavar='COLLECTION-ID',
        help='Export given collection to BibTeX')

parser.add_argument('--dump',
        dest='dump_file',
        metavar='FILENAME',
        help='Dump retrieved data through pprint to FILENAME')

parser.add_argument('--limit',
        dest='limit',
        type=int,
        default=30,
        metavar='N',
        help='Set internal limit of the queries')

cfg = parser.parse_args()

zot = None
if cfg.group:
    zot = zotero.Zotero(cfg.group, 'group', cfg.key)
elif cfg.user:
    zot = zotero.Zotero(cfg.user, 'user', cfg.key)
else:
    id_line = cfgfile.get_with_default('identities', cfg.identity)
    if id_line is None:
        sys.exit('Unknown identity "%s".' % cfg.identity)
    id_parts = id_line.split()
    if len(id_parts) != 2 or (id_parts[0] not in ['user', 'group' ]):
        sys.exit('Wrong identity configuration "%s".' % id_line)
    zot = zotero.Zotero(id_parts[1], id_parts[0], cfg.key)

if not cfg.collection_filter is None:
    all_collections = zot.collections(q=cfg.collection_filter, limit=cfg.limit)
    if cfg.dump_file:
        with open(cfg.dump_file, "w") as f:
            pprint(all_collections, f)
    for col in all_collections:
        print("%s - %s" % (col['key'], col['data']['name']))
    sys.exit()

if cfg.collection_to_bibtex:
    items = []
    start_index = 0
    while True:
        next_items = zot.collection_items(cfg.collection_to_bibtex, limit=cfg.limit, start=start_index)
        if len(next_items) == 0:
            break
        items.extend(next_items)
        start_index = start_index + len(next_items)
    if cfg.dump_file:
        with open(cfg.dump_file, "w") as f:
            pprint(items, f)
    items = sorted(items, key=make_sort_key)
    for item in items:
        item_to_bibtex(item)
    sys.exit()

parser.print_help()
<|endoftext|>


<commit_before>#!/usr/bin/python2

# Copyright 2015 Charles University in Prague
# Copyright 2015 Vojtech Horky
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyzotero import zotero
from pprint import pprint
import sys
import argparse
import pickle

# http://stackoverflow.com/a/518232/1360886
def strip_accents(s):
    import unicodedata
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if unicodedata.category(c) != 'Mn')

def latex_escape(s):
    return s.replace(u'\u2013', '--')

def parse_date_guessing(datestr):
    import datetime
    for fmt in [ "%B %d %Y", "%B %d, %Y", "%B %Y", "%Y" ]:
        try:
            return datetime.datetime.strptime(datestr, fmt)
        except ValueError as e:
            pass
    return datetime.datetime.today()

def skip_useless_words(where):
    useless_words = 'a the on'.split()
    idx = 0
    while where[idx].lower() in useless_words:
        idx = idx + 1
    return where[idx]

def make_bibtex_key(item):
    author = strip_accents(item['data']['creators'][0]['lastName']).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s_%s_%s" % (author, title_start, year)

def make_sort_key(item):
    if item['data']['itemType'] == 'attachment':
        return 'xxx'
    author = strip_accents(item['data']['creators'][0]['lastName']).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s %s %s" % (year, author, title_start)

def item_to_bibtex(item):
    def shall_skip(item):
        if item['data']['itemType'] == 'attachment':
            return True
        else:
            return False

    def bib_type(item):
        if item['data']['itemType'] == 'journalArticle':
            return 'article'
        elif item['data']['itemType'] == 'conferencePaper':
            return 'inproceedings'
        else:
            return 'misc'

    def make_author_list(creators):
        names = []
        for c in creators:
            if c['creatorType'] == 'author':
                names.append('%s, %s' % (c['lastName'], c['firstName']))
        return ' and '.join(names)

    def print_key(key, value):
        print('  %s = {%s},' % (key, value.encode('utf-8')))

    def try_field(bibtexkey, zoterokey, item, escape=True, protect=False, conversion=None):
        if zoterokey in item['data']:
            if item['data'][zoterokey] != '':
                value = item['data'][zoterokey]
                if not conversion is None:
                    value = conversion(value)
                if escape:
                    value = latex_escape(value)
                if protect:
                    value = "{%s}" % value
                print_key(bibtexkey, value)

    if shall_skip(item):
        return

    print('@%s{%s,' % (bib_type(item), make_bibtex_key(item)))

    try_field('title', 'title', item, protect=True)
    print_key('author', make_author_list(item['data']['creators']))

    try_field('booktitle', 'proceedingsTitle', item)
    try_field('doi', 'DOI', item)
    try_field('isbn', 'ISBN', item)
    try_field('issn', 'ISSN', item)
    try_field('journaltitle', 'publicationTitle', item)
    try_field('location', 'place', item)
    try_field('number', 'issue', item)
    try_field('pages', 'pages', item, conversion=lambda x : x.replace('-', '--'))
    try_field('publisher', 'publisher', item)
    try_field('series', 'series', item)
    try_field('url', 'url', item)
    try_field('volume', 'volume', item)

    print('}\n')

parser = argparse.ArgumentParser(description='Command-line client for Zotero')
parser.add_argument('--key',
        dest='key',
        metavar='API-KEY',
        required=True,
        help='Zotero API key (https://www.zotero.org/settings/keys)')
group_or_user = parser.add_mutually_exclusive_group(required=True)
group_or_user.add_argument('--group',
        dest='group',
        metavar='ID',
        type=int,
        help='Group ID (https://www.zotero.org/groups/)')
group_or_user.add_argument('--user',
        dest='user',
        metavar='ID',
        type=int,
        help='User ID (https://www.zotero.org/settings/keys)')
parser.add_argument('--limit',
        dest='limit',
        type=int,
        default=30,
        metavar='N',
        help='Set internal limit of the queries')
parser.add_argument('--list-collections',
        dest='collection_filter',
        nargs='?',
        const='',
        metavar='TITLE',
        help='List your collections (title partial match)')
parser.add_argument('--collection-to-bibtex',
        dest='collection_to_bibtex',
        metavar='COLLECTION-ID',
        help='Export given collection to BibTeX')

cfg = parser.parse_args()

zot = None
if cfg.group is None:
    zot = zotero.Zotero(cfg.user, 'user', cfg.key)
else:
    zot = zotero.Zotero(cfg.group, 'group', cfg.key)

if not cfg.collection_filter is None:
    all_collections = zot.collections(q=cfg.collection_filter, limit=cfg.limit)
    for col in all_collections:
        print("%s - %s" % (col['key'], col['data']['name']))
    sys.exit()

if not cfg.collection_to_bibtex is None:
    items = []
    start_index = 0
    while True:
        next_items = zot.collection_items(cfg.collection_to_bibtex, limit=cfg.limit, start=start_index)
        if len(next_items) == 0:
            break
        items.extend(next_items)
        start_index = start_index + len(next_items)
    items = sorted(items, key=make_sort_key)
    for item in items:
        item_to_bibtex(item)
    sys.exit()

parser.print_help()
<commit_msg>More fields and escaping<commit_after>#!/usr/bin/python2

# Copyright 2015 Charles University in Prague
# Copyright 2015 Vojtech Horky
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pyzotero import zotero
from pprint import pprint
import sys
import argparse
import pickle

# http://stackoverflow.com/a/518232/1360886
def strip_accents(s):
    import unicodedata
    return ''.join(c for c in unicodedata.normalize('NFD', s)
                   if unicodedata.category(c) != 'Mn')

def latex_escape(s):
    return s.replace(u'\u2013', '--')

def parse_date_guessing(datestr):
    import datetime
    for fmt in [ "%B %d %Y", "%B %d, %Y", "%B %Y", "%Y" ]:
        try:
            return datetime.datetime.strptime(datestr, fmt)
        except ValueError as e:
            pass
    return datetime.datetime.today()

def skip_useless_words(where):
    useless_words = 'a the on'.split()
    idx = 0
    while where[idx].lower() in useless_words:
        idx = idx + 1
    return where[idx]

def make_bibtex_key(item):
    author = strip_accents(item['data']['creators'][0]['lastName']).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s_%s_%s" % (author, title_start, year)

def make_sort_key(item):
    if item['data']['itemType'] == 'attachment':
        return 'xxx'
    author = strip_accents(item['data']['creators'][0]['lastName']).lower()
    year = parse_date_guessing(item['data']['date']).year
    title_words = strip_accents(item['data']['title']).split()
    title_start = skip_useless_words(title_words).lower()
    return "%s %s %s" % (year, author, title_start)

def item_to_bibtex(item):
    def shall_skip(item):
        if item['data']['itemType'] == 'attachment':
            return True
        else:
            return False

    def bib_type(item):
        if item['data']['itemType'] == 'journalArticle':
            return 'article'
        elif item['data']['itemType'] == 'conferencePaper':
            return 'inproceedings'
        else:
            return 'misc'

    def make_author_list(creators):
        names = []
        for c in creators:
            if c['creatorType'] == 'author':
                names.append('%s, %s' % (c['lastName'], c['firstName']))
        return ' and '.join(names)

    def print_key(key, value):
        print('  %s = {%s},' % (key, value.encode('utf-8')))

    def try_field(bibtexkey, zoterokey, item, escape=True, protect=False, conversion=None):
        if zoterokey in item['data']:
            if item['data'][zoterokey] != '':
                value = item['data'][zoterokey]
                if not conversion is None:
                    value = conversion(value)
                if escape:
                    value = latex_escape(value)
                if protect:
                    value = "{%s}" % value
                print_key(bibtexkey, value)

    if shall_skip(item):
        return

    print('@%s{%s,' % (bib_type(item), make_bibtex_key(item)))

    try_field('title', 'title', item, protect=True)
    print_key('author', make_author_list(item['data']['creators']))

    print_key('year', '%d' % parse_date_guessing(item['data']['date']).year)
    try_field('booktitle', 'proceedingsTitle', item, protect=True)
    try_field('doi', 'DOI', item)
    try_field('isbn', 'ISBN', item)
    try_field('issn', 'ISSN', item)
    try_field('journaltitle', 'publicationTitle', item, protect=True)
    try_field('location', 'place', item)
    try_field('number', 'issue', item)
    try_field('pages', 'pages', item, conversion=lambda x : x.replace('-', '--'))
    try_field('publisher', 'publisher', item)
    try_field('series', 'series', item, protect=True)
    try_field('url', 'url', item)
    try_field('volume', 'volume', item)
    try_field('shorttitle', 'shortTitle', item)
    try_field('abstract', 'abstractNote', item)

    print('}\n')

parser = argparse.ArgumentParser(description='Command-line client for Zotero')
parser.add_argument('--key',
        dest='key',
        metavar='API-KEY',
        required=True,
        help='Zotero API key (https://www.zotero.org/settings/keys)')
group_or_user = parser.add_mutually_exclusive_group(required=True)
group_or_user.add_argument('--group',
        dest='group',
        metavar='ID',
        type=int,
        help='Group ID (https://www.zotero.org/groups/)')
group_or_user.add_argument('--user',
        dest='user',
        metavar='ID',
        type=int,
        help='User ID (https://www.zotero.org/settings/keys)')
parser.add_argument('--limit',
        dest='limit',
        type=int,
        default=30,
        metavar='N',
        help='Set internal limit of the queries')
parser.add_argument('--list-collections',
        dest='collection_filter',
        nargs='?',
        const='',
        metavar='TITLE',
        help='List your collections (title partial match)')
parser.add_argument('--collection-to-bibtex',
        dest='collection_to_bibtex',
        metavar='COLLECTION-ID',
        help='Export given collection to BibTeX')

cfg = parser.parse_args()

zot = None
if cfg.group is None:
    zot = zotero.Zotero(cfg.user, 'user', cfg.key)
else:
    zot = zotero.Zotero(cfg.group, 'group', cfg.key)

if not cfg.collection_filter is None:
    all_collections = zot.collections(q=cfg.collection_filter, limit=cfg.limit)
    for col in all_collections:
        print("%s - %s" % (col['key'], col['data']['name']))
    sys.exit()

if not cfg.collection_to_bibtex is None:
    items = []
    start_index = 0
    while True:
        next_items = zot.collection_items(cfg.collection_to_bibtex, limit=cfg.limit, start=start_index)
        if len(next_items) == 0:
            break
        items.extend(next_items)
        start_index = start_index + len(next_items)
    items = sorted(items, key=make_sort_key)
    for item in items:
        item_to_bibtex(item)
    sys.exit()

parser.print_help()
<|endoftext|>