<commit_before>from django.conf import settings
from django.utils.functional import cached_property
import os
from django.contrib.auth.models import User
from django.core.files.storage import default_storage
from petitions.utils import generate_unique_upload_filename
from rest_framework import serializers
from petitions.models import Petition, Media, PetitionSign
from rest_framework.fields import empty

IMAGES_UPLOAD_DIRECTORY = 'uploadedImages'

class UserSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = User
        fields = ('url', 'username')

class PetitionSignSerializer(serializers.HyperlinkedModelSerializer):
    author = serializers.ReadOnlyField(source='author.username')
    #petition = serializers.ReadOnlyField(source='petiton.id')
    class Meta:
        model = PetitionSign
        fields = ['petition', 'comment', 'anonymous', 'author']

class PetitionSerializer(serializers.HyperlinkedModelSerializer):
    signs = PetitionSignSerializer(many=True, read_only=True)
    class Meta:
        model = Petition
        fields = ('url', 'title', 'text', 'deadline', 'responsible', 'signs')

class UserSerializerDetail(UserSerializer):
    petitions = PetitionSerializer(many=True)
    class Meta(UserSerializer.Meta):
        fields = UserSerializer.Meta.fields + ('petitions',)

class MediaSerializer(serializers.ModelSerializer):
    id = serializers.IntegerField(read_only=False, required=False)
    class Meta:
        model = Media
        fields = ('id', 'mediaUrl', 'type')

class PetitionSerializerDetail(PetitionSerializer):
    author = UserSerializer(read_only=True)
    media = MediaSerializer(many=True)
    class Meta(PetitionSerializer.Meta):
        fields = PetitionSerializer.Meta.fields + ('author', 'media')

    def create(self, validated_data):
        media_data = validated_data.pop('media')
        petition = Petition.objects.create(**validated_data)
        for media_item in media_data:
            if 'id' in media_item:
                raise serializers.ValidationError("Don't add new media with id field")
            Media.objects.create(petition=petition, **media_item)
        return petition

    def update(self, instance, validated_data):
        if 'media' in validated_data: # for PATCH request
            media_data = validated_data.pop('media')
            updated_media_ids = set(obj['id'] for obj in media_data if 'id' in obj)
            existing_media_ids = set(obj.id for obj in instance.media.all())

            new_media = updated_media_ids.difference(existing_media_ids)
            if len( updated_media_ids.intersection(new_media) ) != 0:
                raise serializers.ValidationError("Don't add new media with id field")

            # update existing items and add new
            for media_item_data in media_data:
                if 'id' in media_item_data:
                    media_instance = Media.objects.get(pk=media_item_data['id'])
                    existing_media_ids.remove(media_item_data['id'])
                else:
                    media_instance = Media()
                media_instance.petition = instance
                for attr, value in media_item_data.items():
                    setattr(media_instance, attr, value)
                media_instance.save()

            # remove deleted items
            deleted_items = existing_media_ids
            for deleted_item_id in deleted_items:
                Media.objects.filter(pk=deleted_item_id).delete()

        return super().update(instance, validated_data)

class ImageUploadSerializer(serializers.Serializer):
    image = serializers.ImageField()

    def create(self, validated_data):
        image_file = validated_data["image"]
        path = generate_unique_upload_filename(IMAGES_UPLOAD_DIRECTORY, 'jpg')
        default_storage.save(os.path.join(settings.MEDIA_ROOT, path), image_file)
        return path
<commit_msg>Added readonly field to PetitionSerializer for representing petitions' status<commit_after>from django.conf import settings
from django.utils.functional import cached_property
import os
from django.contrib.auth.models import User
from django.core.files.storage import default_storage
from petitions.utils import generate_unique_upload_filename
from rest_framework import serializers
from petitions.models import Petition, Media, PetitionSign
from rest_framework.fields import empty

IMAGES_UPLOAD_DIRECTORY = 'uploadedImages'

class UserSerializer(serializers.HyperlinkedModelSerializer):
    class Meta:
        model = User
        fields = ('url', 'username')

class PetitionSignSerializer(serializers.HyperlinkedModelSerializer):
    author = serializers.ReadOnlyField(source='author.username')
    #petition = serializers.ReadOnlyField(source='petiton.id')
    class Meta:
        model = PetitionSign
        fields = ['petition', 'comment', 'anonymous', 'author']

class PetitionSerializer(serializers.HyperlinkedModelSerializer):
    signs = PetitionSignSerializer(many=True, read_only=True)
    status = serializers.ReadOnlyField()
    class Meta:
        model = Petition
        fields = ('url', 'title', 'text', 'deadline', 'responsible', 'signs', 'status')

class UserSerializerDetail(UserSerializer):
    petitions = PetitionSerializer(many=True)
    class Meta(UserSerializer.Meta):
        fields = UserSerializer.Meta.fields + ('petitions',)

class MediaSerializer(serializers.ModelSerializer):
    id = serializers.IntegerField(read_only=False, required=False)
    class Meta:
        model = Media
        fields = ('id', 'mediaUrl', 'type')

class PetitionSerializerDetail(PetitionSerializer):
    author = UserSerializer(read_only=True)
    media = MediaSerializer(many=True)
    class Meta(PetitionSerializer.Meta):
        fields = PetitionSerializer.Meta.fields + ('author', 'media')

    def create(self, validated_data):
        media_data = validated_data.pop('media')
        petition = Petition.objects.create(**validated_data)
        for media_item in media_data:
            if 'id' in media_item:
                raise serializers.ValidationError("Don't add new media with id field")
            Media.objects.create(petition=petition, **media_item)
        return petition

    def update(self, instance, validated_data):
        if 'media' in validated_data: # for PATCH request
            media_data = validated_data.pop('media')
            updated_media_ids = set(obj['id'] for obj in media_data if 'id' in obj)
            existing_media_ids = set(obj.id for obj in instance.media.all())

            new_media = updated_media_ids.difference(existing_media_ids)
            if len( updated_media_ids.intersection(new_media) ) != 0:
                raise serializers.ValidationError("Don't add new media with id field")

            # update existing items and add new
            for media_item_data in media_data:
                if 'id' in media_item_data:
                    media_instance = Media.objects.get(pk=media_item_data['id'])
                    existing_media_ids.remove(media_item_data['id'])
                else:
                    media_instance = Media()
                media_instance.petition = instance
                for attr, value in media_item_data.items():
                    setattr(media_instance, attr, value)
                media_instance.save()

            # remove deleted items
            deleted_items = existing_media_ids
            for deleted_item_id in deleted_items:
                Media.objects.filter(pk=deleted_item_id).delete()

        return super().update(instance, validated_data)

class ImageUploadSerializer(serializers.Serializer):
    image = serializers.ImageField()

    def create(self, validated_data):
        image_file = validated_data["image"]
        path = generate_unique_upload_filename(IMAGES_UPLOAD_DIRECTORY, 'jpg')
        default_storage.save(os.path.join(settings.MEDIA_ROOT, path), image_file)
        return path
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

default_app_config = 'vault.apps.VaultConfig'

__version__ = '1.0.6'
<commit_msg>Vault version 1.0.7<commit_after># -*- coding: utf-8 -*-

default_app_config = 'vault.apps.VaultConfig'

__version__ = '1.0.7'
<|endoftext|>


<commit_before># Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import numpy as np

def get_fc2(supercell,
            primitive,
            displacements,
            forces,
            atom_list=None,
            options=None,
            log_level=0):
    p2s_map = primitive.p2s_map
    is_compact_fc = (atom_list is not None and
                     (atom_list == p2s_map).all())
    fc2 = run_alm(supercell,
                  primitive,
                  displacements,
                  forces,
                  1,
                  is_compact_fc=is_compact_fc,
                  options=options,
                  log_level=log_level)[0]

    if not is_compact_fc and atom_list is not None:
        fc2 = np.array(fc2[atom_list], dtype='double', order='C')

    return fc2

def run_alm(supercell,
            primitive,
            displacements,
            forces,
            maxorder,
            is_compact_fc,
            options=None,
            log_level=0):
    fcs = None  # This is returned.

    lattice = supercell.cell
    positions = supercell.scaled_positions
    numbers = supercell.numbers
    natom = len(numbers)
    p2s_map = primitive.p2s_map
    p2p_map = primitive.p2p_map

    alm_options = _update_options(options)
    num_elems = len(np.unique(numbers))
    if log_level:
        print("---------------------------------"
              " ALM start "
              "--------------------------------")
        print("ALM is a non-trivial force constants calculator. "
              "Please cite the paper:")
        print("T. Tadano and S. Tsuneyuki, "
              "J. Phys. Soc. Jpn. 87, 041015 (2018).")
        print("ALM is developed at https://github.com/ttadano/ALM by T. "
              "Tadano.")
    if log_level == 1:
        print("Increase log-level to watch detailed ALM log.")

    shape = (maxorder, num_elems, num_elems)
    cutoff_radii = -np.ones(shape, dtype='double')
    if alm_options['cutoff'] is not None:
        if len(alm_options['cutoff']) == 1:
            cutoff_radii[:] = alm_options['cutoff'][0]
        elif np.prod(shape) == len(alm_options['cutoff']):
            cutoff_radii[:] = np.reshape(alm_options['cutoff'], shape)
        else:
            raise RuntimeError("Cutoff is not properly set.")

    _disps, _forces, df_msg = _slice_displacements_and_forces(
        displacements,
        forces,
        alm_options['ndata'],
        alm_options['nstart'],
        alm_options['nend'])

    if log_level > 1:
        print("")
        print("  ndata: %d" % len(displacements))
        for key, val in alm_options.items():
            if val is not None:
                print("  %s: %s" % (key, val))
        print("")
        print(" " + "-" * 67)

    if log_level > 0:
        log_level_alm = log_level - 1
    else:
        log_level_alm = 0

    try:
        from alm import ALM, optimizer_control_data_types
    except ImportError:
        raise ImportError("ALM python module was not found.")

    with ALM(lattice, positions, numbers) as alm:
        if log_level > 0:
            if alm_options['cutoff'] is not None:
                for i in range(maxorder):
                    if maxorder > 1:
                        print("fc%d" % (i + 2))
                    print(("cutoff" + " %6s" * num_elems)
                          % tuple(alm.kind_names.values()))
                    for r, kn in zip(cutoff_radii[i], alm.kind_names.values()):
                        print(("   %-3s" + " %6.2f" * num_elems)
                              % ((kn, ) + tuple(r)))
            if df_msg is not None:
                print(df_msg)
        if log_level > 1:
            print("")
        sys.stdout.flush()

        alm.output_filename_prefix = alm_options['output_filename_prefix']
        alm.verbosity = log_level_alm

        alm.define(
            maxorder,
            cutoff_radii=cutoff_radii,
            nbody=alm_options['nbody'],
            symmetrization_basis=alm_options['symmetrization_basis'])
        alm.displacements = _disps
        alm.forces = _forces

        # Mainly for elastic net (or lasso) regression
        optcontrol = {}
        for key in optimizer_control_data_types:
            if key in alm_options:
                optcontrol[key] = alm_options[key]
        if optcontrol:
            alm.optimizer_control = optcontrol
            if ('cross_validation' in optcontrol and
                optcontrol['cross_validation'] > 0):
                alm.optimize(solver=alm_options['solver'])
                optcontrol['cross_validation'] = 0
                optcontrol['l1_alpha'] = alm.cv_l1_alpha
                alm.optimizer_control = optcontrol

        alm.optimize(solver=alm_options['solver'])

        fcs = _extract_fc_from_alm(alm,
                                   natom,
                                   maxorder,
                                   is_compact_fc,
                                   p2s_map=p2s_map,
                                   p2p_map=p2p_map)

    if log_level:
        print("----------------------------------"
              " ALM end "
              "---------------------------------")

    return fcs

def _update_options(fc_calculator_options):
    """Set ALM options with appropriate data types

    fc_calculator_options : str
        This string should be written such as follows:

            "solver = dense, cutoff = 5"

        This string is parsed as collection of settings that are separated by
        comma ','. Each setting has the format of 'option = value'. The value
        is cast to have its appropriate data type for ALM in this method.

    """

    try:
        from alm import optimizer_control_data_types
    except ImportError:
        raise ImportError("ALM python module was not found.")

    # Default settings.
    alm_options = {'solver': 'SimplicialLDLT',
                   'ndata': None,
                   'nstart': None,
                   'nend': None,
                   'nbody': None,
                   'cutoff': None,
                   'symmetrization_basis': 'Lattice',
                   'output_filename_prefix': None}

    if fc_calculator_options is not None:
        alm_option_types = {'cutoff': np.double,
                            'ndata': int,
                            'nstart': int,
                            'nend': int,
                            'nbody': np.intc,
                            'output_filename_prefix': str,
                            'solver': str,
                            'symmetrization_basis': str}
        alm_option_types.update(optimizer_control_data_types)
        for option_str in fc_calculator_options.split(","):
            key, val = [x.strip() for x in option_str.split('=')[:2]]
            if key.lower() in alm_option_types:
                if alm_option_types[key.lower()] is np.double:
                    option_value = np.array(
                        [float(x) for x in val.split()], dtype='double')
                elif alm_option_types[key.lower()] is np.intc:
                    option_value = np.array(
                        [int(x) for x in val.split()], dtype='intc')
                else:
                    option_value = alm_option_types[key.lower()](val)
                alm_options[key] = option_value
    return alm_options

def _slice_displacements_and_forces(d, f, ndata, nstart, nend):
    msg = None
    if ndata is not None:
        _d = d[:ndata]
        _f = f[:ndata]
        msg = "Number of displacement supercells: %d" % ndata
    elif nstart is not None and nend is not None:
        _d = d[nstart - 1:nend]
        _f = f[nstart - 1:nend]
        msg = "Supercell index range: %d - %d" % (nstart, nend)
    else:
        return d, f, None

    return (np.array(_d, dtype='double', order='C'),
            np.array(_f, dtype='double', order='C'), msg)

def _extract_fc_from_alm(alm,
                         natom,
                         maxorder,
                         is_compact_fc,
                         p2s_map=None,
                         p2p_map=None):
    # Harmonic: order=1, 3rd: order=2, ...
    fcs = []
    for order in range(1, maxorder + 1):
        fc = None
        p2s_map_alm = alm.getmap_primitive_to_supercell()[0]
        if (is_compact_fc and
            len(p2s_map_alm) == len(p2s_map) and
            (p2s_map_alm == p2s_map).all()):
            fc_shape = (
                (len(p2s_map), ) + (natom, ) * order + (3, ) * (order + 1))
            fc = np.zeros(fc_shape, dtype='double', order='C')
            for fc_elem, indices in zip(
                    *alm.get_fc(order, mode='origin', permutation=1)):
                v = indices // 3
                c = indices % 3
                selection = (p2p_map[v[0]], ) + tuple(v[1:]) + tuple(c)
                fc[selection] = fc_elem

        if fc is None:
            if is_compact_fc:
                atom_list = p2s_map
            else:
                atom_list = np.arange(natom, dtype=int)
            fc_shape = (
                (len(atom_list), ) + (natom, ) * order + (3, ) * (order + 1))
            fc = np.zeros(fc_shape, dtype='double', order='C')
            for fc_elem, indices in zip(
                    *alm.get_fc(order, mode='all', permutation=1)):
                v = indices // 3
                idx = np.where(atom_list == v[0])[0]
                if len(idx) > 0:
                    c = indices % 3
                    selection = (idx[0], ) + tuple(v[1:]) + tuple(c)
                    fc[selection] = fc_elem

        fcs.append(fc)

    return fcs
<commit_msg>ALM default solver is changed to dense from SimplicialLDLT.<commit_after># Copyright (C) 2018 Atsushi Togo
# All rights reserved.
#
# This file is part of phonopy.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
#   notice, this list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright
#   notice, this list of conditions and the following disclaimer in
#   the documentation and/or other materials provided with the
#   distribution.
#
# * Neither the name of the phonopy project nor the names of its
#   contributors may be used to endorse or promote products derived
#   from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

import sys
import numpy as np

def get_fc2(supercell,
            primitive,
            displacements,
            forces,
            atom_list=None,
            options=None,
            log_level=0):
    p2s_map = primitive.p2s_map
    is_compact_fc = (atom_list is not None and
                     (atom_list == p2s_map).all())
    fc2 = run_alm(supercell,
                  primitive,
                  displacements,
                  forces,
                  1,
                  is_compact_fc=is_compact_fc,
                  options=options,
                  log_level=log_level)[0]

    if not is_compact_fc and atom_list is not None:
        fc2 = np.array(fc2[atom_list], dtype='double', order='C')

    return fc2

def run_alm(supercell,
            primitive,
            displacements,
            forces,
            maxorder,
            is_compact_fc,
            options=None,
            log_level=0):
    fcs = None  # This is returned.

    lattice = supercell.cell
    positions = supercell.scaled_positions
    numbers = supercell.numbers
    natom = len(numbers)
    p2s_map = primitive.p2s_map
    p2p_map = primitive.p2p_map

    alm_options = _update_options(options)
    num_elems = len(np.unique(numbers))
    if log_level:
        print("---------------------------------"
              " ALM start "
              "--------------------------------")
        print("ALM is a non-trivial force constants calculator. "
              "Please cite the paper:")
        print("T. Tadano and S. Tsuneyuki, "
              "J. Phys. Soc. Jpn. 87, 041015 (2018).")
        print("ALM is developed at https://github.com/ttadano/ALM by T. "
              "Tadano.")
    if log_level == 1:
        print("Increase log-level to watch detailed ALM log.")

    shape = (maxorder, num_elems, num_elems)
    cutoff_radii = -np.ones(shape, dtype='double')
    if alm_options['cutoff'] is not None:
        if len(alm_options['cutoff']) == 1:
            cutoff_radii[:] = alm_options['cutoff'][0]
        elif np.prod(shape) == len(alm_options['cutoff']):
            cutoff_radii[:] = np.reshape(alm_options['cutoff'], shape)
        else:
            raise RuntimeError("Cutoff is not properly set.")

    _disps, _forces, df_msg = _slice_displacements_and_forces(
        displacements,
        forces,
        alm_options['ndata'],
        alm_options['nstart'],
        alm_options['nend'])

    if log_level > 1:
        print("")
        print("  ndata: %d" % len(displacements))
        for key, val in alm_options.items():
            if val is not None:
                print("  %s: %s" % (key, val))
        print("")
        print(" " + "-" * 67)

    if log_level > 0:
        log_level_alm = log_level - 1
    else:
        log_level_alm = 0

    try:
        from alm import ALM, optimizer_control_data_types
    except ImportError:
        raise ImportError("ALM python module was not found.")

    with ALM(lattice, positions, numbers) as alm:
        if log_level > 0:
            if alm_options['cutoff'] is not None:
                for i in range(maxorder):
                    if maxorder > 1:
                        print("fc%d" % (i + 2))
                    print(("cutoff" + " %6s" * num_elems)
                          % tuple(alm.kind_names.values()))
                    for r, kn in zip(cutoff_radii[i], alm.kind_names.values()):
                        print(("   %-3s" + " %6.2f" * num_elems)
                              % ((kn, ) + tuple(r)))
            if df_msg is not None:
                print(df_msg)
        if log_level > 1:
            print("")
        sys.stdout.flush()

        alm.output_filename_prefix = alm_options['output_filename_prefix']
        alm.verbosity = log_level_alm

        alm.define(
            maxorder,
            cutoff_radii=cutoff_radii,
            nbody=alm_options['nbody'],
            symmetrization_basis=alm_options['symmetrization_basis'])
        alm.displacements = _disps
        alm.forces = _forces

        # Mainly for elastic net (or lasso) regression
        optcontrol = {}
        for key in optimizer_control_data_types:
            if key in alm_options:
                optcontrol[key] = alm_options[key]
        if optcontrol:
            alm.optimizer_control = optcontrol
            if ('cross_validation' in optcontrol and
                optcontrol['cross_validation'] > 0):
                alm.optimize(solver=alm_options['solver'])
                optcontrol['cross_validation'] = 0
                optcontrol['l1_alpha'] = alm.cv_l1_alpha
                alm.optimizer_control = optcontrol

        alm.optimize(solver=alm_options['solver'])

        fcs = _extract_fc_from_alm(alm,
                                   natom,
                                   maxorder,
                                   is_compact_fc,
                                   p2s_map=p2s_map,
                                   p2p_map=p2p_map)

    if log_level:
        print("----------------------------------"
              " ALM end "
              "---------------------------------")

    return fcs

def _update_options(fc_calculator_options):
    """Set ALM options with appropriate data types

    fc_calculator_options : str
        This string should be written such as follows:

            "solver = dense, cutoff = 5"

        This string is parsed as collection of settings that are separated by
        comma ','. Each setting has the format of 'option = value'. The value
        is cast to have its appropriate data type for ALM in this method.

    """

    try:
        from alm import optimizer_control_data_types
    except ImportError:
        raise ImportError("ALM python module was not found.")

    # Default settings.
    alm_options = {'solver': 'dense',
                   'ndata': None,
                   'nstart': None,
                   'nend': None,
                   'nbody': None,
                   'cutoff': None,
                   'symmetrization_basis': 'Lattice',
                   'output_filename_prefix': None}

    if fc_calculator_options is not None:
        alm_option_types = {'cutoff': np.double,
                            'ndata': int,
                            'nstart': int,
                            'nend': int,
                            'nbody': np.intc,
                            'output_filename_prefix': str,
                            'solver': str,
                            'symmetrization_basis': str}
        alm_option_types.update(optimizer_control_data_types)
        for option_str in fc_calculator_options.split(","):
            key, val = [x.strip() for x in option_str.split('=')[:2]]
            if key.lower() in alm_option_types:
                if alm_option_types[key.lower()] is np.double:
                    option_value = np.array(
                        [float(x) for x in val.split()], dtype='double')
                elif alm_option_types[key.lower()] is np.intc:
                    option_value = np.array(
                        [int(x) for x in val.split()], dtype='intc')
                else:
                    option_value = alm_option_types[key.lower()](val)
                alm_options[key] = option_value
    return alm_options

def _slice_displacements_and_forces(d, f, ndata, nstart, nend):
    msg = None
    if ndata is not None:
        _d = d[:ndata]
        _f = f[:ndata]
        msg = "Number of displacement supercells: %d" % ndata
    elif nstart is not None and nend is not None:
        _d = d[nstart - 1:nend]
        _f = f[nstart - 1:nend]
        msg = "Supercell index range: %d - %d" % (nstart, nend)
    else:
        return d, f, None

    return (np.array(_d, dtype='double', order='C'),
            np.array(_f, dtype='double', order='C'), msg)

def _extract_fc_from_alm(alm,
                         natom,
                         maxorder,
                         is_compact_fc,
                         p2s_map=None,
                         p2p_map=None):
    # Harmonic: order=1, 3rd: order=2, ...
    fcs = []
    for order in range(1, maxorder + 1):
        fc = None
        p2s_map_alm = alm.getmap_primitive_to_supercell()[0]
        if (is_compact_fc and
            len(p2s_map_alm) == len(p2s_map) and
            (p2s_map_alm == p2s_map).all()):
            fc_shape = (
                (len(p2s_map), ) + (natom, ) * order + (3, ) * (order + 1))
            fc = np.zeros(fc_shape, dtype='double', order='C')
            for fc_elem, indices in zip(
                    *alm.get_fc(order, mode='origin', permutation=1)):
                v = indices // 3
                c = indices % 3
                selection = (p2p_map[v[0]], ) + tuple(v[1:]) + tuple(c)
                fc[selection] = fc_elem

        if fc is None:
            if is_compact_fc:
                atom_list = p2s_map
            else:
                atom_list = np.arange(natom, dtype=int)
            fc_shape = (
                (len(atom_list), ) + (natom, ) * order + (3, ) * (order + 1))
            fc = np.zeros(fc_shape, dtype='double', order='C')
            for fc_elem, indices in zip(
                    *alm.get_fc(order, mode='all', permutation=1)):
                v = indices // 3
                idx = np.where(atom_list == v[0])[0]
                if len(idx) > 0:
                    c = indices % 3
                    selection = (idx[0], ) + tuple(v[1:]) + tuple(c)
                    fc[selection] = fc_elem

        fcs.append(fc)

    return fcs
<|endoftext|>


<commit_before>import sqlite3
from chi_leg_scraper import ChicagoLegistar

uri = 'http://chicago.legistar.com/Legislation.aspx'
scraper = ChicagoLegistar(uri)

conn = sqlite3.connect("/home/ec2-user/legistar-scrape/chicago_legislation.db")
c = conn.cursor()

c.execute('select date(max(intro_date)) from legislation')
last_date = c.fetchone()[0]

legislation = scraper.searchLegislation('')
# Remove the final date field
[legs.pop(4) for legs in legislation]

c.executemany('INSERT OR IGNORE INTO legislation '
             '(id, type, status, intro_date, main_sponsor, title, url) '
              'VALUES '
              '(?, ?, ?, ?, ?, ?, ?)',
              legislation)

conn.commit()
conn.close()
<commit_msg>introduced last date into page slurps<commit_after>import sqlite3
from chi_leg_scraper import ChicagoLegistar

uri = 'http://chicago.legistar.com/Legislation.aspx'
scraper = ChicagoLegistar(uri)

conn = sqlite3.connect("/home/ec2-user/legistar-scrape/chicago_legislation.db")
c = conn.cursor()

c.execute('select date(max(intro_date)) from legislation')
last_date = c.fetchone()[0]

legislation = scraper.searchLegislation('', last_date)
# Remove the final date field
[legs.pop(4) for legs in legislation]

c.executemany('INSERT OR IGNORE INTO legislation '
             '(id, type, status, intro_date, main_sponsor, title, url) '
              'VALUES '
              '(?, ?, ?, ?, ?, ?, ?)',
              legislation)

conn.commit()
conn.close()
<|endoftext|>


<commit_before># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, print_function
from builtins import zip
from builtins import map
from builtins import range
import os
import random
import unittest

# Python 3 compatibility imports
from six.moves import xrange

from toil.lib.bioio import getTempFile
from toil.job import Job
from toil.test import ToilTest, slow
import time
import sys
from threading import Thread, Event
import logging
logger = logging.getLogger( __name__ )
from unittest import skipIf
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
from toil.leader import FailedJobsException, DeadlockException

class JobServiceTest(ToilTest):
    """
    Tests testing the Job.Service class
    """

    @slow
    def testServiceSerialization(self):
        """
        Tests that a service can receive a promise without producing a serialization
        error.
        """
        job = Job()
        service = TestServiceSerialization("woot")
        startValue = job.addService(service) # Add a first service to job
        subService = TestServiceSerialization(startValue) # Now create a child of
        # that service that takes the start value promise from the parent service
        job.addService(subService, parentService=service) # This should work if
        # serialization on services is working correctly.

        self.runToil(job)

    @slow
    def testService(self, checkpoint=False):
        """
        Tests the creation of a Job.Service with random failures of the worker.
        """
        for test in range(2):
            outFile = getTempFile(rootDir=self._createTempDir()) # Temporary file
            messageInt = random.randint(1, sys.maxsize)
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTest, outFile, messageInt, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEquals(int(open(outFile, 'r').readline()), messageInt)
            finally:
                os.remove(outFile)

    @slow
    def testServiceDeadlock(self):
        """
        Creates a job with more services than maxServices, checks that deadlock is detected.
        """
        outFile = getTempFile(rootDir=self._createTempDir())
        try:
            def makeWorkflow():
                job = Job()
                r1 = job.addService(TestServiceSerialization("woot1"))
                r2 = job.addService(TestServiceSerialization("woot2"))
                r3 = job.addService(TestServiceSerialization("woot3"))
                job.addChildFn(fnTest, [ r1, r2, r3 ], outFile)
                return job

            # This should fail as too few services available
            try:
                self.runToil(makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5)
            except DeadlockException:
                print("Got expected deadlock exception")
            else:
                assert 0

            # This should pass, as adequate services available
            self.runToil(makeWorkflow(), maxServiceJobs=3)
            # Check we get expected output
            assert open(outFile, 'r').read() == "woot1 woot2 woot3"
        finally:
            os.remove(outFile)

    def testServiceWithCheckpoints(self):
        """
        Tests the creation of a Job.Service with random failures of the worker, making the root job use checkpointing to
        restart the subtree.
        """
        self.testService(checkpoint=True)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating a chain of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFile = getTempFile(rootDir=self._createTempDir())
            messages = [ random.randint(1, sys.maxsize) for i in range(3) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestRecursive, outFile, messages, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEquals(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceParallelRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating parallel chains of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFiles = [ getTempFile(rootDir=self._createTempDir()) for j in range(2) ]
            messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestParallelRecursive, outFiles, messageBundles, checkpoint=True)

                # Run the workflow repeatedly until success
                self.runToil(t, retryCount=2)

                # Check output
                for (messages, outFile) in zip(messageBundles, outFiles):
                    self.assertEquals(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                list(map(os.remove, outFiles))

    def runToil(self, rootJob, retryCount=1, badWorker=0.5, badWorkedFailInterval=0.05, maxServiceJobs=sys.maxsize, deadlockWait=60):
        # Create the runner for the workflow.
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = "DEBUG"

        options.retryCount = retryCount
        options.badWorker = badWorker
        options.badWorkerFailInterval = badWorkedFailInterval
        options.servicePollingInterval = 1
        options.maxServiceJobs = maxServiceJobs
        options.deadlockWait=deadlockWait

        # Run the workflow
        totalTrys = 0
        while True:
            try:
                Job.Runner.startToil(rootJob, options)
                break
            except FailedJobsException as e:
                i = e.numberOfFailedJobs
                if totalTrys > 40: #p(fail after this many restarts) = 0.5**32
                    self.fail() #Exceeded a reasonable number of restarts
                totalTrys += 1
                options.restart = True

def serviceTest(job, outFile, messageInt):
    """
    Creates one service and one accessing job, which communicate with two files to establish
    that both run concurrently.
    """
    #Clean out out-file
    open(outFile, 'w').close()
    randInt = random.randint(1, sys.maxsize) # We create a random number that is added to messageInt and subtracted by the serviceAccessor, to prove that
    # when service test is checkpointed and restarted there is never a connection made between an earlier service and later serviceAccessor, or vice versa.
    job.addChildJobFn(serviceAccessor, job.addService(TestService(messageInt + randInt)), outFile, randInt)

def serviceTestRecursive(job, outFile, messages):
    """
    Creates a chain of services and accessing jobs, each paired together.
    """
    if len(messages) > 0:
        #Clean out out-file
        open(outFile, 'w').close()
        randInt = random.randint(1, sys.maxsize)
        service = TestService(messages[0] + randInt)
        child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

        for i in range(1, len(messages)):
            randInt = random.randint(1, sys.maxsize)
            service2 = TestService(messages[i] + randInt, cores=0.1)
            child = child.addChildJobFn(serviceAccessor,
                                        job.addService(service2, parentService=service),
                                        outFile, randInt, cores=0.1)
            service = service2

def serviceTestParallelRecursive(job, outFiles, messageBundles):
    """
    Creates multiple chains of services and accessing jobs.
    """
    for messages, outFile in zip(messageBundles, outFiles):
        #Clean out out-file
        open(outFile, 'w').close()
        if len(messages) > 0:
            randInt = random.randint(1, sys.maxsize)
            service = TestService(messages[0] + randInt)
            child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

            for i in range(1, len(messages)):
                randInt = random.randint(1, sys.maxsize)
                service2 = TestService(messages[i] + randInt, cores=0.1)
                child = child.addChildJobFn(serviceAccessor,
                                            job.addService(service2, parentService=service),
                                            outFile, randInt, cores=0.1)
                service = service2

class TestService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        While established the service repeatedly:
             - reads an integer i from the inJobStoreFileID file
             - writes i and messageInt to the outJobStoreFileID file
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        assert self.disk is not None
        assert self.memory is not None
        assert self.cores is not None
        self.terminate = Event()
        self.error = Event()
        inJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        outJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        self.serviceThread = Thread(target=self.serviceWorker,
                                    args=(job.fileStore.jobStore, self.terminate, self.error,
                                          inJobStoreID, outJobStoreID,
                                          self.messageInt))
        self.serviceThread.start()
        return (inJobStoreID, outJobStoreID)

    def stop(self, job):
        self.terminate.set()
        self.serviceThread.join()

    def check(self):
        if self.error.isSet():
            raise RuntimeError("Service worker failed")
        return True

    @staticmethod
    def serviceWorker(jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt):
        try:
            while True:
                if terminate.isSet(): # Quit if we've got the terminate signal
                    logger.debug("Demo service worker being told to quit")
                    return

                time.sleep(0.2) # Sleep to avoid thrashing

                # Try reading a line from the input file
                try:
                    with jobStore.readFileStream(inJobStoreID) as fH:
                        line = fH.readline()
                except:
                    logger.debug("Something went wrong reading a line")
                    raise

                # Try converting the input line into an integer
                try:
                    inputInt = int(line)
                except ValueError:
                    logger.debug("Tried casting input line to integer but got error: %s" % line)
                    continue

                # Write out the resulting read integer and the message
                with jobStore.updateFileStream(outJobStoreID) as fH:
                    fH.write("%s %s\n" % (inputInt, messageInt))
        except:
            error.set()
            raise

def serviceAccessor(job, communicationFiles, outFile, randInt):
    """
    Writes a random integer iinto the inJobStoreFileID file, then tries 10 times reading
    from outJobStoreFileID to get a pair of integers, the first equal to i the second written into the outputFile.
    """
    inJobStoreFileID, outJobStoreFileID = communicationFiles

    # Get a random integer
    key = random.randint(1, sys.maxsize)

    # Write the integer into the file
    logger.debug("Writing key to inJobStoreFileID")
    with job.fileStore.jobStore.updateFileStream(inJobStoreFileID) as fH:
        fH.write("%s\n" % key)

    logger.debug("Trying to read key and message from outJobStoreFileID")
    for i in range(10): # Try 10 times over
        time.sleep(0.2) #Avoid thrashing

        # Try reading an integer from the input file and writing out the message
        with job.fileStore.jobStore.readFileStream(outJobStoreFileID) as fH:
            line = fH.readline()

        tokens = line.split()
        if len(tokens) != 2:
            continue

        key2, message = tokens

        if int(key2) == key:
            logger.debug("Matched key's: %s, writing message: %s with randInt: %s" % (key, int(message) - randInt, randInt))
            with open(outFile, 'a') as fH:
                fH.write("%s\n" % (int(message) - randInt))
            return

    assert 0 # Job failed to get info from the service

class TestServiceSerialization(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        Trivial service for testing serialization.
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        return self.messageInt

    def stop(self, job):
        pass

    def check(self):
        return True

def fnTest(strings, outputFile):
    """
    Function concatenates the strings together and writes them to the output file
    """
    with open(outputFile, 'w') as fH:
        fH.write(" ".join(strings))

<commit_msg>skip testServiceDeadlock without 4+ cores<commit_after># Copyright (C) 2015-2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, print_function
from builtins import zip
from builtins import map
from builtins import range
import os
import random
import unittest

# Python 3 compatibility imports
from six.moves import xrange

from toil.lib.bioio import getTempFile
from toil.job import Job
from toil.test import ToilTest, slow
import time
import sys
from threading import Thread, Event
import logging
logger = logging.getLogger( __name__ )
from unittest import skipIf
from toil.batchSystems.singleMachine import SingleMachineBatchSystem
from toil.leader import FailedJobsException, DeadlockException

class JobServiceTest(ToilTest):
    """
    Tests testing the Job.Service class
    """

    @slow
    def testServiceSerialization(self):
        """
        Tests that a service can receive a promise without producing a serialization
        error.
        """
        job = Job()
        service = TestServiceSerialization("woot")
        startValue = job.addService(service) # Add a first service to job
        subService = TestServiceSerialization(startValue) # Now create a child of
        # that service that takes the start value promise from the parent service
        job.addService(subService, parentService=service) # This should work if
        # serialization on services is working correctly.

        self.runToil(job)

    @slow
    def testService(self, checkpoint=False):
        """
        Tests the creation of a Job.Service with random failures of the worker.
        """
        for test in range(2):
            outFile = getTempFile(rootDir=self._createTempDir()) # Temporary file
            messageInt = random.randint(1, sys.maxsize)
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTest, outFile, messageInt, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEquals(int(open(outFile, 'r').readline()), messageInt)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceDeadlock(self):
        """
        Creates a job with more services than maxServices, checks that deadlock is detected.
        """
        outFile = getTempFile(rootDir=self._createTempDir())
        try:
            def makeWorkflow():
                job = Job()
                r1 = job.addService(TestServiceSerialization("woot1"))
                r2 = job.addService(TestServiceSerialization("woot2"))
                r3 = job.addService(TestServiceSerialization("woot3"))
                job.addChildFn(fnTest, [ r1, r2, r3 ], outFile)
                return job

            # This should fail as too few services available
            try:
                self.runToil(makeWorkflow(), badWorker=0.0, maxServiceJobs=2, deadlockWait=5)
            except DeadlockException:
                print("Got expected deadlock exception")
            else:
                assert 0

            # This should pass, as adequate services available
            self.runToil(makeWorkflow(), maxServiceJobs=3)
            # Check we get expected output
            assert open(outFile, 'r').read() == "woot1 woot2 woot3"
        finally:
            os.remove(outFile)

    def testServiceWithCheckpoints(self):
        """
        Tests the creation of a Job.Service with random failures of the worker, making the root job use checkpointing to
        restart the subtree.
        """
        self.testService(checkpoint=True)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating a chain of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFile = getTempFile(rootDir=self._createTempDir())
            messages = [ random.randint(1, sys.maxsize) for i in range(3) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestRecursive, outFile, messages, checkpoint=checkpoint)

                # Run the workflow repeatedly until success
                self.runToil(t)

                # Check output
                self.assertEquals(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                os.remove(outFile)

    @slow
    @skipIf(SingleMachineBatchSystem.numCores < 4, 'Need at least four cores to run this test')
    def testServiceParallelRecursive(self, checkpoint=True):
        """
        Tests the creation of a Job.Service, creating parallel chains of services and accessing jobs.
        Randomly fails the worker.
        """
        for test in range(1):
            # Temporary file
            outFiles = [ getTempFile(rootDir=self._createTempDir()) for j in range(2) ]
            messageBundles = [ [ random.randint(1, sys.maxsize) for i in range(3) ] for j in range(2) ]
            try:
                # Wire up the services/jobs
                t = Job.wrapJobFn(serviceTestParallelRecursive, outFiles, messageBundles, checkpoint=True)

                # Run the workflow repeatedly until success
                self.runToil(t, retryCount=2)

                # Check output
                for (messages, outFile) in zip(messageBundles, outFiles):
                    self.assertEquals(list(map(int, open(outFile, 'r').readlines())), messages)
            finally:
                list(map(os.remove, outFiles))

    def runToil(self, rootJob, retryCount=1, badWorker=0.5, badWorkedFailInterval=0.05, maxServiceJobs=sys.maxsize, deadlockWait=60):
        # Create the runner for the workflow.
        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
        options.logLevel = "DEBUG"

        options.retryCount = retryCount
        options.badWorker = badWorker
        options.badWorkerFailInterval = badWorkedFailInterval
        options.servicePollingInterval = 1
        options.maxServiceJobs = maxServiceJobs
        options.deadlockWait=deadlockWait

        # Run the workflow
        totalTrys = 0
        while True:
            try:
                Job.Runner.startToil(rootJob, options)
                break
            except FailedJobsException as e:
                i = e.numberOfFailedJobs
                if totalTrys > 40: #p(fail after this many restarts) = 0.5**32
                    self.fail() #Exceeded a reasonable number of restarts
                totalTrys += 1
                options.restart = True

def serviceTest(job, outFile, messageInt):
    """
    Creates one service and one accessing job, which communicate with two files to establish
    that both run concurrently.
    """
    #Clean out out-file
    open(outFile, 'w').close()
    randInt = random.randint(1, sys.maxsize) # We create a random number that is added to messageInt and subtracted by the serviceAccessor, to prove that
    # when service test is checkpointed and restarted there is never a connection made between an earlier service and later serviceAccessor, or vice versa.
    job.addChildJobFn(serviceAccessor, job.addService(TestService(messageInt + randInt)), outFile, randInt)

def serviceTestRecursive(job, outFile, messages):
    """
    Creates a chain of services and accessing jobs, each paired together.
    """
    if len(messages) > 0:
        #Clean out out-file
        open(outFile, 'w').close()
        randInt = random.randint(1, sys.maxsize)
        service = TestService(messages[0] + randInt)
        child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

        for i in range(1, len(messages)):
            randInt = random.randint(1, sys.maxsize)
            service2 = TestService(messages[i] + randInt, cores=0.1)
            child = child.addChildJobFn(serviceAccessor,
                                        job.addService(service2, parentService=service),
                                        outFile, randInt, cores=0.1)
            service = service2

def serviceTestParallelRecursive(job, outFiles, messageBundles):
    """
    Creates multiple chains of services and accessing jobs.
    """
    for messages, outFile in zip(messageBundles, outFiles):
        #Clean out out-file
        open(outFile, 'w').close()
        if len(messages) > 0:
            randInt = random.randint(1, sys.maxsize)
            service = TestService(messages[0] + randInt)
            child = job.addChildJobFn(serviceAccessor, job.addService(service), outFile, randInt)

            for i in range(1, len(messages)):
                randInt = random.randint(1, sys.maxsize)
                service2 = TestService(messages[i] + randInt, cores=0.1)
                child = child.addChildJobFn(serviceAccessor,
                                            job.addService(service2, parentService=service),
                                            outFile, randInt, cores=0.1)
                service = service2

class TestService(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        While established the service repeatedly:
             - reads an integer i from the inJobStoreFileID file
             - writes i and messageInt to the outJobStoreFileID file
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        assert self.disk is not None
        assert self.memory is not None
        assert self.cores is not None
        self.terminate = Event()
        self.error = Event()
        inJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        outJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
        self.serviceThread = Thread(target=self.serviceWorker,
                                    args=(job.fileStore.jobStore, self.terminate, self.error,
                                          inJobStoreID, outJobStoreID,
                                          self.messageInt))
        self.serviceThread.start()
        return (inJobStoreID, outJobStoreID)

    def stop(self, job):
        self.terminate.set()
        self.serviceThread.join()

    def check(self):
        if self.error.isSet():
            raise RuntimeError("Service worker failed")
        return True

    @staticmethod
    def serviceWorker(jobStore, terminate, error, inJobStoreID, outJobStoreID, messageInt):
        try:
            while True:
                if terminate.isSet(): # Quit if we've got the terminate signal
                    logger.debug("Demo service worker being told to quit")
                    return

                time.sleep(0.2) # Sleep to avoid thrashing

                # Try reading a line from the input file
                try:
                    with jobStore.readFileStream(inJobStoreID) as fH:
                        line = fH.readline()
                except:
                    logger.debug("Something went wrong reading a line")
                    raise

                # Try converting the input line into an integer
                try:
                    inputInt = int(line)
                except ValueError:
                    logger.debug("Tried casting input line to integer but got error: %s" % line)
                    continue

                # Write out the resulting read integer and the message
                with jobStore.updateFileStream(outJobStoreID) as fH:
                    fH.write("%s %s\n" % (inputInt, messageInt))
        except:
            error.set()
            raise

def serviceAccessor(job, communicationFiles, outFile, randInt):
    """
    Writes a random integer iinto the inJobStoreFileID file, then tries 10 times reading
    from outJobStoreFileID to get a pair of integers, the first equal to i the second written into the outputFile.
    """
    inJobStoreFileID, outJobStoreFileID = communicationFiles

    # Get a random integer
    key = random.randint(1, sys.maxsize)

    # Write the integer into the file
    logger.debug("Writing key to inJobStoreFileID")
    with job.fileStore.jobStore.updateFileStream(inJobStoreFileID) as fH:
        fH.write("%s\n" % key)

    logger.debug("Trying to read key and message from outJobStoreFileID")
    for i in range(10): # Try 10 times over
        time.sleep(0.2) #Avoid thrashing

        # Try reading an integer from the input file and writing out the message
        with job.fileStore.jobStore.readFileStream(outJobStoreFileID) as fH:
            line = fH.readline()

        tokens = line.split()
        if len(tokens) != 2:
            continue

        key2, message = tokens

        if int(key2) == key:
            logger.debug("Matched key's: %s, writing message: %s with randInt: %s" % (key, int(message) - randInt, randInt))
            with open(outFile, 'a') as fH:
                fH.write("%s\n" % (int(message) - randInt))
            return

    assert 0 # Job failed to get info from the service

class TestServiceSerialization(Job.Service):
    def __init__(self, messageInt, *args, **kwargs):
        """
        Trivial service for testing serialization.
        """
        Job.Service.__init__(self, *args, **kwargs)
        self.messageInt = messageInt

    def start(self, job):
        return self.messageInt

    def stop(self, job):
        pass

    def check(self):
        return True

def fnTest(strings, outputFile):
    """
    Function concatenates the strings together and writes them to the output file
    """
    with open(outputFile, 'w') as fH:
        fH.write(" ".join(strings))

<|endoftext|>


<commit_before>import ROOT
from .logger import log
from .core import Object
from . import defaults
from .info import __version_info__, __version__

import warnings

# show deprecation warnings
warnings.filterwarnings('always', category=DeprecationWarning)

class ROOTError(RuntimeError):
    """
    Exception class representing a ROOT error/warning message.
    """
    def __init__(self, level, location, msg):
        self.level, self.location, self.msg = level, location, msg

    def __str__(self):
        return "level={0}, loc='{1}', msg='{2}'".format(
            self.level, self.location, self.msg)

def rootpy_source_dir():
    import rootpy
    from os.path import abspath, dirname
    from inspect import getfile
    from sys import modules
    path = dirname(getfile(modules[__name__]))
    absp = abspath(path)
    return path, absp

_ROOTPY_SOURCE_PATH, _ROOTPY_SOURCE_ABSPATH = rootpy_source_dir()
del rootpy_source_dir

'''
All rootpy wrappers are registered below. This dict maps the ROOT class name to
the path to the rootpy class or a tuple of both the path and keyword arguments
used in the dynamic_cls classmethod (see the ``Hist``, ``Hist2D`` and
``Hist3D`` classes in ``plotting.hist``).

This way rootpy is aware of all classes within the package that inherit ROOT
classes without needing to import everything with rootpy is first imported.
This registry is required to cast ROOT objects into the rootpy form when
extracted from a ROOT TFile for example.
'''
INIT_REGISTRY = {
    'TTree': 'tree.tree.Tree',

    'TDirectoryFile': 'io.file.Directory',

    'TCanvas': 'plotting.canvas.Canvas',
    'TPad': 'plotting.canvas.Pad',
    'TLegend': 'plotting.legend.Legend',

    'TGraphAsymmErrors': 'plotting.graph.Graph',
    'TGraph2D': 'plotting.graph.Graph2D',

    'TH1C': ('plotting.hist.Hist', dict(type='C')),
    'TH1S': ('plotting.hist.Hist', dict(type='S')),
    'TH1I': ('plotting.hist.Hist', dict(type='I')),
    'TH1F': ('plotting.hist.Hist', dict(type='F')),
    'TH1D': ('plotting.hist.Hist', dict(type='D')),

    'TH2C': ('plotting.hist.Hist2D', dict(type='C')),
    'TH2S': ('plotting.hist.Hist2D', dict(type='S')),
    'TH2I': ('plotting.hist.Hist2D', dict(type='I')),
    'TH2F': ('plotting.hist.Hist2D', dict(type='F')),
    'TH2D': ('plotting.hist.Hist2D', dict(type='D')),

    'TH3C': ('plotting.hist.Hist3D', dict(type='C')),
    'TH3S': ('plotting.hist.Hist3D', dict(type='S')),
    'TH3I': ('plotting.hist.Hist3D', dict(type='I')),
    'TH3F': ('plotting.hist.Hist3D', dict(type='F')),
    'TH3D': ('plotting.hist.Hist3D', dict(type='D')),

    'THStack': 'plotting.hist.HistStack',

    'TVector2': 'math.physics.vector.TVector2',
    'TVector3': 'math.physics.vector.TVector3',
    'TLorentzVector': 'math.physics.vector.LorentzVector',
    'TRotation': 'math.physics.vector.Rotation',
    'TLorentzRotation': 'math.physics.vector.LorentzRotation',
}

if ROOT.gROOT.GetVersionInt() >= 52800:
    INIT_REGISTRY['TEfficiency'] = 'plotting.hist.Efficiency'

# this dict is populated as classes are registered at runtime
REGISTRY = {}

def asrootpy(thing, **kwargs):

    # is this thing already converted?
    if isinstance(thing, Object):
        return thing

    thing_cls = thing.__class__
    rootpy_cls = lookup(thing_cls)
    if rootpy_cls is None:
        # warn that this class is not wrapped in rootpy
        # return the original
        warnings.warn(
                "A subclass of ``%s`` is not implemented in rootpy. "
                "Returning the original object instead." % thing_cls)
        return thing

    # cast
    thing.__class__ = rootpy_cls
    # call the _post_init if one exists
    if hasattr(thing, '_post_init'):
        thing._post_init(**kwargs)

    return thing

def lookup(cls):

    cls_name = cls.__name__
    return lookup_by_name(cls_name)

def lookup_by_name(cls_name):

    if cls_name in REGISTRY:
        return REGISTRY[cls_name]
    if cls_name not in INIT_REGISTRY:
        return None
    entry = INIT_REGISTRY[cls_name]
    if isinstance(entry, tuple):
        path, dynamic_kwargs = entry
    elif isinstance(entry, basestring):
        path = entry
        dynamic_kwargs = None
    path_tokens = path.split('.')
    path, rootpy_cls_name = '.'.join(path_tokens[:-1]), path_tokens[-1]
    rootpy_module = __import__(
            path, globals(), locals(), [rootpy_cls_name], -1)
    rootpy_cls = getattr(rootpy_module, rootpy_cls_name)
    if dynamic_kwargs is not None:
        rootpy_cls = rootpy_cls.dynamic_cls(**dynamic_kwargs)
    REGISTRY[cls_name] = rootpy_cls
    return rootpy_cls

class register(object):

    def __init__(self, names=None, builtin=False):

        if names is not None:
            if type(names) not in (list, tuple):
                names = [names]
        self.names = names
        self.builtin = builtin

    def __call__(self, cls):

        if issubclass(cls, Object):
            # all rootpy classes which inherit from ROOT classes
            # must place the ROOT base class as the last class
            # in the inheritance list and inherit from Object
            rootbase = cls.__bases__[-1]
            cls_names = [rootbase.__name__]
        else:
            cls_names = [cls.__name__]

        if self.names is not None:
            cls_names += self.names

        for name in cls_names:
            if name in REGISTRY:
                warnings.warn("Duplicate registration of class %s" % name)
            REGISTRY[name] = cls
        return cls

def create(cls_name, *args, **kwargs):

    try:
        cls = getattr(ROOT, cls_name)
        obj = cls(*args, **kwargs)
        return asrootpy(obj)
    except:
        return None
<commit_msg>import ROOT last<commit_after>from .logger import log
from . import defaults
from .core import Object
from .info import __version_info__, __version__
import ROOT

import warnings

# show deprecation warnings
warnings.filterwarnings('always', category=DeprecationWarning)

class ROOTError(RuntimeError):
    """
    Exception class representing a ROOT error/warning message.
    """
    def __init__(self, level, location, msg):
        self.level, self.location, self.msg = level, location, msg

    def __str__(self):
        return "level={0}, loc='{1}', msg='{2}'".format(
            self.level, self.location, self.msg)

def rootpy_source_dir():
    import rootpy
    from os.path import abspath, dirname
    from inspect import getfile
    from sys import modules
    path = dirname(getfile(modules[__name__]))
    absp = abspath(path)
    return path, absp

_ROOTPY_SOURCE_PATH, _ROOTPY_SOURCE_ABSPATH = rootpy_source_dir()
del rootpy_source_dir

'''
All rootpy wrappers are registered below. This dict maps the ROOT class name to
the path to the rootpy class or a tuple of both the path and keyword arguments
used in the dynamic_cls classmethod (see the ``Hist``, ``Hist2D`` and
``Hist3D`` classes in ``plotting.hist``).

This way rootpy is aware of all classes within the package that inherit ROOT
classes without needing to import everything with rootpy is first imported.
This registry is required to cast ROOT objects into the rootpy form when
extracted from a ROOT TFile for example.
'''
INIT_REGISTRY = {
    'TTree': 'tree.tree.Tree',

    'TDirectoryFile': 'io.file.Directory',

    'TCanvas': 'plotting.canvas.Canvas',
    'TPad': 'plotting.canvas.Pad',
    'TLegend': 'plotting.legend.Legend',

    'TGraphAsymmErrors': 'plotting.graph.Graph',
    'TGraph2D': 'plotting.graph.Graph2D',

    'TH1C': ('plotting.hist.Hist', dict(type='C')),
    'TH1S': ('plotting.hist.Hist', dict(type='S')),
    'TH1I': ('plotting.hist.Hist', dict(type='I')),
    'TH1F': ('plotting.hist.Hist', dict(type='F')),
    'TH1D': ('plotting.hist.Hist', dict(type='D')),

    'TH2C': ('plotting.hist.Hist2D', dict(type='C')),
    'TH2S': ('plotting.hist.Hist2D', dict(type='S')),
    'TH2I': ('plotting.hist.Hist2D', dict(type='I')),
    'TH2F': ('plotting.hist.Hist2D', dict(type='F')),
    'TH2D': ('plotting.hist.Hist2D', dict(type='D')),

    'TH3C': ('plotting.hist.Hist3D', dict(type='C')),
    'TH3S': ('plotting.hist.Hist3D', dict(type='S')),
    'TH3I': ('plotting.hist.Hist3D', dict(type='I')),
    'TH3F': ('plotting.hist.Hist3D', dict(type='F')),
    'TH3D': ('plotting.hist.Hist3D', dict(type='D')),

    'THStack': 'plotting.hist.HistStack',

    'TVector2': 'math.physics.vector.TVector2',
    'TVector3': 'math.physics.vector.TVector3',
    'TLorentzVector': 'math.physics.vector.LorentzVector',
    'TRotation': 'math.physics.vector.Rotation',
    'TLorentzRotation': 'math.physics.vector.LorentzRotation',
}

if ROOT.gROOT.GetVersionInt() >= 52800:
    INIT_REGISTRY['TEfficiency'] = 'plotting.hist.Efficiency'

# this dict is populated as classes are registered at runtime
REGISTRY = {}

def asrootpy(thing, **kwargs):

    # is this thing already converted?
    if isinstance(thing, Object):
        return thing

    thing_cls = thing.__class__
    rootpy_cls = lookup(thing_cls)
    if rootpy_cls is None:
        # warn that this class is not wrapped in rootpy
        # return the original
        warnings.warn(
                "A subclass of ``%s`` is not implemented in rootpy. "
                "Returning the original object instead." % thing_cls)
        return thing

    # cast
    thing.__class__ = rootpy_cls
    # call the _post_init if one exists
    if hasattr(thing, '_post_init'):
        thing._post_init(**kwargs)

    return thing

def lookup(cls):

    cls_name = cls.__name__
    return lookup_by_name(cls_name)

def lookup_by_name(cls_name):

    if cls_name in REGISTRY:
        return REGISTRY[cls_name]
    if cls_name not in INIT_REGISTRY:
        return None
    entry = INIT_REGISTRY[cls_name]
    if isinstance(entry, tuple):
        path, dynamic_kwargs = entry
    elif isinstance(entry, basestring):
        path = entry
        dynamic_kwargs = None
    path_tokens = path.split('.')
    path, rootpy_cls_name = '.'.join(path_tokens[:-1]), path_tokens[-1]
    rootpy_module = __import__(
            path, globals(), locals(), [rootpy_cls_name], -1)
    rootpy_cls = getattr(rootpy_module, rootpy_cls_name)
    if dynamic_kwargs is not None:
        rootpy_cls = rootpy_cls.dynamic_cls(**dynamic_kwargs)
    REGISTRY[cls_name] = rootpy_cls
    return rootpy_cls

class register(object):

    def __init__(self, names=None, builtin=False):

        if names is not None:
            if type(names) not in (list, tuple):
                names = [names]
        self.names = names
        self.builtin = builtin

    def __call__(self, cls):

        if issubclass(cls, Object):
            # all rootpy classes which inherit from ROOT classes
            # must place the ROOT base class as the last class
            # in the inheritance list and inherit from Object
            rootbase = cls.__bases__[-1]
            cls_names = [rootbase.__name__]
        else:
            cls_names = [cls.__name__]

        if self.names is not None:
            cls_names += self.names

        for name in cls_names:
            if name in REGISTRY:
                warnings.warn("Duplicate registration of class %s" % name)
            REGISTRY[name] = cls
        return cls

def create(cls_name, *args, **kwargs):

    try:
        cls = getattr(ROOT, cls_name)
        obj = cls(*args, **kwargs)
        return asrootpy(obj)
    except:
        return None
<|endoftext|>


<commit_before><commit_msg>Fixed #4060 -- Databrowse no longer requires admin site to be installed<commit_after><|endoftext|>


<commit_before>from __future__ import absolute_import
import os
import nox

BLACK_PATHS = ["iam_groups_authn"]

if os.path.exists("samples"):
    BLACK_PATHS.append("samples")

@nox.session
def lint(session):
    """Run linters.
    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.install("-r", "requirements-test.txt")
    session.install("-r", "requirements.txt")
    session.run("black", "--check", *BLACK_PATHS)
<commit_msg>chore: add header<commit_after># Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import os
import nox

BLACK_PATHS = ["iam_groups_authn"]

if os.path.exists("samples"):
    BLACK_PATHS.append("samples")

@nox.session
def lint(session):
    """Run linters.
    Returns a failure if the linters find linting errors or sufficiently
    serious code quality issues.
    """
    session.install("-r", "requirements-test.txt")
    session.install("-r", "requirements.txt")
    session.run("black", "--check", *BLACK_PATHS)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re

from bs4 import BeautifulSoup
from utils import utils, inspector

# https://www.nlrb.gov/who-we-are/inspector-general
archive = 1998

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - Add a published date for 'OIG-F-16-12-01' on
# https://www.nlrb.gov/reports-guidance/reports/oig-audit-reports

AUDIT_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-audit-reports"
INSPECTION_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-inspection-reports"
SEMIANNUAL_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-semiannual-reports"

REPORT_PUBLISHED_MAP = {
  'OIG-F-16-12-01': datetime.datetime(2011, 12, 14),
}

REPORT_URLS = {
  "audit": AUDIT_REPORTS_URL,
  "inspection": INSPECTION_REPORTS_URL,
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit and inspections reports
  for report_type, reports_url in REPORT_URLS.items():
    doc = BeautifulSoup(utils.download(reports_url))
    results = doc.select("div.field-item")
    for result in results:
      report = report_from(result, report_type, year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = BeautifulSoup(utils.download(SEMIANNUAL_REPORTS_URL))
  results = doc.select("div.field-item")
  for result in results:
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_from(result, report_type, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_id, title = link.text.split(maxsplit=1)
  report_id = report_id.rstrip(":").rstrip(",")

  title = title.strip()

  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    published_on_text = re.findall('\((.*?)\)', title)[-1]
    try:
      published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')
    except ValueError:
      published_on = datetime.datetime.strptime(published_on_text, '%B %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'nlrb',
    'inspector_url': "https://www.nlrb.gov/who-we-are/inspector-general",
    'agency': 'nlrb',
    'agency_name': "National Labor Relations Board",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

def semiannual_report_from(result, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  title = "Semiannual report - {}".format(link.text.strip())

  published_on_text = link.text.split("-")[-1].strip()
  published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'nlrb',
    'inspector_url': "https://www.nlrb.gov/who-we-are/inspector-general",
    'agency': 'nlrb',
    'agency_name': "National Labor Relations Board",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[nlrb] Improve date parsing<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re

from bs4 import BeautifulSoup
from utils import utils, inspector

# https://www.nlrb.gov/who-we-are/inspector-general
archive = 1998

# options:
#   standard since/year options for a year range to fetch from.
#
# Notes for IG's web team:
# - Add a published date for 'OIG-F-16-12-01' on
# https://www.nlrb.gov/reports-guidance/reports/oig-audit-reports

AUDIT_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-audit-reports"
INSPECTION_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-inspection-reports"
SEMIANNUAL_REPORTS_URL = "https://www.nlrb.gov/reports-guidance/reports/oig-semiannual-reports"

REPORT_PUBLISHED_MAP = {
  'OIG-F-16-12-01': datetime.datetime(2011, 12, 14),
}

REPORT_URLS = {
  "audit": AUDIT_REPORTS_URL,
  "inspection": INSPECTION_REPORTS_URL,
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # Pull the audit and inspections reports
  for report_type, reports_url in REPORT_URLS.items():
    doc = BeautifulSoup(utils.download(reports_url))
    results = doc.select("div.field-item")
    for result in results:
      report = report_from(result, report_type, year_range)
      if report:
        inspector.save_report(report)

  # Pull the semiannual reports
  doc = BeautifulSoup(utils.download(SEMIANNUAL_REPORTS_URL))
  results = doc.select("div.field-item")
  for result in results:
    report = semiannual_report_from(result, year_range)
    if report:
      inspector.save_report(report)

def report_from(result, report_type, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_id, title = link.text.split(maxsplit=1)
  report_id = report_id.rstrip(":").rstrip(",")

  title = title.strip()

  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    for paren_text in re.findall('\((.*?)\)', title):
      try:
        published_on = datetime.datetime.strptime(paren_text, '%B %d, %Y')
        break
      except ValueError:
        pass
      try:
        published_on = datetime.datetime.strptime(paren_text, '%B %Y')
        break
      except ValueError:
        pass

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'nlrb',
    'inspector_url': "https://www.nlrb.gov/who-we-are/inspector-general",
    'agency': 'nlrb',
    'agency_name': "National Labor Relations Board",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

def semiannual_report_from(result, year_range):
  link = result.find("a")
  report_url = link.get('href')
  report_filename = report_url.split("/")[-1]
  report_id, _ = os.path.splitext(report_filename)

  title = "Semiannual report - {}".format(link.text.strip())

  published_on_text = link.text.split("-")[-1].strip()
  published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  report = {
    'inspector': 'nlrb',
    'inspector_url': "https://www.nlrb.gov/who-we-are/inspector-general",
    'agency': 'nlrb',
    'agency_name': "National Labor Relations Board",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>


<commit_before># Copyright 2014 Benjamin Woodruff, Colin Chan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import functools
import logging
import json

import flask
from flask import g
from flask import request
from sqlalchemy import desc

from .algorithm import journey_scores
from .db import Session
from .db.models import Account
from .db.models import Journey
from .db.models import Route
from .db.models import Waypoint
from .db.transform import parse_time
from .db.transform import to_dict
from .db.transform import to_list

logger = logging.getLogger("routemaster.web")
app = flask.Flask("routemaster")

def get_account_id(request):
    """Validate the request's session and return the account id.

    If the session is not valid, an exception will be raised.

    Currently this just returns "test" (the test account) every time.
    """
    return "test"

def get_or_404(type, **kwargs):
    q = g.db.query(type).filter_by(**kwargs).first()
    if not q:
        flask.abort(404)
    return q

def json_response(func):
    @functools.wraps(func)
    def f(*args, **kwargs):
        data = func(*args, **kwargs)
        json_data = json.dumps(data, ensure_ascii=False, indent=2)
        response = flask.make_response(json_data)
        response.headers["content-type"] = "application/json; charset=utf-8"
        return response
    return f

@app.before_request
def setup_db_session():
    g.db = Session()

@app.route("/account/<aid>")
@json_response
def get_account(aid):
    return to_dict(get_or_404(Account, id=aid))

@app.route("/account/<aid>/recent")
@json_response
def get_account_recent(aid):
    query = (g.db.query(Journey).filter_by(account_id=aid)
             .order_by(desc(Journey.start_time_utc)))
    return to_list(query.all())

@app.route("/hello")
@json_response
def hello():
    return {"error": "Hello, world!"}

@app.route("/journey", methods=["POST"])
@json_response
def store_journey():
    data = request.get_json()
    logger.debug(data)
    journey = Journey(
        id=data['id'],
        account_id=get_account_id(request),
        visibility=data['visibility'],
        start_time_utc=parse_time(data['startTimeUtc']),
        stop_time_utc=parse_time(data['stopTimeUtc']),
        start_place_id=data.get('startPlaceId', None),
        stop_place_id=data.get('stopPlaceId', None),
    )
    g.db.add(journey)

    waypoints = []
    for w in data['waypoints']:
        waypoint = Waypoint(
            journey=journey,
            time_utc=parse_time(w['timeUtc']),
            accuracy_m=w['accuracyM'],
            latitude=w['latitude'],
            longitude=w['longitude'],
            height_m=w['heightM'],
        )
        g.db.add(waypoint)
        waypoints.append(waypoint)

    if len(waypoints) < 2:
        # TODO: return 400 Bad Request?
        ...

    journey.efficiency, journey.distance_m = journey_scores(waypoints)

    g.db.commit()
    return to_dict(journey)

@app.route("/journey/<jid>")
@json_response
def get_journey(jid):
    return to_dict(get_or_404(Journey, id=jid))

@app.route("/route/<int:rid>")
@json_response
def get_route(rid):
    return to_dict(get_or_404(Route, id=rid))
<commit_msg>Get the account ID from request headers<commit_after># Copyright 2014 Benjamin Woodruff, Colin Chan
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import functools
import logging
import json

import flask
from flask import g
from flask import request
from sqlalchemy import desc

from .algorithm import journey_scores
from .db import Session
from .db.models import Account
from .db.models import Journey
from .db.models import Route
from .db.models import Waypoint
from .db.transform import parse_time
from .db.transform import to_dict
from .db.transform import to_list

logger = logging.getLogger("routemaster.web")
app = flask.Flask("routemaster")

def get_account_id(request):
    """Extract the account id from the request headers"""
    return request.headers.get('RouteMasterAccountId') or "test"

def get_or_404(type, **kwargs):
    q = g.db.query(type).filter_by(**kwargs).first()
    if not q:
        flask.abort(404)
    return q

def json_response(func):
    @functools.wraps(func)
    def f(*args, **kwargs):
        data = func(*args, **kwargs)
        json_data = json.dumps(data, ensure_ascii=False, indent=2)
        response = flask.make_response(json_data)
        response.headers["content-type"] = "application/json; charset=utf-8"
        return response
    return f

@app.before_request
def setup_db_session():
    g.db = Session()

@app.route("/account/<aid>")
@json_response
def get_account(aid):
    return to_dict(get_or_404(Account, id=aid))

@app.route("/account/<aid>/recent")
@json_response
def get_account_recent(aid):
    query = (g.db.query(Journey).filter_by(account_id=aid)
             .order_by(desc(Journey.start_time_utc)))
    return to_list(query.all())

@app.route("/hello")
@json_response
def hello():
    return {"error": "Hello, world!"}

@app.route("/journey", methods=["POST"])
@json_response
def store_journey():
    data = request.get_json()
    logger.debug(data)
    journey = Journey(
        id=data['id'],
        account_id=get_account_id(request),
        visibility=data['visibility'],
        start_time_utc=parse_time(data['startTimeUtc']),
        stop_time_utc=parse_time(data['stopTimeUtc']),
        start_place_id=data.get('startPlaceId', None),
        stop_place_id=data.get('stopPlaceId', None),
    )
    g.db.add(journey)

    waypoints = []
    for w in data['waypoints']:
        waypoint = Waypoint(
            journey=journey,
            time_utc=parse_time(w['timeUtc']),
            accuracy_m=w['accuracyM'],
            latitude=w['latitude'],
            longitude=w['longitude'],
            height_m=w['heightM'],
        )
        g.db.add(waypoint)
        waypoints.append(waypoint)

    if len(waypoints) < 2:
        # TODO: return 400 Bad Request?
        ...

    journey.efficiency, journey.distance_m = journey_scores(waypoints)

    g.db.commit()
    return to_dict(journey)

@app.route("/journey/<jid>")
@json_response
def get_journey(jid):
    return to_dict(get_or_404(Journey, id=jid))

@app.route("/route/<int:rid>")
@json_response
def get_route(rid):
    return to_dict(get_or_404(Route, id=rid))
<|endoftext|>


<commit_before><commit_msg>add initial migration<commit_after><|endoftext|>


<commit_before>"""
VerseBot for reddit
By Matthieu Grieger
regex.py
Copyright (c) 2015 Matthieu Grieger (MIT License)
"""

import re

def find_verses(message_body):
    """ Uses regex to search comment body for verse quotations. Returns
    a list of matches if found, None otherwise. """

    regex = (r"(?<=\[)(?P<book>[\d\w\s]+)(?P<chapter>\d+)\:?(?P<verse>\d+"
        r"\-?\d*)?\s*\(?(?P<translation>[\w\-\d]+)?\)?(?=\])")

    matches = re.findall(regex, message_body)
    if len(matches) == 0:
        return None
    else:
        return matches

def find_already_quoted_verses(message_body):
    """ Uses regex to search an existing VerseBot response for verse
    quotations. Used for removing invalid statistics from database. """

    regex = (r"\[\*\*(?P<book>[\d\w\s]+)\d+\:?\d*\-?\d*\s\|\s[\w\d\s]+"
        r"\((?P<translation>[\w\-\d]+)?\)")

    matches = re.findall(regex, message_body)
    if len(matches) == 0:
        return None
    else:
        return matches

def find_default_translations(message_body):
    """ Uses regex to search a private message for default translations
    when a user sends a default translation request. """

    regex = (r"OT\:\s(?P<ot>[\w\-\d]+),\sNT\:\s(?P<nt>[\w\-\d]+),"
        r"\sDeut\:\s(?P<deut>[\w\-\d]+)")

    match = re.findall(regex, message_body)
    if len(match) == 0:
        return None, None, None
    else:
        return match[0][0], match[0][1], match[0][2]

def find_subreddit_in_request(message_body):
    """ Uses regex to search a private message for a subreddit when a
    user sends a default translation request. """

    regex = (r"Subreddit\:\s(?P<subreddit>[\w\d_]+),")

    match = re.findall(regex, message_body)
    if len(match) == 0:
        return None
    else:
        return match[0]
<commit_msg>Small regex fix<commit_after>"""
VerseBot for reddit
By Matthieu Grieger
regex.py
Copyright (c) 2015 Matthieu Grieger (MIT License)
"""

import re

def find_verses(message_body):
    """ Uses regex to search comment body for verse quotations. Returns
    a list of matches if found, None otherwise. """

    regex = (r"(?<=\[)(?P<book>[\d\w\s]+)\s(?P<chapter>\d+)\:?(?P<verse>\d+"
        r"\-?\d*)?\s*\(?(?P<translation>[\w\-\d]+)?\)?(?=\])")

    matches = re.findall(regex, message_body)
    if len(matches) == 0:
        return None
    else:
        return matches

def find_already_quoted_verses(message_body):
    """ Uses regex to search an existing VerseBot response for verse
    quotations. Used for removing invalid statistics from database. """

    regex = (r"\[\*\*(?P<book>[\d\w\s]+)\d+\:?\d*\-?\d*\s\|\s[\w\d\s]+"
        r"\((?P<translation>[\w\-\d]+)?\)")

    matches = re.findall(regex, message_body)
    if len(matches) == 0:
        return None
    else:
        return matches

def find_default_translations(message_body):
    """ Uses regex to search a private message for default translations
    when a user sends a default translation request. """

    regex = (r"OT\:\s(?P<ot>[\w\-\d]+),\sNT\:\s(?P<nt>[\w\-\d]+),"
        r"\sDeut\:\s(?P<deut>[\w\-\d]+)")

    match = re.findall(regex, message_body)
    if len(match) == 0:
        return None, None, None
    else:
        return match[0][0], match[0][1], match[0][2]

def find_subreddit_in_request(message_body):
    """ Uses regex to search a private message for a subreddit when a
    user sends a default translation request. """

    regex = (r"Subreddit\:\s(?P<subreddit>[\w\d_]+),")

    match = re.findall(regex, message_body)
    if len(match) == 0:
        return None
    else:
        return match[0]
<|endoftext|>