<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
from spack.operating_systems.mac_os import macos_version

import os
import sys

MACOS_VERSION = macos_version() if sys.platform == 'darwin' else None

class Graphviz(AutotoolsPackage):
    """Graph Visualization Software"""

    homepage = 'http://www.graphviz.org'
    git      = 'https://gitlab.com/graphviz/graphviz.git'

    # This commit hash is tag='stable_release_2.42.2'
    version('2.42.2', commit='da4c2ec6f24ca1b6d1752c6b5bc4389e55682147')
    # This commit hash is tag='stable_release_2.40.1'
    version('2.40.1', commit='67cd2e5121379a38e0801cc05cce5033f8a2a609')

    conflicts('%gcc@:5.9',
              when='@2.40.1+qt ^qt@5:',
              msg='graphviz-2.40.1 needs gcc-6 or greater to compile with QT5 '
              'suppport')

    # Language bindings
    language_bindings = ['java']

    # Additional language bindings are nominally supported by GraphViz via SWIG
    # but are untested and need the proper dependencies added:
    # language_bindings += ['sharp', 'go', 'guile', 'io', 'lua', 'ocaml',
    #                       'perl', 'php', 'python', 'r', 'ruby', 'tcl']

    for lang in language_bindings:
        variant(lang, default=False,
                description='Enable for optional {0} language '
                'bindings'.format(lang))

    # Feature variants
    variant('expat', default=False,
            description='Build with Expat support (enables HTML-like labels)')
    variant('gts', default=False,
            description='Build with GNU Triangulated Surface Library')
    variant('ghostscript', default=False,
            description='Build with Ghostscript support')
    variant('gtkplus', default=False,
            description='Build with GTK+ support')
    variant('libgd', default=False,
            description='Build with libgd support (more output formats)')
    variant('pangocairo', default=False,
            description='Build with pango+cairo support (more output formats)')
    variant('poppler', default=False,
            description='Build with poppler support (pdf formats)')
    variant('qt', default=False,
            description='Build with Qt support')
    variant('quartz', default=(MACOS_VERSION is not None),
            description='Build with Quartz and PDF support')
    variant('x', default=False,
            description='Use the X Window System')

    patch('http://www.linuxfromscratch.org/patches/blfs/9.0/graphviz-2.40.1-qt5-1.patch',
          sha256='bd532df325df811713e311d17aaeac3f5d6075ea4fd0eae8d989391e6afba930',
          when='@:2.40+qt^qt@5:')
    patch('https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/master/easybuild/easyconfigs/g/Graphviz/Graphviz-2.38.0_icc_sfio.patch',
          sha256='393a0a772315a89dcc970b5efd4765d22dba83493d7956303673eb89c45b949f',
          level=0,
          when='@:2.40%intel')
    patch('https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/master/easybuild/easyconfigs/g/Graphviz/Graphviz-2.40.1_icc_vmalloc.patch',
          sha256='813e6529e79161a18b0f24a969b7de22f8417b2e942239e658b5402884541bc2',
          when='@:2.40%intel')

    if not MACOS_VERSION:
        conflicts('+quartz',
                  msg="Graphviz can only be build with Quartz on macOS.")
    elif MACOS_VERSION >= Version('10.9'):
        # Doesn't detect newer mac os systems as being new
        patch('fix-quartz-darwin.patch')

    # Language dependencies
    depends_on('java', when='+java')
    for lang in language_bindings:
        depends_on('swig', when=('+' + lang))

    # Feature dependencies
    depends_on('expat', when='+expat')
    depends_on('libgd', when='+libgd')
    depends_on('fontconfig', when='+libgd')
    depends_on('freetype', when='+libgd')
    depends_on('ghostscript', when='+ghostscript')
    depends_on('gtkplus', when='+gtkplus')
    depends_on('gts', when='+gts')
    depends_on('cairo+pdf', when='+pangocairo')
    depends_on('fontconfig', when='+pangocairo')
    depends_on('freetype', when='+pangocairo')
    depends_on('glib', when='+pangocairo')
    depends_on('libpng', when='+pangocairo')
    depends_on('pango', when='+pangocairo')
    depends_on('poppler+glib', when='+poppler')
    depends_on('zlib')
    depends_on('qt', when='+qt')
    depends_on('libx11', when="+x")

    # Build dependencies
    depends_on('pkgconfig', type='build')
    # The following are needed when building from git
    depends_on('automake', type='build')
    depends_on('autoconf', type='build')
    depends_on('bison', type='build')
    depends_on('flex', type='build')
    depends_on('libtool', type='build')

    parallel = False

    def autoreconf(self, spec, prefix):
        # We need to generate 'configure' when checking out sources from git
        # If configure exists nothing needs to be done
        if os.path.exists(self.configure_abs_path):
            return
        # Else bootstrap (disabling auto-configure with NOCONFIG)
        bash = which('bash')
        bash('./autogen.sh', 'NOCONFIG')

    def setup_build_environment(self, env):
        if '+quartz' in self.spec:
            env.set('OBJC', self.compiler.cc)

    @when('%clang platform=darwin')
    def patch(self):
        # When using Clang, replace GCC's libstdc++ with LLVM's libc++
        mkdirs = ['cmd/dot', 'cmd/edgepaint', 'cmd/mingle', 'plugin/gdiplus']
        filter_file(r'-lstdc\+\+', '-lc++', 'configure.ac',
                    *(d + '/Makefile.am' for d in mkdirs))

    def configure_args(self):
        spec = self.spec
        args = ['--disable-silent-rules']

        use_swig = False
        for lang in self.language_bindings:
            if '+' + lang in spec:
                use_swig = True
                args.append('--enable-' + lang)

        args.append('--{0}-swig'.format('enable' if use_swig else 'disable'))

        for var in ["expat", "gts", "ghostscript", "libgd", "pangocairo",
                    "poppler", "qt", "quartz", "x"]:
            args += self.with_or_without(var)

        args.append('--{0}-gtk'.format(
            "with" if "+gtkplus" in spec else "without"))

        return args
<commit_msg>graphviz: fix cairo backend (#16511)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *
from spack.operating_systems.mac_os import macos_version

import os
import sys

MACOS_VERSION = macos_version() if sys.platform == 'darwin' else None

class Graphviz(AutotoolsPackage):
    """Graph Visualization Software"""

    homepage = 'http://www.graphviz.org'
    git      = 'https://gitlab.com/graphviz/graphviz.git'

    # This commit hash is tag='stable_release_2.42.2'
    version('2.42.2', commit='da4c2ec6f24ca1b6d1752c6b5bc4389e55682147')
    # This commit hash is tag='stable_release_2.40.1'
    version('2.40.1', commit='67cd2e5121379a38e0801cc05cce5033f8a2a609')

    conflicts('%gcc@:5.9',
              when='@2.40.1+qt ^qt@5:',
              msg='graphviz-2.40.1 needs gcc-6 or greater to compile with QT5 '
              'suppport')

    # Language bindings
    language_bindings = ['java']

    # Additional language bindings are nominally supported by GraphViz via SWIG
    # but are untested and need the proper dependencies added:
    # language_bindings += ['sharp', 'go', 'guile', 'io', 'lua', 'ocaml',
    #                       'perl', 'php', 'python', 'r', 'ruby', 'tcl']

    for lang in language_bindings:
        variant(lang, default=False,
                description='Enable for optional {0} language '
                'bindings'.format(lang))

    # Feature variants
    variant('expat', default=False,
            description='Build with Expat support (enables HTML-like labels)')
    variant('gts', default=False,
            description='Build with GNU Triangulated Surface Library')
    variant('ghostscript', default=False,
            description='Build with Ghostscript support')
    variant('gtkplus', default=False,
            description='Build with GTK+ support')
    variant('libgd', default=False,
            description='Build with libgd support (more output formats)')
    variant('pangocairo', default=False,
            description='Build with pango+cairo support (more output formats)')
    variant('poppler', default=False,
            description='Build with poppler support (pdf formats)')
    variant('qt', default=False,
            description='Build with Qt support')
    variant('quartz', default=(MACOS_VERSION is not None),
            description='Build with Quartz and PDF support')
    variant('x', default=False,
            description='Use the X Window System')

    patch('http://www.linuxfromscratch.org/patches/blfs/9.0/graphviz-2.40.1-qt5-1.patch',
          sha256='bd532df325df811713e311d17aaeac3f5d6075ea4fd0eae8d989391e6afba930',
          when='@:2.40+qt^qt@5:')
    patch('https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/master/easybuild/easyconfigs/g/Graphviz/Graphviz-2.38.0_icc_sfio.patch',
          sha256='393a0a772315a89dcc970b5efd4765d22dba83493d7956303673eb89c45b949f',
          level=0,
          when='@:2.40%intel')
    patch('https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/master/easybuild/easyconfigs/g/Graphviz/Graphviz-2.40.1_icc_vmalloc.patch',
          sha256='813e6529e79161a18b0f24a969b7de22f8417b2e942239e658b5402884541bc2',
          when='@:2.40%intel')

    if not MACOS_VERSION:
        conflicts('+quartz',
                  msg="Graphviz can only be build with Quartz on macOS.")
    elif MACOS_VERSION >= Version('10.9'):
        # Doesn't detect newer mac os systems as being new
        patch('fix-quartz-darwin.patch')

    # Language dependencies
    depends_on('java', when='+java')
    for lang in language_bindings:
        depends_on('swig', when=('+' + lang))

    # Feature dependencies
    depends_on('expat', when='+expat')
    depends_on('libgd', when='+libgd')
    depends_on('fontconfig', when='+libgd')
    depends_on('freetype', when='+libgd')
    depends_on('ghostscript', when='+ghostscript')
    depends_on('gtkplus', when='+gtkplus')
    depends_on('gts', when='+gts')
    depends_on('cairo+pdf+png+svg', when='+pangocairo')
    depends_on('fontconfig', when='+pangocairo')
    depends_on('freetype', when='+pangocairo')
    depends_on('glib', when='+pangocairo')
    depends_on('libpng', when='+pangocairo')
    depends_on('pango', when='+pangocairo')
    depends_on('poppler+glib', when='+poppler')
    depends_on('zlib')
    depends_on('qt', when='+qt')
    depends_on('libx11', when="+x")

    # Build dependencies
    depends_on('pkgconfig', type='build')
    # The following are needed when building from git
    depends_on('automake', type='build')
    depends_on('autoconf', type='build')
    depends_on('bison', type='build')
    depends_on('flex', type='build')
    depends_on('libtool', type='build')

    parallel = False

    def autoreconf(self, spec, prefix):
        # We need to generate 'configure' when checking out sources from git
        # If configure exists nothing needs to be done
        if os.path.exists(self.configure_abs_path):
            return
        # Else bootstrap (disabling auto-configure with NOCONFIG)
        bash = which('bash')
        bash('./autogen.sh', 'NOCONFIG')

    def setup_build_environment(self, env):
        if '+quartz' in self.spec:
            env.set('OBJC', self.compiler.cc)

    @when('%clang platform=darwin')
    def patch(self):
        # When using Clang, replace GCC's libstdc++ with LLVM's libc++
        mkdirs = ['cmd/dot', 'cmd/edgepaint', 'cmd/mingle', 'plugin/gdiplus']
        filter_file(r'-lstdc\+\+', '-lc++', 'configure.ac',
                    *(d + '/Makefile.am' for d in mkdirs))

    def configure_args(self):
        spec = self.spec
        args = ['--disable-silent-rules']

        use_swig = False
        for lang in self.language_bindings:
            if '+' + lang in spec:
                use_swig = True
                args.append('--enable-' + lang)

        args.append('--{0}-swig'.format('enable' if use_swig else 'disable'))

        for var in ["expat", "gts", "ghostscript", "libgd", "pangocairo",
                    "poppler", "qt", "quartz", "x"]:
            args += self.with_or_without(var)

        args.append('--{0}-gtk'.format(
            "with" if "+gtkplus" in spec else "without"))

        return args
<|endoftext|>


<commit_before># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyTables(PythonPackage):
    """PyTables is a package for managing hierarchical datasets and designed to
    efficiently and easily cope with extremely large amounts of data."""

    homepage = "http://www.pytables.org/"
    url      = "https://github.com/PyTables/PyTables/archive/v3.5.2.tar.gz"

    import_modules = [
        'tables', 'tables.misc', 'tables.nodes', 'tables.scripts'
    ]

    version('3.5.2', sha256='e4fc6f1194f02a8b10ff923e77364fb70710592f620d7de35f4d4e064dc70e91')
    version('3.4.4', '2cd52095ebb097f5bf58fa65dc6574bb')
    version('3.3.0', '056c161ae0fd2d6e585b766adacf3b0b')
    version('3.2.2', '7cbb0972e4d6580f629996a5bed92441',
            url='https://github.com/PyTables/PyTables/archive/v.3.2.2.tar.gz')

    variant('zlib', default=True, description='Support for zlib compression')
    variant('bzip2', default=False, description='Support for bzip2 compression')
    variant('lzo', default=False, description='Support for lzo compression')

    depends_on('python@2.7:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-six@1.9.0:', type=('build', 'run'))
    depends_on('hdf5@1.8.4:1.8.999', when='@:3.3.99')
    depends_on('hdf5@1.8.4:', when='@3.4.0:')
    depends_on('py-numpy@1.9.3:', type=('build', 'run'))
    depends_on('py-numexpr@2.6.2:', type=('build', 'run'))
    # Versions prior to 3.3 must build with the internal blosc due to a lock
    # problem in a multithreaded environment.
    depends_on('c-blosc@1.4.1:', when='@3.3.0:')
    depends_on('zlib', when='+zlib')
    depends_on('bzip2', when='+bzip2')
    depends_on('lzo', when='+lzo')
    depends_on('py-mock@2.0:', type='test')

    def setup_environment(self, spack_env, run_env):
        spack_env.set('HDF5_DIR', self.spec['hdf5'].prefix)
        if '+bzip2' in self.spec:
            spack_env.set('BZIP2_DIR', self.spec['bzip2'].prefix)
        if '+lzo' in self.spec:
            spack_env.set('LZO_DIR', self.spec['lzo'].prefix)
        if '^c-blosc' in self.spec:
            spack_env.set('BLOSC_DIR', self.spec['c-blosc'].prefix)
<commit_msg>Re-add py-cython dependency to py-tables (#12338)<commit_after># Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyTables(PythonPackage):
    """PyTables is a package for managing hierarchical datasets and designed to
    efficiently and easily cope with extremely large amounts of data."""

    homepage = "http://www.pytables.org/"
    url      = "https://github.com/PyTables/PyTables/archive/v3.5.2.tar.gz"

    import_modules = [
        'tables', 'tables.misc', 'tables.nodes', 'tables.scripts'
    ]

    version('3.5.2', sha256='e4fc6f1194f02a8b10ff923e77364fb70710592f620d7de35f4d4e064dc70e91')
    version('3.4.4', '2cd52095ebb097f5bf58fa65dc6574bb')
    version('3.3.0', '056c161ae0fd2d6e585b766adacf3b0b')
    version('3.2.2', '7cbb0972e4d6580f629996a5bed92441',
            url='https://github.com/PyTables/PyTables/archive/v.3.2.2.tar.gz')

    variant('zlib', default=True, description='Support for zlib compression')
    variant('bzip2', default=False, description='Support for bzip2 compression')
    variant('lzo', default=False, description='Support for lzo compression')

    # requirements.txt
    depends_on('python@2.6:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-cython', type='build')
    depends_on('py-numpy@1.9.3:', type=('build', 'run'))
    depends_on('py-numexpr@2.6.2:', type=('build', 'run'))
    depends_on('py-six@1.9.0:', type=('build', 'run'))
    depends_on('py-mock@2.0:', type='test')
    # tables/req_versions.py
    depends_on('hdf5@1.8.4:1.8.999', when='@:3.3.99')
    depends_on('hdf5@1.8.4:', when='@3.4.0:')
    # Versions prior to 3.3 must build with the internal blosc due to a lock
    # problem in a multithreaded environment.
    depends_on('c-blosc@1.4.1:', when='@3.3.0:')
    depends_on('zlib', when='+zlib')
    depends_on('bzip2', when='+bzip2')
    depends_on('lzo', when='+lzo')

    def setup_environment(self, spack_env, run_env):
        spack_env.set('HDF5_DIR', self.spec['hdf5'].prefix)
        if '+bzip2' in self.spec:
            spack_env.set('BZIP2_DIR', self.spec['bzip2'].prefix)
        if '+lzo' in self.spec:
            spack_env.set('LZO_DIR', self.spec['lzo'].prefix)
        if '^c-blosc' in self.spec:
            spack_env.set('BLOSC_DIR', self.spec['c-blosc'].prefix)
<|endoftext|>


<commit_before>from collections import defaultdict

from django.views.generic import TemplateView
from django.shortcuts import get_object_or_404

from popolo.models import Organization, Membership

from candidates.models import OrganizationExtra, PostExtra
from elections.mixins import ElectionMixin

class PartyListView(ElectionMixin, TemplateView):
    template_name = 'candidates/party-list.html'

    def get_context_data(self, **kwargs):
        context = super(PartyListView, self).get_context_data(**kwargs)
        context['parties_extra'] = OrganizationExtra.objects \
            .filter(
                base__memberships_on_behalf_of__extra__election=self.election_data,
                base__classification='Party'
            ).distinct().select_related('base').order_by('base__name')
        return context

def get_post_group_stats(posts):
    total = 0
    candidates = 0
    proportion = 0
    for post, members in posts.items():
        total += 1
        candidates += len(members)
    if total > 0:
        proportion = candidates / float(total)
    return {
        'proportion': proportion,
        'total': total,
        'candidates': candidates,
        'missing': total - candidates,
        'show_all': proportion > 0.3,
    }

class PartyDetailView(ElectionMixin, TemplateView):
    template_name = 'candidates/party.html'

    def get_context_data(self, **kwargs):
        context = super(PartyDetailView, self).get_context_data(**kwargs)
        party_id = kwargs['organization_id']
        party = get_object_or_404(Organization, extra__slug=party_id)

        # Make the party emblems conveniently available in the context too:
        context['emblems'] = [
            image for image in
            party.extra.images \
                .prefetch_related('extra').order_by('-is_primary')
        ]
        all_post_groups = PostExtra.objects.values_list('group', flat=True).distinct()
        by_post_group = {
            pg: {
                'stats': None,
                'posts_with_memberships': defaultdict(list)
            }
            for pg in all_post_groups
        }
        for membership in Membership.objects.filter(
            on_behalf_of=party,
            extra__election=self.election_data,
            role=self.election_data.candidate_membership_role
        ).select_related().prefetch_related('post__extra', 'person__extra'):
            person = membership.person
            post = membership.post
            post_group = post.extra.group
            by_post_group[post_group]['posts_with_memberships'][post].append({
                'membership': membership,
                'person': person,
                'post': post,
            })
        # That'll only find the posts that someone from the party is
        # actually standing for, so add any other posts...
        for post_extra in self.election_data.posts.select_related('base').all():
            post = post_extra.base
            post_group = post_extra.group
            post_group_data = by_post_group[post_group]
            posts_with_memberships = post_group_data['posts_with_memberships']
            posts_with_memberships.setdefault(post, [])
        context['party'] = party
        context['party_name'] = party.name
        for post_group, data in by_post_group.items():
            posts_with_memberships = data['posts_with_memberships']
            by_post_group[post_group]['stats'] = get_post_group_stats(
                posts_with_memberships
            )
            data['posts_with_memberships'] = sorted(
                posts_with_memberships.items(),
                key=lambda t: t[0].label,
            )
        context['candidates_by_post_group'] = sorted(
            [
                (pg, data) for pg, data
                in by_post_group.items()
                if pg in all_post_groups
            ],
            key=lambda k: k[0]
        )
        return context
<commit_msg>Only include posts from the right election in party pages<commit_after>from collections import defaultdict

from django.views.generic import TemplateView
from django.shortcuts import get_object_or_404

from popolo.models import Organization, Membership

from candidates.models import OrganizationExtra, PostExtra
from elections.mixins import ElectionMixin

class PartyListView(ElectionMixin, TemplateView):
    template_name = 'candidates/party-list.html'

    def get_context_data(self, **kwargs):
        context = super(PartyListView, self).get_context_data(**kwargs)
        context['parties_extra'] = OrganizationExtra.objects \
            .filter(
                base__memberships_on_behalf_of__extra__election=self.election_data,
                base__classification='Party'
            ).distinct().select_related('base').order_by('base__name')
        return context

def get_post_group_stats(posts):
    total = 0
    candidates = 0
    proportion = 0
    for post, members in posts.items():
        total += 1
        candidates += len(members)
    if total > 0:
        proportion = candidates / float(total)
    return {
        'proportion': proportion,
        'total': total,
        'candidates': candidates,
        'missing': total - candidates,
        'show_all': proportion > 0.3,
    }

class PartyDetailView(ElectionMixin, TemplateView):
    template_name = 'candidates/party.html'

    def get_context_data(self, **kwargs):
        context = super(PartyDetailView, self).get_context_data(**kwargs)
        party_id = kwargs['organization_id']
        party = get_object_or_404(Organization, extra__slug=party_id)

        # Make the party emblems conveniently available in the context too:
        context['emblems'] = [
            image for image in
            party.extra.images \
                .prefetch_related('extra').order_by('-is_primary')
        ]
        all_post_groups = self.election_data.posts.values_list('group', flat=True).distinct()
        by_post_group = {
            pg: {
                'stats': None,
                'posts_with_memberships': defaultdict(list)
            }
            for pg in all_post_groups
        }
        for membership in Membership.objects.filter(
            on_behalf_of=party,
            extra__election=self.election_data,
            role=self.election_data.candidate_membership_role
        ).select_related().prefetch_related('post__extra', 'person__extra'):
            person = membership.person
            post = membership.post
            post_group = post.extra.group
            by_post_group[post_group]['posts_with_memberships'][post].append({
                'membership': membership,
                'person': person,
                'post': post,
            })
        # That'll only find the posts that someone from the party is
        # actually standing for, so add any other posts...
        for post_extra in self.election_data.posts.select_related('base').all():
            post = post_extra.base
            post_group = post_extra.group
            post_group_data = by_post_group[post_group]
            posts_with_memberships = post_group_data['posts_with_memberships']
            posts_with_memberships.setdefault(post, [])
        context['party'] = party
        context['party_name'] = party.name
        for post_group, data in by_post_group.items():
            posts_with_memberships = data['posts_with_memberships']
            by_post_group[post_group]['stats'] = get_post_group_stats(
                posts_with_memberships
            )
            data['posts_with_memberships'] = sorted(
                posts_with_memberships.items(),
                key=lambda t: t[0].label,
            )
        context['candidates_by_post_group'] = sorted(
            [
                (pg, data) for pg, data
                in by_post_group.items()
                if pg in all_post_groups
            ],
            key=lambda k: k[0]
        )
        return context
<|endoftext|>


<commit_before>
import boto3
import os
import json

from base64 import b64decode
from botocore.exceptions import ClientError

#Use encrypted environment variables in Lambda, fallback to awscli creds on dev machine
try:
    ENCRYPTED_ACCESS_KEY = os.environ['ACCESS_KEY_ID']
    DECRYPTED_ACCESS_KEY = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_ACCESS_KEY))['Plaintext'].decode("utf-8")

    ENCRYPTED_SECRET_KEY = os.environ['SECRET_ACCESS_KEY']
    DECRYPTED_SECRET_KEY = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_SECRET_KEY))['Plaintext'].decode("utf-8")

    ec2 = boto3.resource('ec2', aws_access_key_id=DECRYPTED_ACCESS_KEY, aws_secret_access_key=DECRYPTED_SECRET_KEY)
except ClientError as ex:
    if ex.response['Error']['Code'] == 'InvalidCiphertextException':
        ec2 = boto3.resource('ec2')
except KeyError:
    ec2 = boto3.resource('ec2')

def get_num_instances():
    running_filter = {'Name': 'instance-state-name', 'Values': ['running']}
    instances = ec2.instances.filter(Filters=[running_filter])
    running_instances = [instance.id for instance in instances]
    output = {
                "dialogAction": {
                    "type": "Close",
                    "fulfillmentState": "Fulfilled",
                    "message": {
                        "contentType": "PlainText",
                        "content": "There are {0} instances running.".format(len(running_instances))
                    }
                }
            }
    print (output)
    return output

def lambda_handler(event, context):
    output = None
    if event["currentIntent"]["name"] == "RunningInstances":
        output = get_num_instances()
    if output is not None:
        return output
    else:
        return None

<commit_msg>Add instance status function<commit_after>
import boto3
import os
import json

from base64 import b64decode
from botocore.exceptions import ClientError

#Use encrypted environment variables in Lambda, fallback to awscli creds on dev machine
try:
    ENCRYPTED_ACCESS_KEY = os.environ['ACCESS_KEY_ID']
    DECRYPTED_ACCESS_KEY = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_ACCESS_KEY))['Plaintext'].decode("utf-8")

    ENCRYPTED_SECRET_KEY = os.environ['SECRET_ACCESS_KEY']
    DECRYPTED_SECRET_KEY = boto3.client('kms').decrypt(CiphertextBlob=b64decode(ENCRYPTED_SECRET_KEY))['Plaintext'].decode("utf-8")

    ec2 = boto3.resource('ec2', aws_access_key_id=DECRYPTED_ACCESS_KEY, aws_secret_access_key=DECRYPTED_SECRET_KEY)
except ClientError as ex:
    if ex.response['Error']['Code'] == 'InvalidCiphertextException':
        ec2 = boto3.resource('ec2')
except KeyError:
    ec2 = boto3.resource('ec2')

def get_num_instances():
    running_filter = {'Name': 'instance-state-name', 'Values': ['running']}
    instances = ec2.instances.filter(Filters=[running_filter])
    running_instances = [instance.id for instance in instances]
    output = {
                "dialogAction": {
                    "type": "Close",
                    "fulfillmentState": "Fulfilled",
                    "message": {
                        "contentType": "PlainText",
                        "content": "There are {0} instances running.".format(len(running_instances))
                    }
                }
            }
    print (output)
    return output

def get_instance_status():
    status = []
    instances = ec2.instances.filter(InstanceIds=[])
    for instance in instances:
        if instance.platform:
            platform = instance.platform
        else:
            platform = 'Linux'
        status.append("{0}, a {1} instance, is currently {2}.".format(instance.id, platform, instance.state["Name"]))

    output = {
                "dialogAction": {
                    "type": "Close",
                    "fulfillmentState": "Fulfilled",
                    "message": {
                        "contentType": "PlainText",
                        "content": ''.join(status)
                    }
                }
            }
    print (output)
    return output

def lambda_handler(event, context):
    output = None
    if event["currentIntent"]["name"] == "RunningInstances":
        output = get_num_instances()
    elif event["currentIntent"]["name"] == "InstanceStatus":
        output = get_instance_status()

    if output is not None:
        return output
    else:
        return None

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""This module allow to developers to get all users of GitHub that have a given
city in their profile. For example, if I want getting all users from London,
I will get all users that have London in their profiles (they could live in London or not)

Author: Israel Blancas @iblancasa
Original idea: https://github.com/JJ/github-city-rankings
License:

The MIT License (MIT)
    Copyright (c) 2015 Israel Blancas @iblancasa (http://iblancasa.com/)

    Permission is hereby granted, free of charge, to any person obtaining a copy of this software
    and associated documentation files (the “Software”), to deal in the Software without
    restriction, including without limitation the rights to use, copy, modify, merge, publish,
    distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom
    the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all copies or
    substantial portions of the Software.

    THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
    INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
    PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
    FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.

"""

import urllib.request
import threading
import datetime
import calendar
import time
import queue
import json
from urllib.error import HTTPError
from dateutil.relativedelta import relativedelta
import os
from multiprocessing import Lock
import sys
import logging
import coloredlogs
sys.path.append(os.getcwd())
from GitHubUser import *

class GitHubCity:
    """Manager of a GithubCity.

    Attributes:
        _city (str): Name of the city (private).
        _myusers (set): Name of all users in a city (private).
        _githubID (str): ID of your GitHub application.
        _githubSecret (str): secretGH of your GitHub application.
        _dataUsers (List[GitHubUser]): the list of GitHub users.
        _excluded (set): list of names of excluded users.
        _names (Queue): Queue with all users that we still have to process.
        _threads (set): Set of active Threads.
        _logger (logger): Logger.
    """

    def __init__(self, city, githubID, githubSecret, excludedJSON=None, debug=False):
        """Constructor of the class.

        Note:
            To get your ID and secret, you will need to create an application in
            https://github.com/settings/applications/new .

        Args:
            city (str): Name of the city you want to search about.
            githubID (str): ID of your GitHub application.
            githubSecret (str): Secret of your GitHub application.
            excludedJSON (dir): Excluded users of the ranking (see schemaExcluded.json)
            debug (bool): Show a log in your terminal. Default: False

        Returns:
            a new instance of GithubCity class

        """

        if city==None:
            raise Exception("City is not defined")

        self._city = city

        if githubID==None:
            raise Exception("No GitHub ID inserted")
        self._githubID = githubID

        if githubSecret==None:
            raise Exception("No GitHub Secret inserted")
        self._githubSecret = githubSecret

        self._names = queue.Queue()
        self._myusers = set()
        self._excluded = set()

        self._dataUsers = []

        self._threads = set()

        if excludedJSON:
            for e in excludedJSON:
                self._excluded.add(e["name"])

        self._logger = logging.getLogger("GitHubCity")

        if debug:
            coloredlogs.install(level='DEBUG')

        self._fin = False

        self._l = Lock()
        self._lo = Lock()

    def _addUser(self, new_user):
        """Add new users to the list (private).

        Note:
            This method is private.
            If the user is yet in the list (or in excluded users list), he/she will not be added

        Args:
            new_user (str): name of a GitHub user to include in the ranking

        """
        if not new_user in self._myusers and not new_user in self._excluded:
            self._myusers.add(new_user)
            myNewUser = GitHubUser(new_user)
            myNewUser.getData()
            self._dataUsers.append(myNewUser)
            self._logger.debug("NEW USER "+new_user+" "+str(len(self._dataUsers)+1)+" "+\
            threading.current_thread().name)

    def _readAPI(self, url):
        """Read a petition to the GitHub API (private).

        Note:
            This method is private.
            If max number of request is reached, method will stop 1 min.

        Args:
            url (str): URL to query.

        Returns:
            The response of the API -a dictionary with these fields-:
                * total_count (int): number of total users that match with the search
                * incomplete_results (bool): https://developer.github.com/v3/search/#timeouts-and-incomplete-results
                * items (List[dict]): a list with the users that match with the search
        """

        code = 0
        hdr = {'User-Agent': 'curl/7.43.0 (x86_64-ubuntu) libcurl/7.43.0 OpenSSL/1.0.1k zlib/1.2.8 gh-rankings-grx',
               'Accept': 'application/vnd.github.v3.text-match+json'
               }
        while code != 200:
            req = urllib.request.Request(url, headers=hdr)
            self._logger.debug("Getting data from "+url)
            try:
                response = urllib.request.urlopen(req)
                code = response.code
            except urllib.error.URLError as e:
                reset = int(e.getheader("X-RateLimit-Reset"))
                now_sec = calendar.timegm(datetime.datetime.utcnow().utctimetuple())
                self._logger.warning("Limit of API. Wait: "+str(reset - now_sec)+" secs")
                time.sleep(reset - now_sec)
                code = e.code

        data = json.loads(response.read().decode('utf-8'))
        response.close()
        return data

    def _getURL(self, page=1, start_date=None, final_date=None,order="asc"):
        """Get the API's URL to query to get data about users (private).

        Note:
            This method is private.

        Args:
            page (int): number of the page.
            start_date (datetime.date): start date of the range to search users.
            final_date (datetime.date): final date of the range to search users.
            order (str): order of the query. Valid values are 'asc' or 'desc'. Default: asc

        Returns:
            The URL (str) to query.

        """
        if not start_date or not final_date:
            url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \
                "&order=desc&q=sort:joined+type:user+location:" + self._city + \
                "&sort=joined&order=asc&per_page=100&page=" + str(page)
        else:
            url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \
                "&order=desc&q=sort:joined+type:user+location:" + self._city + \
                "+created:" + start_date.strftime("%Y-%m-%d") +\
                ".." + final_date.strftime("%Y-%m-%d") +\
                "&sort=joined&order="+order+"&per_page=100&page=" + str(page)

        return url

    def _processUsers(self):
        """Process users of the queue (get from the queue an add user) (private)

            Note:
                This method is private.

        """
        time.sleep(2)
        while not self._fin or not self._names.empty():
            self._l.acquire()
            try:
                new_user = self._names.get(False)
            except queue.Empty:
                self._l.release()
            else:
                self._l.release()
                self._addUser(new_user)
                self._logger.debug(str(self._names.qsize())+" users to process")

    def _launchThreads(self, numThreads):
        """Launch some threads and call to 'processUsers' (private)

        Note:
            This method is private.

        """
        i = 0
        while i<numThreads:
            i+=1
            newThr = threading.Thread(target=self._processUsers)
            newThr.setDaemon(True)
            self._threads.add(newThr)
            newThr.start()

    def _getPeriodUsers(self, start_date, final_date):
        """Get all the users given a period (private).

        Note:
            This method is private.
            User's names are added to the private _name attribute.

        Args:
            start_date (datetime.date): start date of the range to search users.
            final_date (datetime.date): final date of the range to search users.

        """
        self._logger.info("Getting users from "+start_date.strftime("%Y-%m-%d")+" to "+\
        final_date.strftime("%Y-%m-%d"))

        url = self._getURL(1, start_date, final_date)
        data = self._readAPI(url)

        total_pages = 10000
        page = 1

        while total_pages>=page:
            url = self._getURL(page, start_date, final_date)
            data = self._readAPI(url)
            for u in data['items']:
                self._names.put(u["login"])
            total_count = data["total_count"]
            total_pages = int(total_count / 100) + 1
            page += 1

    def getCityUsers(self):
        """Get all the users from the city.
        """
        self._fin = False
        self._threads = set()

        comprobationURL = self._getURL()
        comprobationData = self._readAPI(comprobationURL)

        self._launchThreads(20)

        for i in self._intervals:
            self._getPeriodUsers(i[0], i[1])

        self._fin = True

        for t in self._threads:
            t.join()

    def _validInterval(self, start, finish):
        """Given a valid interval, check if the interval is correct (less than 1000 users).
        If the interval is correct, it will be added to '_intervals' attribute. Else,
        interval will be split in two news intervals and these intervals will be
        checked.

        Args:
            start (datetime.date): start date of the interval
            finish (datetime.date): finish date of the interval

        Note:
            This method is private.
            Valid periods are added to the private _intervals attribute.

        """
        data = self._readAPI(self._getURL(1,start,finish))
        if data["total_count"]>=1000:
            middle = start + (finish - start)/2
            self._validInterval(start,middle)
            self._validInterval(middle,finish)
        else:
            self._intervals.add((start,finish))
            self._logger.debug("Valid interval: "+start.strftime("%Y-%m-%d")+" to "+\
            finish.strftime("%Y-%m-%d"))

    def calculateBestIntervals(self):
        """Calcules valid intervals of a city (with less than 1000 users)
        """
        self._intervals = None
        comprobation = self._readAPI(self._getURL())
        self._intervals = set()
        self._bigCity = True
        self._validInterval(datetime.date(2008, 1, 1), datetime.datetime.now().date())
        self._logger.info("Total number of intervals: "+ str(len(self._intervals)))

    def getTotalUsers(self):
        """Get the number of calculated users
        Returns:
            Number (int) of calculated users
        """
        return self._dataUsers
<commit_msg>Removing unused things<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""This module allow to developers to get all users of GitHub that have a given
city in their profile. For example, if I want getting all users from London,
I will get all users that have London in their profiles (they could live in London or not)

Author: Israel Blancas @iblancasa
Original idea: https://github.com/JJ/github-city-rankings
License:

The MIT License (MIT)
    Copyright (c) 2015 Israel Blancas @iblancasa (http://iblancasa.com/)

    Permission is hereby granted, free of charge, to any person obtaining a copy of this software
    and associated documentation files (the “Software”), to deal in the Software without
    restriction, including without limitation the rights to use, copy, modify, merge, publish,
    distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom
    the Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in all copies or
    substantial portions of the Software.

    THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
    INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
    PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
    FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
    ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
    IN THE SOFTWARE.

"""

import urllib.request
import threading
import datetime
import calendar
import time
import queue
import json
from urllib.error import HTTPError
from dateutil.relativedelta import relativedelta
import os
from multiprocessing import Lock
import sys
import logging
import coloredlogs
sys.path.append(os.getcwd())
from GitHubUser import *

class GitHubCity:
    """Manager of a GithubCity.

    Attributes:
        _city (str): Name of the city (private).
        _myusers (set): Name of all users in a city (private).
        _githubID (str): ID of your GitHub application.
        _githubSecret (str): secretGH of your GitHub application.
        _dataUsers (List[GitHubUser]): the list of GitHub users.
        _excluded (set): list of names of excluded users.
        _names (Queue): Queue with all users that we still have to process.
        _threads (set): Set of active Threads.
        _logger (logger): Logger.
        _l (Lock): lock to solve problems with threads.
    """

    def __init__(self, city, githubID, githubSecret, excludedJSON=None, debug=False):
        """Constructor of the class.

        Note:
            To get your ID and secret, you will need to create an application in
            https://github.com/settings/applications/new .

        Args:
            city (str): Name of the city you want to search about.
            githubID (str): ID of your GitHub application.
            githubSecret (str): Secret of your GitHub application.
            excludedJSON (dir): Excluded users of the ranking (see schemaExcluded.json)
            debug (bool): Show a log in your terminal. Default: False

        Returns:
            a new instance of GithubCity class

        """

        if city==None:
            raise Exception("City is not defined")

        self._city = city

        if githubID==None:
            raise Exception("No GitHub ID inserted")
        self._githubID = githubID

        if githubSecret==None:
            raise Exception("No GitHub Secret inserted")
        self._githubSecret = githubSecret

        self._names = queue.Queue()
        self._myusers = set()
        self._excluded = set()
        self._dataUsers = []
        self._threads = set()

        if excludedJSON:
            for e in excludedJSON:
                self._excluded.add(e["name"])

        self._logger = logging.getLogger("GitHubCity")

        if debug:
            coloredlogs.install(level='DEBUG')

        self._fin = False

        self._l = Lock()

    def _addUser(self, new_user):
        """Add new users to the list (private).

        Note:
            This method is private.
            If the user is yet in the list (or in excluded users list), he/she will not be added

        Args:
            new_user (str): name of a GitHub user to include in the ranking

        """
        if not new_user in self._myusers and not new_user in self._excluded:
            self._myusers.add(new_user)
            myNewUser = GitHubUser(new_user)
            myNewUser.getData()
            self._dataUsers.append(myNewUser)
            self._logger.debug("NEW USER "+new_user+" "+str(len(self._dataUsers)+1)+" "+\
            threading.current_thread().name)

    def _readAPI(self, url):
        """Read a petition to the GitHub API (private).

        Note:
            This method is private.
            If max number of request is reached, method will stop 1 min.

        Args:
            url (str): URL to query.

        Returns:
            The response of the API -a dictionary with these fields-:
                * total_count (int): number of total users that match with the search
                * incomplete_results (bool): https://developer.github.com/v3/search/#timeouts-and-incomplete-results
                * items (List[dict]): a list with the users that match with the search
        """

        code = 0
        hdr = {'User-Agent': 'curl/7.43.0 (x86_64-ubuntu) libcurl/7.43.0 OpenSSL/1.0.1k zlib/1.2.8 gh-rankings-grx',
               'Accept': 'application/vnd.github.v3.text-match+json'
               }
        while code != 200:
            req = urllib.request.Request(url, headers=hdr)
            self._logger.debug("Getting data from "+url)
            try:
                response = urllib.request.urlopen(req)
                code = response.code
            except urllib.error.URLError as e:
                reset = int(e.getheader("X-RateLimit-Reset"))
                now_sec = calendar.timegm(datetime.datetime.utcnow().utctimetuple())
                self._logger.warning("Limit of API. Wait: "+str(reset - now_sec)+" secs")
                time.sleep(reset - now_sec)
                code = e.code

        data = json.loads(response.read().decode('utf-8'))
        response.close()
        return data

    def _getURL(self, page=1, start_date=None, final_date=None,order="asc"):
        """Get the API's URL to query to get data about users (private).

        Note:
            This method is private.

        Args:
            page (int): number of the page.
            start_date (datetime.date): start date of the range to search users.
            final_date (datetime.date): final date of the range to search users.
            order (str): order of the query. Valid values are 'asc' or 'desc'. Default: asc

        Returns:
            The URL (str) to query.

        """
        if not start_date or not final_date:
            url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \
                "&order=desc&q=sort:joined+type:user+location:" + self._city + \
                "&sort=joined&order=asc&per_page=100&page=" + str(page)
        else:
            url = "https://api.github.com/search/users?client_id=" + self._githubID + "&client_secret=" + self._githubSecret + \
                "&order=desc&q=sort:joined+type:user+location:" + self._city + \
                "+created:" + start_date.strftime("%Y-%m-%d") +\
                ".." + final_date.strftime("%Y-%m-%d") +\
                "&sort=joined&order="+order+"&per_page=100&page=" + str(page)

        return url

    def _processUsers(self):
        """Process users of the queue (get from the queue an add user) (private)

            Note:
                This method is private.

        """
        while not self._fin or not self._names.empty():
            self._l.acquire()
            try:
                new_user = self._names.get(False)
            except queue.Empty:
                self._l.release()
            else:
                self._l.release()
                self._addUser(new_user)
                self._logger.debug(str(self._names.qsize())+" users to process")

    def _launchThreads(self, numThreads):
        """Launch some threads and call to 'processUsers' (private)

        Note:
            This method is private.

        """
        i = 0
        while i<numThreads:
            i+=1
            newThr = threading.Thread(target=self._processUsers)
            newThr.setDaemon(True)
            self._threads.add(newThr)
            newThr.start()

    def _getPeriodUsers(self, start_date, final_date):
        """Get all the users given a period (private).

        Note:
            This method is private.
            User's names are added to the private _name attribute.

        Args:
            start_date (datetime.date): start date of the range to search users.
            final_date (datetime.date): final date of the range to search users.

        """
        self._logger.info("Getting users from "+start_date.strftime("%Y-%m-%d")+" to "+\
        final_date.strftime("%Y-%m-%d"))

        url = self._getURL(1, start_date, final_date)
        data = self._readAPI(url)

        total_pages = 10000
        page = 1

        while total_pages>=page:
            url = self._getURL(page, start_date, final_date)
            data = self._readAPI(url)
            for u in data['items']:
                self._names.put(u["login"])
            total_count = data["total_count"]
            total_pages = int(total_count / 100) + 1
            page += 1

    def getCityUsers(self):
        """Get all the users from the city.
        """
        self._fin = False
        self._threads = set()

        comprobationURL = self._getURL()
        comprobationData = self._readAPI(comprobationURL)

        self._launchThreads(20)

        for i in self._intervals:
            self._getPeriodUsers(i[0], i[1])

        self._fin = True

        for t in self._threads:
            t.join()

    def _validInterval(self, start, finish):
        """Given a valid interval, check if the interval is correct (less than 1000 users).
        If the interval is correct, it will be added to '_intervals' attribute. Else,
        interval will be split in two news intervals and these intervals will be
        checked.

        Args:
            start (datetime.date): start date of the interval
            finish (datetime.date): finish date of the interval

        Note:
            This method is private.
            Valid periods are added to the private _intervals attribute.

        """
        data = self._readAPI(self._getURL(1,start,finish))
        if data["total_count"]>=1000:
            middle = start + (finish - start)/2
            self._validInterval(start,middle)
            self._validInterval(middle,finish)
        else:
            self._intervals.add((start,finish))
            self._logger.debug("Valid interval: "+start.strftime("%Y-%m-%d")+" to "+\
            finish.strftime("%Y-%m-%d"))

    def calculateBestIntervals(self):
        """Calcules valid intervals of a city (with less than 1000 users)
        """
        self._intervals = None
        comprobation = self._readAPI(self._getURL())
        self._intervals = set()
        self._bigCity = True
        self._validInterval(datetime.date(2008, 1, 1), datetime.datetime.now().date())
        self._logger.info("Total number of intervals: "+ str(len(self._intervals)))

    def getTotalUsers(self):
        """Get the number of calculated users
        Returns:
            Number (int) of calculated users
        """
        return len(self._dataUsers)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2014-2015 Matthias Klumpp <mak@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program.

import os
import glob
import shutil
import logging as log
import lmdb
from math import pow

def tobytes(s):
    if isinstance(s, bytes):
        return s
    return bytes(s, 'utf-8')

class DataCache:
    """ A LMDB based cache for the DEP-11 generator """

    def __init__(self, media_dir):
        self._pkgdb = None
        self._hintsdb = None
        self._datadb = None
        self._statsdb = None
        self._dbenv = None
        self.cache_dir = None
        self._opened = False

        self.media_dir = media_dir

        # set a huge map size to be futureproof.
        # This means we're cruel to non-64bit users, but this
        # software is supposed to be run on 64bit machines anyway.
        self._map_size = pow(1024, 4)

    def open(self, cachedir):
        self._dbenv = lmdb.open(cachedir, max_dbs=4, map_size=self._map_size)

        self._pkgdb = self._dbenv.open_db(b'packages')
        self._hintsdb = self._dbenv.open_db(b'hints')
        self._datadb = self._dbenv.open_db(b'metadata')
        self._statsdb = self._dbenv.open_db(b'statistics')

        self._opened = True
        self.cache_dir = cachedir
        return True

    def close(self):
        if not self._opened:
            return
        self._dbenv.close()

        self._pkgdb = None
        self._hintsdb = None
        self._datadb = None
        self._dbenv = None
        self._statsdb = None
        self._opened = False

    def reopen(self):
        if self._opened:
            return
        self.close()
        self.open(self.cache_dir)

    def metadata_exists(self, global_id):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb) as txn:
            return txn.get(gid) != None

    def get_metadata(self, global_id):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb) as dtxn:
                d = dtxn.get(tobytes(gid))
                if not d:
                    return None
                return str(d, 'utf-8')

    def set_metadata(self, global_id, yaml_data):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb, write=True) as txn:
            txn.put(gid, tobytes(yaml_data))

    def set_package_ignore(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
            txn.put(pkgid, b'ignore')

    def get_cpt_gids_for_pkg(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            cs_str = txn.get(pkgid)
            if not cs_str:
                return None
            cs_str = str(cs_str, 'utf-8')
            if cs_str == 'ignore' or cs_str == 'seen':
                return None
            gids = cs_str.split("\n")
            return gids

    def get_metadata_for_pkg(self, pkgid):
        gids = self.get_cpt_gids_for_pkg(pkgid)
        if not gids:
            return None

        data = ""
        for gid in gids:
            d = self.get_metadata(gid)
            if d:
                data += d
        return data

    def set_components(self, pkgid, cpts):
        # if the package has no components,
        # mark it as always-ignore
        if len(cpts) == 0:
            self.set_package_ignore(pkgid)
            return

        pkgid = tobytes(pkgid)

        gids = list()
        hints_str = ""
        for cpt in cpts:
            # check for ignore-reasons first, to avoid a database query
            if not cpt.has_ignore_reason():
                if self.metadata_exists(cpt.global_id):
                    gids.append(cpt.global_id)
                else:
                    # get the metadata in YAML format
                    md_yaml = cpt.to_yaml_doc()
                    # we need to check for ignore reasons again, since generating
                    # the YAML doc may have raised more errors
                    if not cpt.has_ignore_reason():
                        self.set_metadata(cpt.global_id, md_yaml)
                        gids.append(cpt.global_id)

            hints_yml = cpt.get_hints_yaml()
            if hints_yml:
                hints_str += hints_yml

        self.set_hints(pkgid, hints_str)
        if gids:
            with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
                txn.put(pkgid, bytes("\n".join(gids), 'utf-8'))
        elif hints_str:
            # we need to set some value for this package, to show that we've seen it
            with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
                txn.put(pkgid, b'seen')

    def get_hints(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._hintsdb) as txn:
            hints = txn.get(pkgid)
            if hints:
                hints = str(hints, 'utf-8')
            return hints

    def set_hints(self, pkgid, hints_yml):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._hintsdb, write=True) as txn:
            txn.put(pkgid, tobytes(hints_yml))

    def _cleanup_empty_dirs(self, d):
        parent = os.path.abspath(os.path.join(d, os.pardir))
        if not os.path.isdir(parent):
            return
        if not os.listdir(parent):
            os.rmdir(parent)
        parent = os.path.abspath(os.path.join(parent, os.pardir))
        if not os.path.isdir(parent):
            return
        if not os.listdir(parent):
            os.rmdir(parent)

    def remove_package(self, pkgid):
        log.debug("Dropping package: %s" % (pkgid))
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb, write=True) as pktxn:
            pktxn.delete(pkgid)
        with self._dbenv.begin(db=self._hintsdb, write=True) as htxn:
            htxn.delete(pkgid)

    def is_ignored(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            return txn.get(pkgid) == b'ignore'

    def package_exists(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            return txn.get(pkgid) != None

    def get_packages_not_in_set(self, pkgset):
        res = set()
        if not pkgset:
            pkgset = set()
        with self._dbenv.begin(db=self._pkgdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not str(key, 'utf-8') in pkgset:
                    res.add(key)
        return res

    def remove_orphaned_components(self):
        gid_pkg = dict()

        with self._dbenv.begin(db=self._pkgdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not value or value == b'ignore' or value == b'seen':
                    continue
                value = str(value, 'utf-8')
                gids = value.split("\n")
                for gid in gids:
                    if not gid_pkg.get(gid):
                        gid_pkg[gid] = list()
                    gid_pkg[gid].append(key)

        # remove the media and component data, if component is orphaned
        with self._dbenv.begin(db=self._datadb) as dtxn:
            cursor = dtxn.cursor()
            for gid, yaml in cursor:
                gid = str(gid, 'utf-8')

                # Check if we have a package which is still referencing this component
                pkgs = gid_pkg.get(gid)
                if pkgs:
                    continue

                # drop cached media
                dirs = glob.glob(os.path.join(self.media_dir, "*", gid))
                if dirs:
                    shutil.rmtree(dirs[0])
                    log.info("Expired media: %s" % (gid))
                    # remove possibly empty directories
                    self._cleanup_empty_dirs(dirs[0])

                # drop component from db
                with self._dbenv.begin(db=self._datadb, write=True) as dtxn:
                    dtxn.delete(tobytes(gid))

    def set_stats(self, timestamp, data):
        data = tobytes(data)
        tstamp = timestamp.to_bytes(10, byteorder='big')
        with self._dbenv.begin(db=self._statsdb, write=True) as txn:
            txn.put(tstamp, data)

    def get_stats(self):
        stats = dict()

        with self._dbenv.begin(db=self._statsdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not value:
                    continue
                value = str(value, 'utf-8')
                key = int.from_bytes(key, byteorder='big')
                stats[key] = value
        return stats
<commit_msg>Ensure we don't accumulate cruft in the media cache<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2014-2015 Matthias Klumpp <mak@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program.

import os
import glob
import shutil
import logging as log
import lmdb
from math import pow

def tobytes(s):
    if isinstance(s, bytes):
        return s
    return bytes(s, 'utf-8')

class DataCache:
    """ A LMDB based cache for the DEP-11 generator """

    def __init__(self, media_dir):
        self._pkgdb = None
        self._hintsdb = None
        self._datadb = None
        self._statsdb = None
        self._dbenv = None
        self.cache_dir = None
        self._opened = False

        self.media_dir = media_dir

        # set a huge map size to be futureproof.
        # This means we're cruel to non-64bit users, but this
        # software is supposed to be run on 64bit machines anyway.
        self._map_size = pow(1024, 4)

    def open(self, cachedir):
        self._dbenv = lmdb.open(cachedir, max_dbs=4, map_size=self._map_size)

        self._pkgdb = self._dbenv.open_db(b'packages')
        self._hintsdb = self._dbenv.open_db(b'hints')
        self._datadb = self._dbenv.open_db(b'metadata')
        self._statsdb = self._dbenv.open_db(b'statistics')

        self._opened = True
        self.cache_dir = cachedir
        return True

    def close(self):
        if not self._opened:
            return
        self._dbenv.close()

        self._pkgdb = None
        self._hintsdb = None
        self._datadb = None
        self._dbenv = None
        self._statsdb = None
        self._opened = False

    def reopen(self):
        if self._opened:
            return
        self.close()
        self.open(self.cache_dir)

    def metadata_exists(self, global_id):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb) as txn:
            return txn.get(gid) != None

    def get_metadata(self, global_id):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb) as dtxn:
                d = dtxn.get(tobytes(gid))
                if not d:
                    return None
                return str(d, 'utf-8')

    def set_metadata(self, global_id, yaml_data):
        gid = tobytes(global_id)
        with self._dbenv.begin(db=self._datadb, write=True) as txn:
            txn.put(gid, tobytes(yaml_data))

    def set_package_ignore(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
            txn.put(pkgid, b'ignore')

    def get_cpt_gids_for_pkg(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            cs_str = txn.get(pkgid)
            if not cs_str:
                return None
            cs_str = str(cs_str, 'utf-8')
            if cs_str == 'ignore' or cs_str == 'seen':
                return None
            gids = cs_str.split("\n")
            return gids

    def get_metadata_for_pkg(self, pkgid):
        gids = self.get_cpt_gids_for_pkg(pkgid)
        if not gids:
            return None

        data = ""
        for gid in gids:
            d = self.get_metadata(gid)
            if d:
                data += d
        return data

    def set_components(self, pkgid, cpts):
        # if the package has no components,
        # mark it as always-ignore
        if len(cpts) == 0:
            self.set_package_ignore(pkgid)
            return

        pkgid = tobytes(pkgid)

        gids = list()
        hints_str = ""
        for cpt in cpts:
            # check for ignore-reasons first, to avoid a database query
            if cpt.has_ignore_reason():
                # if we ignore the component, make sure the on-disk media cache
                # does not contain traces of it (e.g. from downloading data or
                # rendering screenshots before hitting an error)
                if self._remove_media_for_gid(cpt.global_id):
                    log.info("Removed cached media for: %s" % (cpt.global_id))
            else:
                if self.metadata_exists(cpt.global_id):
                    gids.append(cpt.global_id)
                else:
                    # get the metadata in YAML format
                    md_yaml = cpt.to_yaml_doc()
                    # we need to check for ignore reasons again, since generating
                    # the YAML doc may have raised more errors
                    if not cpt.has_ignore_reason():
                        self.set_metadata(cpt.global_id, md_yaml)
                        gids.append(cpt.global_id)

            hints_yml = cpt.get_hints_yaml()
            if hints_yml:
                hints_str += hints_yml

        self.set_hints(pkgid, hints_str)
        if gids:
            with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
                txn.put(pkgid, bytes("\n".join(gids), 'utf-8'))
        elif hints_str:
            # we need to set some value for this package, to show that we've seen it
            with self._dbenv.begin(db=self._pkgdb, write=True) as txn:
                txn.put(pkgid, b'seen')

    def get_hints(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._hintsdb) as txn:
            hints = txn.get(pkgid)
            if hints:
                hints = str(hints, 'utf-8')
            return hints

    def set_hints(self, pkgid, hints_yml):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._hintsdb, write=True) as txn:
            txn.put(pkgid, tobytes(hints_yml))

    def _cleanup_empty_dirs(self, d):
        parent = os.path.abspath(os.path.join(d, os.pardir))
        if not os.path.isdir(parent):
            return
        if not os.listdir(parent):
            os.rmdir(parent)
        parent = os.path.abspath(os.path.join(parent, os.pardir))
        if not os.path.isdir(parent):
            return
        if not os.listdir(parent):
            os.rmdir(parent)

    def remove_package(self, pkgid):
        log.debug("Dropping package: %s" % (pkgid))
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb, write=True) as pktxn:
            pktxn.delete(pkgid)
        with self._dbenv.begin(db=self._hintsdb, write=True) as htxn:
            htxn.delete(pkgid)

    def is_ignored(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            return txn.get(pkgid) == b'ignore'

    def package_exists(self, pkgid):
        pkgid = tobytes(pkgid)
        with self._dbenv.begin(db=self._pkgdb) as txn:
            return txn.get(pkgid) != None

    def get_packages_not_in_set(self, pkgset):
        res = set()
        if not pkgset:
            pkgset = set()
        with self._dbenv.begin(db=self._pkgdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not str(key, 'utf-8') in pkgset:
                    res.add(key)
        return res

    def _remove_media_for_gid(self, gid):
        if not self.media_dir:
            return False
        dirs = glob.glob(os.path.join(self.media_dir, "*", gid))
        if dirs:
            shutil.rmtree(dirs[0])
            # remove possibly empty directories
            self._cleanup_empty_dirs(dirs[0])
            return True

    def remove_orphaned_components(self):
        gid_pkg = dict()

        with self._dbenv.begin(db=self._pkgdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not value or value == b'ignore' or value == b'seen':
                    continue
                value = str(value, 'utf-8')
                gids = value.split("\n")
                for gid in gids:
                    if not gid_pkg.get(gid):
                        gid_pkg[gid] = list()
                    gid_pkg[gid].append(key)

        # remove the media and component data, if component is orphaned
        with self._dbenv.begin(db=self._datadb) as dtxn:
            cursor = dtxn.cursor()
            for gid, yaml in cursor:
                gid = str(gid, 'utf-8')

                # Check if we have a package which is still referencing this component
                pkgs = gid_pkg.get(gid)
                if pkgs:
                    continue

                # drop cached media
                if self._remove_media_for_gid(gid):
                    log.info("Expired media: %s" % (gid))

                # drop component from db
                with self._dbenv.begin(db=self._datadb, write=True) as dtxn:
                    dtxn.delete(tobytes(gid))

    def set_stats(self, timestamp, data):
        data = tobytes(data)
        tstamp = timestamp.to_bytes(10, byteorder='big')
        with self._dbenv.begin(db=self._statsdb, write=True) as txn:
            txn.put(tstamp, data)

    def get_stats(self):
        stats = dict()

        with self._dbenv.begin(db=self._statsdb) as txn:
            cursor = txn.cursor()
            for key, value in cursor:
                if not value:
                    continue
                value = str(value, 'utf-8')
                key = int.from_bytes(key, byteorder='big')
                stats[key] = value
        return stats
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

# Etalab-CKAN-Harvesters -- Harvesters for Etalab's CKAN
# By: Emmanuel Raviart <emmanuel@raviart.com>
#
# Copyright (C) 2013 Emmanuel Raviart
# http://github.com/etalab/etalab-ckan-harvesters
#
# This file is part of Etalab-CKAN-Harvesters.
#
# Etalab-CKAN-Harvesters is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Etalab-CKAN-Harvesters is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Helpers for harvesters"""

def get_extra(instance, key, default = UnboundLocalError):
    for extra in (instance.get('extras') or []):
        if extra['key'] == key:
            return extra.get('value')
    if default is UnboundLocalError:
        raise KeyError(key)
    return default

def pop_extra(instance, key, default = UnboundLocalError):
    for index, extra in enumerate(instance.get('extras') or []):
        if extra['key'] == key:
            del instance['extras'][index]
            return extra.get('value')
    if default is UnboundLocalError:
        raise KeyError(key)
    return default

def set_extra(instance, key, value):
    if instance.get('extras') is None:
        instance['extras'] = []
    for extra in instance['extras']:
        if extra['key'] == key:
            extra['value'] = value
            return
    instance['extras'].append(dict(
        key = key,
        value = value,
        ))

<commit_msg>Code cleanup.<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

# Etalab-CKAN-Harvesters -- Harvesters for Etalab's CKAN
# By: Emmanuel Raviart <emmanuel@raviart.com>
#
# Copyright (C) 2013 Emmanuel Raviart
# http://github.com/etalab/etalab-ckan-harvesters
#
# This file is part of Etalab-CKAN-Harvesters.
#
# Etalab-CKAN-Harvesters is free software; you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# Etalab-CKAN-Harvesters is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""Helpers for harvesters"""

def get_extra(instance, key, default = UnboundLocalError):
    for extra in (instance.get('extras') or []):
        if extra['key'] == key:
            return extra.get('value')
    if default is UnboundLocalError:
        raise KeyError(key)
    return default

def pop_extra(instance, key, default = UnboundLocalError):
    for index, extra in enumerate(instance.get('extras') or []):
        if extra['key'] == key:
            del instance['extras'][index]
            return extra.get('value')
    if default is UnboundLocalError:
        raise KeyError(key)
    return default

def set_extra(instance, key, value):
    if instance.get('extras') is None:
        instance['extras'] = []
    for extra in instance['extras']:
        if extra['key'] == key:
            extra['value'] = value
            return
    instance['extras'].append(dict(
        key = key,
        value = value,
        ))
<|endoftext|>


<commit_before>from ckan_api_client.utils import WrappedList

from .base import BaseObject
from .fields import (StringField, GroupsField, ExtrasField, ListField,
                     BoolField, IntegerField)

__all__ = ['ResourcesField', 'CkanDataset', 'CkanResource']

class ResourcesField(ListField):
    """
    The ResourcesField should behave pretty much as a list field,
    but will keep track of changes, and make sure all elements
    are CkanResources.
    """

    def validate(self, instance, name, value):
        if isinstance(value, ResourcesList):
            return value
        return ResourcesList(value)

    def serialize(self, instance, name):
        value = self.get(instance, name)
        assert isinstance(value, ResourcesList)
        return [
            r.serialize() for r in value
        ]

    def is_equivalent(self, instance, name, other, ignore_key=True):
        ## We are now comparing two ResourcesList instances,
        ## but we need to ignore all the key fields when comparing
        ## resources

        our_value = getattr(instance, name)
        other_value = getattr(other, name)
        assert isinstance(our_value, ResourcesList)
        assert isinstance(other_value, ResourcesList)

        ## Different length -- clearly two different things
        if len(our_value) != len(other_value):
            return False

        ## Compare resources one-by-one, by calling their "is_equivalent"
        ## methods.
        for resource1, resource2 in zip(our_value, other_value):
            if not resource1.is_equivalent(resource2, ignore_key=ignore_key):
                return False

        return True

class ResourcesList(WrappedList):
    def __init__(self, initial=None):
        if initial is None:
            super(ResourcesList, self).__init__()
        else:
            super(ResourcesList, self).__init__(
                self._check_item(item) for item in initial)

    def _check_item(self, value):
        if isinstance(value, dict):
            return CkanResource(value)
        if not isinstance(value, CkanResource):
            raise TypeError("Invalid resource. Must be a CkanResource "
                            "or a dict, got {0!r} instead."
                            .format(type(value)))
        return value

    def __setitem__(self, name, value):
        value = self._check_item(value)
        super(ResourcesList, self).__setitem__(name, value)

    def insert(self, pos, item):
        item = self._check_item(item)
        return super(ResourcesList, self).insert(pos, item)

    def __contains__(self, item):
        try:
            item = self._check_item(item)
        except TypeError:
            ## Invalid type -- cannot be contained
            return False
        return super(ResourcesList, self).__contains__(item)

class CkanDataset(BaseObject):
    id = StringField()

    ## Core fields
    name = StringField()
    title = StringField()

    author = StringField()
    author_email = StringField()
    license_id = StringField()
    maintainer = StringField()
    maintainer_email = StringField()
    notes = StringField()
    owner_org = StringField()
    private = BoolField(default=False)
    state = StringField(default='active')
    type = StringField(default='dataset')
    url = StringField()

    ## Special fields
    extras = ExtrasField()
    groups = GroupsField()
    resources = ResourcesField()
    # relationships = ListField()

class CkanResource(BaseObject):
    id = StringField()

    description = StringField()
    format = StringField()
    mimetype = StringField()
    mimetype_inner = StringField()
    name = StringField()
    position = StringField()
    resource_type = StringField()
    size = StringField()
    url = StringField()
    url_type = StringField()

    def __eq__(self, other):
        if isinstance(other, dict):
            return self == CkanResource(other)
        return super(CkanResource, self).__eq__(other)
<commit_msg>[#18] id field of dataset/resource now marked as key<commit_after>from ckan_api_client.utils import WrappedList

from .base import BaseObject
from .fields import (StringField, GroupsField, ExtrasField, ListField,
                     BoolField, IntegerField)

__all__ = ['ResourcesField', 'CkanDataset', 'CkanResource']

class ResourcesField(ListField):
    """
    The ResourcesField should behave pretty much as a list field,
    but will keep track of changes, and make sure all elements
    are CkanResources.
    """

    def validate(self, instance, name, value):
        if isinstance(value, ResourcesList):
            return value
        return ResourcesList(value)

    def serialize(self, instance, name):
        value = self.get(instance, name)
        assert isinstance(value, ResourcesList)
        return [
            r.serialize() for r in value
        ]

    def is_equivalent(self, instance, name, other, ignore_key=True):
        ## We are now comparing two ResourcesList instances,
        ## but we need to ignore all the key fields when comparing
        ## resources

        our_value = getattr(instance, name)
        other_value = getattr(other, name)
        assert isinstance(our_value, ResourcesList)
        assert isinstance(other_value, ResourcesList)

        ## Different length -- clearly two different things
        if len(our_value) != len(other_value):
            return False

        ## Compare resources one-by-one, by calling their "is_equivalent"
        ## methods.
        for resource1, resource2 in zip(our_value, other_value):
            if not resource1.is_equivalent(resource2, ignore_key=ignore_key):
                return False

        return True

class ResourcesList(WrappedList):
    def __init__(self, initial=None):
        if initial is None:
            super(ResourcesList, self).__init__()
        else:
            super(ResourcesList, self).__init__(
                self._check_item(item) for item in initial)

    def _check_item(self, value):
        if isinstance(value, dict):
            return CkanResource(value)
        if not isinstance(value, CkanResource):
            raise TypeError("Invalid resource. Must be a CkanResource "
                            "or a dict, got {0!r} instead."
                            .format(type(value)))
        return value

    def __setitem__(self, name, value):
        value = self._check_item(value)
        super(ResourcesList, self).__setitem__(name, value)

    def insert(self, pos, item):
        item = self._check_item(item)
        return super(ResourcesList, self).insert(pos, item)

    def __contains__(self, item):
        try:
            item = self._check_item(item)
        except TypeError:
            ## Invalid type -- cannot be contained
            return False
        return super(ResourcesList, self).__contains__(item)

class CkanDataset(BaseObject):
    id = StringField(is_key=True)

    ## Core fields
    name = StringField()
    title = StringField()

    author = StringField()
    author_email = StringField()
    license_id = StringField()
    maintainer = StringField()
    maintainer_email = StringField()
    notes = StringField()
    owner_org = StringField()
    private = BoolField(default=False)
    state = StringField(default='active')
    type = StringField(default='dataset')
    url = StringField()

    ## Special fields
    extras = ExtrasField()
    groups = GroupsField()
    resources = ResourcesField()
    # relationships = ListField()

class CkanResource(BaseObject):
    id = StringField(is_key=True)

    description = StringField()
    format = StringField()
    mimetype = StringField()
    mimetype_inner = StringField()
    name = StringField()
    position = StringField()
    resource_type = StringField()
    size = StringField()
    url = StringField()
    url_type = StringField()

    def __eq__(self, other):
        if isinstance(other, dict):
            return self == CkanResource(other)
        return super(CkanResource, self).__eq__(other)
<|endoftext|>


<commit_before>from argparse import ArgumentParser

from .eval_model import evaluate_model

parser = ArgumentParser(description='eval_kinetic_models: Evaluate '
                                    'performance of kinetic models using '
                                    'experimental ignition delay data.'
                        )
parser.add_argument('-m', '--model',
                    type=str,
                    required=True,
                    help='Input model filename (e.g., mech.cti).'
                    )
parser.add_argument('-k', '--model-keys',
                    type=str,
                    dest='model_keys_file',
                    required=True,
                    help='JSON file with keys for species in models.'
                    )
parser.add_argument('-d', '--dataset',
                    type=str,
                    required=True,
                    help='Filename for list of datasets.'
                    )
parser.add_argument('-dp', '--data-path',
                    type=str,
                    dest='data_path',
                    required=False,
                    default='data',
                    help='Local directory holding dataset files.'
                    )
parser.add_argument('-mp', '--model-path',
                    type=str,
                    dest='model_path',
                    required=False,
                    default='models',
                    help='Local directory holding model files.'
                    )
parser.add_argument('-rp', '--results-path',
                    type=str,
                    dest='results_path',
                    required=False,
                    default='results',
                    help='Local directory holding result HDF5 files.'
                    )
parser.add_argument('-v', '--model-variant',
                    type=str,
                    dest='model_variant_file',
                    required=False,
                    help='JSON with variants for models for, e.g., bath '
                         'gases and pressures.'
                    )
parser.add_argument('-nt', '--num-threads',
                    type=int,
                    dest='num_threads',
                    default=multiprocessing.cpu_count()-1 or 1,
                    required=False,
                    help='The number of threads to use to run simulations in
                         'parallel.'
                    )
args = parser.parse_args()

evaluate_model(args.model, args.model_keys_file, args.dataset,
               args.data_path, args.model_path, args.results_path,
               args.model_variant_file, args.num_threads
               )
<commit_msg>added (somehow) missing end quote in __main__<commit_after>from argparse import ArgumentParser

from .eval_model import evaluate_model

parser = ArgumentParser(description='eval_kinetic_models: Evaluate '
                                    'performance of kinetic models using '
                                    'experimental ignition delay data.'
                        )
parser.add_argument('-m', '--model',
                    type=str,
                    required=True,
                    help='Input model filename (e.g., mech.cti).'
                    )
parser.add_argument('-k', '--model-keys',
                    type=str,
                    dest='model_keys_file',
                    required=True,
                    help='JSON file with keys for species in models.'
                    )
parser.add_argument('-d', '--dataset',
                    type=str,
                    required=True,
                    help='Filename for list of datasets.'
                    )
parser.add_argument('-dp', '--data-path',
                    type=str,
                    dest='data_path',
                    required=False,
                    default='data',
                    help='Local directory holding dataset files.'
                    )
parser.add_argument('-mp', '--model-path',
                    type=str,
                    dest='model_path',
                    required=False,
                    default='models',
                    help='Local directory holding model files.'
                    )
parser.add_argument('-rp', '--results-path',
                    type=str,
                    dest='results_path',
                    required=False,
                    default='results',
                    help='Local directory holding result HDF5 files.'
                    )
parser.add_argument('-v', '--model-variant',
                    type=str,
                    dest='model_variant_file',
                    required=False,
                    help='JSON with variants for models for, e.g., bath '
                         'gases and pressures.'
                    )
parser.add_argument('-nt', '--num-threads',
                    type=int,
                    dest='num_threads',
                    default=multiprocessing.cpu_count()-1 or 1,
                    required=False,
                    help='The number of threads to use to run simulations in '
                         'parallel.'
                    )
args = parser.parse_args()

evaluate_model(args.model, args.model_keys_file, args.dataset,
               args.data_path, args.model_path, args.results_path,
               args.model_variant_file, args.num_threads
               )
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libsharp(AutotoolsPackage):
    """Libsharp is a code library for spherical harmonic transforms (SHTs) and
    spin-weighted spherical harmonic transforms, which evolved from the libpsht
    library."""

    variant('openmp', default=True, description='Build with openmp support')
    variant('mpi', default=True, description='Build with MPI support')

    homepage = "https://github.com/Libsharp/libsharp"
    git      = "https://github.com/Libsharp/libsharp.git"

    version('1.0.0', commit='cc4753ff4b0ef393f0d4ada41a175c6d1dd85d71', preferred=True)
    version('2018-01-17', commit='593d4eba67d61827191c32fb94bf235cb31205e1')

    depends_on('autoconf', type='build')
    depends_on('mpi', when='+mpi')

    patch('arm.patch', when='@2018-01-17 target=aarch64:')
    patch('1.0.0-arm.patch', when='@1.0.0 target=aarch64:')

    def autoreconf(self, spec, prefix):
        """Generate autotools configuration"""
        bash = which('bash')
        bash('autoconf')

    def configure_args(self):
        args = []
        if '+openmp' not in self.spec:
            args.append("--disable-openmp")
        if '+mpi' not in self.spec:
            args.append("--disable-mpi")
        return args

    def install(self, spec, prefix):
        install_tree('auto/include', join_path(prefix, 'include'))
        install_tree('auto/lib', join_path(prefix, 'lib'))
<commit_msg>Install libsharp header files to include/libsharp. (#16847)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class Libsharp(AutotoolsPackage):
    """Libsharp is a code library for spherical harmonic transforms (SHTs) and
    spin-weighted spherical harmonic transforms, which evolved from the libpsht
    library."""

    variant('openmp', default=True, description='Build with openmp support')
    variant('mpi', default=True, description='Build with MPI support')

    homepage = "https://github.com/Libsharp/libsharp"
    git      = "https://github.com/Libsharp/libsharp.git"

    version('1.0.0', commit='cc4753ff4b0ef393f0d4ada41a175c6d1dd85d71', preferred=True)
    version('2018-01-17', commit='593d4eba67d61827191c32fb94bf235cb31205e1')

    depends_on('autoconf', type='build')
    depends_on('mpi', when='+mpi')

    patch('arm.patch', when='@2018-01-17 target=aarch64:')
    patch('1.0.0-arm.patch', when='@1.0.0 target=aarch64:')

    def autoreconf(self, spec, prefix):
        """Generate autotools configuration"""
        bash = which('bash')
        bash('autoconf')

    def configure_args(self):
        args = []
        if '+openmp' not in self.spec:
            args.append("--disable-openmp")
        if '+mpi' not in self.spec:
            args.append("--disable-mpi")
        return args

    def install(self, spec, prefix):
        # Libsharp's only caller healpix include headers like 'libsharp/xxx.h'
        # Install xxx.h to include/libsharp
        install_tree('auto/include', prefix.include.libsharp)
        install_tree('auto/lib', prefix.lib)
<|endoftext|>