<commit_before># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from flask import flash, request, session, jsonify
from sqlalchemy.orm import joinedload, subqueryload
from werkzeug.exceptions import Forbidden

from indico.modules.events.abstracts.controllers.base import RHAbstractBase, RHAbstractsBase
from indico.modules.events.abstracts.controllers.common import (DisplayAbstractListMixin, CustomizeAbstractListMixin,
                                                                AbstractsExportPDFMixin, AbstractsExportCSV,
                                                                AbstractsExportExcel, AbstractsDownloadAttachmentsMixin)
from indico.modules.events.abstracts.forms import (AbstractCommentForm, AbstractJudgmentForm,
                                                   AbstractReviewedForTracksForm, build_review_form)
from indico.modules.events.abstracts.lists import AbstractListGeneratorDisplay
from indico.modules.events.abstracts.models.abstracts import Abstract, AbstractState
from indico.modules.events.abstracts.models.comments import AbstractComment
from indico.modules.events.abstracts.models.reviews import AbstractReview
from indico.modules.events.abstracts.operations import (judge_abstract, reset_abstract_state, withdraw_abstract,
                                                        create_abstract_comment, delete_abstract_comment,
                                                        update_abstract_comment, create_abstract_review,
                                                        update_abstract_review, update_reviewed_for_tracks)
from indico.modules.events.abstracts.util import get_track_reviewer_abstract_counts, get_user_tracks
from indico.modules.events.abstracts.views import WPDisplayAbstractsReviewing, render_abstract_page
from indico.modules.events.tracks.models.tracks import Track
from indico.util.i18n import _
from indico.web.flask.templating import get_template_module
from indico.web.util import jsonify_data, jsonify_template

class RHListOtherAbstracts(RHAbstractsBase):
    """AJAX endpoint that lists all abstracts in the event (dict representation)."""

    def _checkProtection(self):
        if not session.user:
            raise Forbidden
        RHAbstractsBase._checkProtection(self)

    def _checkParams(self, params):
        RHAbstractsBase._checkParams(self, params)
        self.excluded_ids = set(request.form.getlist('excluded_abstract_id'))

    def _process(self):
        query = (Abstract.query
                 .with_parent(self.event_new)
                 .filter(Abstract.state.notin_({AbstractState.duplicate, AbstractState.merged}))
                 .options(joinedload('submitter').lazyload('*'),
                          subqueryload('reviewed_for_tracks'),
                          subqueryload('person_links').joinedload('person').joinedload('user'))
                 .order_by(Abstract.friendly_id))

        if self.excluded_ids:
            query = query.filter(Abstract.id.notin_(self.excluded_ids))

        result = [{'id': abstract.id, 'friendly_id': abstract.friendly_id, 'title': abstract.title,
                   'full_title': '#{}: {}'.format(abstract.friendly_id, abstract.title)}
                  for abstract in query
                  if abstract.can_access(session.user)]
        return jsonify(result)

class RHJudgeAbstract(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_judge(session.user, check_state=True)

    def _process(self):
        form = AbstractJudgmentForm(abstract=self.abstract)
        if form.validate_on_submit():
            judgment_data, abstract_data = form.split_data
            judge_abstract(self.abstract, abstract_data, judge=session.user, **judgment_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/judge.html')
        return jsonify(html=tpl.render_decision_box(self.abstract, form))

class RHResetAbstractState(RHAbstractBase):
    def _check_abstract_protection(self):
        if self.abstract.state == AbstractState.submitted:
            return False
        # manages can always reset
        if self.event_new.can_manage(session.user):
            return True
        # judges can reset if the abstract has not been withdrawn
        return self.abstract.can_judge(session.user) and self.abstract.state != AbstractState.withdrawn

    def _process(self):
        if self.abstract.state != AbstractState.submitted:
            reset_abstract_state(self.abstract)
            flash(_("Abstract state has been reset"), 'success')
        return jsonify_data(html=render_abstract_page(self.abstract, management=self.management))

class RHWithdrawAbstract(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_withdraw(session.user, check_state=True)

    def _process(self):
        if self.abstract.state != AbstractState.withdrawn:
            withdraw_abstract(self.abstract)
            flash(_("Abstract has been withdrawn"), 'success')
        return jsonify_data(html=render_abstract_page(self.abstract, management=self.management))

class RHDisplayAbstractListBase(RHAbstractsBase):
    """Base class for all abstract list operations"""

    normalize_url_spec = {
        'locators': {
            lambda self: self.track
        }
    }

    def _checkParams(self, params):
        RHAbstractsBase._checkParams(self, params)
        self.track = Track.get_one(request.view_args['track_id'])
        self.list_generator = AbstractListGeneratorDisplay(event=self.event_new, track=self.track)

    def _checkProtection(self):
        if not self.track.can_review_abstracts(session.user) and not self.track.can_convene(session.user):
            raise Forbidden

class RHSubmitAbstractReview(RHAbstractBase):
    """Review an abstract in a specific track"""

    normalize_url_spec = {
        'locators': {
            lambda self: self.abstract,
            lambda self: self.track
        }
    }

    def _check_abstract_protection(self):
        if self.abstract.get_reviews(user=session.user, track=self.track):
            return False
        return self.abstract.can_review(session.user, check_state=True)

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.track = Track.get_one(request.view_args['track_id'])

    def _process(self):
        form = build_review_form(self.abstract, self.track)
        if form.validate_on_submit():
            create_abstract_review(self.abstract, self.track, session.user, **form.split_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/review.html')
        return jsonify(html=tpl.render_review_form(form, self.abstract, self.track))

class RHEditAbstractReview(RHAbstractBase):
    normalize_url_spec = {
        'locators': {
            lambda self: self.review
        }
    }

    def _check_abstract_protection(self):
        return self.review.can_edit(session.user, check_state=True)

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.review = AbstractReview.get_one(request.view_args['review_id'])

    def _process(self):
        form = build_review_form(review=self.review)
        if form.validate_on_submit():
            update_abstract_review(self.review, **form.split_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/review.html')
        return jsonify(html=tpl.render_review_form(form, review=self.review))

class RHSubmitAbstractComment(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_comment(session.user)

    def _process(self):
        form = AbstractCommentForm(abstract=self.abstract, user=session.user)
        if form.validate_on_submit():
            create_abstract_comment(self.abstract, form.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/review.html')
        return jsonify(html=tpl.render_comment_form(form, self.abstract))

class RHAbstractCommentBase(RHAbstractBase):
    normalize_url_spec = {
        'locators': {
            lambda self: self.comment
        }
    }

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.comment = AbstractComment.get_one(request.view_args['comment_id'])

    def _checkProtection(self):
        RHAbstractBase._checkProtection(self)
        if not self.comment.can_edit(session.user):
            raise Forbidden

class RHEditAbstractComment(RHAbstractCommentBase):
    def _process(self):
        form = AbstractCommentForm(obj=self.comment, abstract=self.abstract, user=session.user,
                                   prefix='edit-comment-{}-'.format(self.comment.id))
        if form.validate_on_submit():
            update_abstract_comment(self.comment, form.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/review.html')
        return jsonify(html=tpl.render_comment_form(form, self.abstract, comment=self.comment))

class RHDeleteAbstractComment(RHAbstractCommentBase):
    def _process(self):
        delete_abstract_comment(self.comment)
        return jsonify_data(flash=False)

class RHDisplayReviewableTracks(RHAbstractsBase):
    def _checkProtection(self):
        if not session.user:
            raise Forbidden
        RHAbstractsBase._checkProtection(self)

    def _process(self):
        track_reviewer_abstract_count = get_track_reviewer_abstract_counts(self.event_new, session.user)
        return WPDisplayAbstractsReviewing.render_template('display/tracks.html', self._conf, event=self.event_new,
                                                           abstract_count=track_reviewer_abstract_count,
                                                           tracks=get_user_tracks(self.event_new, session.user))

class RHDisplayReviewableTrackAbstracts(DisplayAbstractListMixin, RHDisplayAbstractListBase):
    view_class = WPDisplayAbstractsReviewing
    template = 'display/abstracts.html'

    def _render_template(self, **kwargs):
        return DisplayAbstractListMixin._render_template(self, track=self.track, **kwargs)

class RHDisplayAbstractListCustomize(CustomizeAbstractListMixin, RHDisplayAbstractListBase):
    view_class = WPDisplayAbstractsReviewing

class RHDisplayAbstractsActionsBase(RHDisplayAbstractListBase):
    """Base class for classes performing actions on abstract"""

    def _checkParams(self, params):
        RHDisplayAbstractListBase._checkParams(self, params)
        ids = map(int, request.form.getlist('abstract_id'))
        self.abstracts = Abstract.query.with_parent(self.track, 'abstracts_reviewed').filter(Abstract.id.in_(ids)).all()

class RHDisplayAbstractsDownloadAttachments(AbstractsDownloadAttachmentsMixin, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportPDF(AbstractsExportPDFMixin, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportCSV(AbstractsExportCSV, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportExcel(AbstractsExportExcel, RHDisplayAbstractsActionsBase):
    pass

class RHEditReviewedForTrackList(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_judge(session.user, check_state=True)

    def _process(self):
        form = AbstractReviewedForTracksForm(event=self.event_new, obj=self.abstract)
        if form.validate_on_submit():
            update_reviewed_for_tracks(self.abstract, form.reviewed_for_tracks.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        return jsonify_template('events/abstracts/forms/edit_review_tracks.html', form=form)
<commit_msg>Fix abstract reviewing/commenting<commit_after># This file is part of Indico.
# Copyright (C) 2002 - 2016 European Organization for Nuclear Research (CERN).
#
# Indico is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 3 of the
# License, or (at your option) any later version.
#
# Indico is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Indico; if not, see <http://www.gnu.org/licenses/>.

from __future__ import unicode_literals

from flask import flash, request, session, jsonify
from sqlalchemy.orm import joinedload, subqueryload
from werkzeug.exceptions import Forbidden

from indico.modules.events.abstracts.controllers.base import RHAbstractBase, RHAbstractsBase
from indico.modules.events.abstracts.controllers.common import (DisplayAbstractListMixin, CustomizeAbstractListMixin,
                                                                AbstractsExportPDFMixin, AbstractsExportCSV,
                                                                AbstractsExportExcel, AbstractsDownloadAttachmentsMixin)
from indico.modules.events.abstracts.forms import (AbstractCommentForm, AbstractJudgmentForm,
                                                   AbstractReviewedForTracksForm, build_review_form)
from indico.modules.events.abstracts.lists import AbstractListGeneratorDisplay
from indico.modules.events.abstracts.models.abstracts import Abstract, AbstractState
from indico.modules.events.abstracts.models.comments import AbstractComment
from indico.modules.events.abstracts.models.reviews import AbstractReview
from indico.modules.events.abstracts.operations import (judge_abstract, reset_abstract_state, withdraw_abstract,
                                                        create_abstract_comment, delete_abstract_comment,
                                                        update_abstract_comment, create_abstract_review,
                                                        update_abstract_review, update_reviewed_for_tracks)
from indico.modules.events.abstracts.util import get_track_reviewer_abstract_counts, get_user_tracks
from indico.modules.events.abstracts.views import WPDisplayAbstractsReviewing, render_abstract_page
from indico.modules.events.tracks.models.tracks import Track
from indico.util.i18n import _
from indico.web.flask.templating import get_template_module
from indico.web.util import jsonify_data, jsonify_template

class RHListOtherAbstracts(RHAbstractsBase):
    """AJAX endpoint that lists all abstracts in the event (dict representation)."""

    def _checkProtection(self):
        if not session.user:
            raise Forbidden
        RHAbstractsBase._checkProtection(self)

    def _checkParams(self, params):
        RHAbstractsBase._checkParams(self, params)
        self.excluded_ids = set(request.form.getlist('excluded_abstract_id'))

    def _process(self):
        query = (Abstract.query
                 .with_parent(self.event_new)
                 .filter(Abstract.state.notin_({AbstractState.duplicate, AbstractState.merged}))
                 .options(joinedload('submitter').lazyload('*'),
                          subqueryload('reviewed_for_tracks'),
                          subqueryload('person_links').joinedload('person').joinedload('user'))
                 .order_by(Abstract.friendly_id))

        if self.excluded_ids:
            query = query.filter(Abstract.id.notin_(self.excluded_ids))

        result = [{'id': abstract.id, 'friendly_id': abstract.friendly_id, 'title': abstract.title,
                   'full_title': '#{}: {}'.format(abstract.friendly_id, abstract.title)}
                  for abstract in query
                  if abstract.can_access(session.user)]
        return jsonify(result)

class RHJudgeAbstract(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_judge(session.user, check_state=True)

    def _process(self):
        form = AbstractJudgmentForm(abstract=self.abstract)
        if form.validate_on_submit():
            judgment_data, abstract_data = form.split_data
            judge_abstract(self.abstract, abstract_data, judge=session.user, **judgment_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/judge.html')
        return jsonify(html=tpl.render_decision_box(self.abstract, form))

class RHResetAbstractState(RHAbstractBase):
    def _check_abstract_protection(self):
        if self.abstract.state == AbstractState.submitted:
            return False
        # manages can always reset
        if self.event_new.can_manage(session.user):
            return True
        # judges can reset if the abstract has not been withdrawn
        return self.abstract.can_judge(session.user) and self.abstract.state != AbstractState.withdrawn

    def _process(self):
        if self.abstract.state != AbstractState.submitted:
            reset_abstract_state(self.abstract)
            flash(_("Abstract state has been reset"), 'success')
        return jsonify_data(html=render_abstract_page(self.abstract, management=self.management))

class RHWithdrawAbstract(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_withdraw(session.user, check_state=True)

    def _process(self):
        if self.abstract.state != AbstractState.withdrawn:
            withdraw_abstract(self.abstract)
            flash(_("Abstract has been withdrawn"), 'success')
        return jsonify_data(html=render_abstract_page(self.abstract, management=self.management))

class RHDisplayAbstractListBase(RHAbstractsBase):
    """Base class for all abstract list operations"""

    normalize_url_spec = {
        'locators': {
            lambda self: self.track
        }
    }

    def _checkParams(self, params):
        RHAbstractsBase._checkParams(self, params)
        self.track = Track.get_one(request.view_args['track_id'])
        self.list_generator = AbstractListGeneratorDisplay(event=self.event_new, track=self.track)

    def _checkProtection(self):
        if not self.track.can_review_abstracts(session.user) and not self.track.can_convene(session.user):
            raise Forbidden

class RHSubmitAbstractReview(RHAbstractBase):
    """Review an abstract in a specific track"""

    normalize_url_spec = {
        'locators': {
            lambda self: self.abstract,
            lambda self: self.track
        }
    }

    def _check_abstract_protection(self):
        if self.abstract.get_reviews(user=session.user, track=self.track):
            return False
        return self.abstract.can_review(session.user, check_state=True)

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.track = Track.get_one(request.view_args['track_id'])

    def _process(self):
        form = build_review_form(self.abstract, self.track)
        if form.validate_on_submit():
            create_abstract_review(self.abstract, self.track, session.user, **form.split_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/_timeline_forms.html')
        return jsonify(html=tpl.render_review_form(form, self.abstract, self.track))

class RHEditAbstractReview(RHAbstractBase):
    normalize_url_spec = {
        'locators': {
            lambda self: self.review
        }
    }

    def _check_abstract_protection(self):
        return self.review.can_edit(session.user, check_state=True)

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.review = AbstractReview.get_one(request.view_args['review_id'])

    def _process(self):
        form = build_review_form(review=self.review)
        if form.validate_on_submit():
            update_abstract_review(self.review, **form.split_data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/_timeline_forms.html')
        return jsonify(html=tpl.render_review_form(form, review=self.review))

class RHSubmitAbstractComment(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_comment(session.user)

    def _process(self):
        form = AbstractCommentForm(abstract=self.abstract, user=session.user)
        if form.validate_on_submit():
            create_abstract_comment(self.abstract, form.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/_timeline_forms.html')
        return jsonify(html=tpl.render_comment_form(form, self.abstract))

class RHAbstractCommentBase(RHAbstractBase):
    normalize_url_spec = {
        'locators': {
            lambda self: self.comment
        }
    }

    def _checkParams(self, params):
        RHAbstractBase._checkParams(self, params)
        self.comment = AbstractComment.get_one(request.view_args['comment_id'])

    def _checkProtection(self):
        RHAbstractBase._checkProtection(self)
        if not self.comment.can_edit(session.user):
            raise Forbidden

class RHEditAbstractComment(RHAbstractCommentBase):
    def _process(self):
        form = AbstractCommentForm(obj=self.comment, abstract=self.abstract, user=session.user,
                                   prefix='edit-comment-{}-'.format(self.comment.id))
        if form.validate_on_submit():
            update_abstract_comment(self.comment, form.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        tpl = get_template_module('events/abstracts/abstract/_timeline_forms.html')
        return jsonify(html=tpl.render_comment_form(form, self.abstract, comment=self.comment))

class RHDeleteAbstractComment(RHAbstractCommentBase):
    def _process(self):
        delete_abstract_comment(self.comment)
        return jsonify_data(flash=False)

class RHDisplayReviewableTracks(RHAbstractsBase):
    def _checkProtection(self):
        if not session.user:
            raise Forbidden
        RHAbstractsBase._checkProtection(self)

    def _process(self):
        track_reviewer_abstract_count = get_track_reviewer_abstract_counts(self.event_new, session.user)
        return WPDisplayAbstractsReviewing.render_template('display/tracks.html', self._conf, event=self.event_new,
                                                           abstract_count=track_reviewer_abstract_count,
                                                           tracks=get_user_tracks(self.event_new, session.user))

class RHDisplayReviewableTrackAbstracts(DisplayAbstractListMixin, RHDisplayAbstractListBase):
    view_class = WPDisplayAbstractsReviewing
    template = 'display/abstracts.html'

    def _render_template(self, **kwargs):
        return DisplayAbstractListMixin._render_template(self, track=self.track, **kwargs)

class RHDisplayAbstractListCustomize(CustomizeAbstractListMixin, RHDisplayAbstractListBase):
    view_class = WPDisplayAbstractsReviewing

class RHDisplayAbstractsActionsBase(RHDisplayAbstractListBase):
    """Base class for classes performing actions on abstract"""

    def _checkParams(self, params):
        RHDisplayAbstractListBase._checkParams(self, params)
        ids = map(int, request.form.getlist('abstract_id'))
        self.abstracts = Abstract.query.with_parent(self.track, 'abstracts_reviewed').filter(Abstract.id.in_(ids)).all()

class RHDisplayAbstractsDownloadAttachments(AbstractsDownloadAttachmentsMixin, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportPDF(AbstractsExportPDFMixin, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportCSV(AbstractsExportCSV, RHDisplayAbstractsActionsBase):
    pass

class RHDisplayAbstractsExportExcel(AbstractsExportExcel, RHDisplayAbstractsActionsBase):
    pass

class RHEditReviewedForTrackList(RHAbstractBase):
    def _check_abstract_protection(self):
        return self.abstract.can_judge(session.user, check_state=True)

    def _process(self):
        form = AbstractReviewedForTracksForm(event=self.event_new, obj=self.abstract)
        if form.validate_on_submit():
            update_reviewed_for_tracks(self.abstract, form.reviewed_for_tracks.data)
            return jsonify_data(flash=False, html=render_abstract_page(self.abstract, management=self.management))
        return jsonify_template('events/abstracts/forms/edit_review_tracks.html', form=form)
<|endoftext|>


<commit_before># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyAttrs(PythonPackage):
    """Classes Without Boilerplate"""

    homepage = "http://attrs.org/"
    url      = "https://pypi.io/packages/source/a/attrs/attrs-19.2.0.tar.gz"

    import_modules = ['attr']

    version('19.2.0', sha256='f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396')
    version('18.1.0', sha256='e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b')
    version('16.3.0', sha256='80203177723e36f3bbe15aa8553da6e80d47bfe53647220ccaa9ad7a5e473ccc')

    depends_on('python@2.7:2.8,3.4:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')

    depends_on('py-coverage', type='test')
    depends_on('py-hypothesis', type='test')
    depends_on('py-pympler', type='test')
    depends_on('py-pytest', type='test')
    depends_on('py-six', type='test')
    depends_on('py-zope-interface', type='test')
<commit_msg>Add extra version of py-attrs (#14969)<commit_after># Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyAttrs(PythonPackage):
    """Classes Without Boilerplate"""

    homepage = "http://attrs.org/"
    url      = "https://pypi.io/packages/source/a/attrs/attrs-19.2.0.tar.gz"

    import_modules = ['attr']

    version('19.2.0', sha256='f913492e1663d3c36f502e5e9ba6cd13cf19d7fab50aa13239e420fef95e1396')
    version('19.1.0', sha256='f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399')
    version('18.1.0', sha256='e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b')
    version('16.3.0', sha256='80203177723e36f3bbe15aa8553da6e80d47bfe53647220ccaa9ad7a5e473ccc')

    depends_on('python@2.7:2.8,3.4:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')

    depends_on('py-coverage', type='test')
    depends_on('py-hypothesis', type='test')
    depends_on('py-pympler', type='test')
    depends_on('py-pytest', type='test')
    depends_on('py-six', type='test')
    depends_on('py-zope-interface', type='test')
<|endoftext|>


<commit_before># Template tag
from datetime import date, timedelta

from django import template
from agenda.events.models import Event
from django.db.models import Q

register = template.Library()

def get_last_day_of_month(year, month):
    if (month == 12):
        year += 1
        month = 1
    else:
        month += 1
    return date(year, month, 1) - timedelta(1)

def month_cal(year, month, region = None):

    first_day_of_month = date(year, month, 1)
    last_day_of_month = get_last_day_of_month(year, month)
    first_day_of_calendar = first_day_of_month - timedelta(first_day_of_month.isoweekday())
    last_day_of_calendar = last_day_of_month + timedelta(7 - last_day_of_month.isoweekday())
    today = date.today();

    # Filter local events for given region, include national and international events
    if region != None:
      q = Q(city__region=region,scope="L") | Q(scope="I") | Q(scope="N")
    else:
      q = Q()

    event_list = Event.objects.filter (
        start_time__gte=first_day_of_calendar,
        end_time__lte=last_day_of_calendar,
        moderated=True).filter (q)

    month_cal = []
    week = []
    week_headers = []

    i = 0
    day = first_day_of_calendar
    while day <= last_day_of_calendar:
        if i < 7:
            week_headers.append(day)

        cal_day = {}
        cal_day['day'] = day
        cal_day['event'] = False

        day_events = []
        for event in event_list:
            if day >= event.start_time.date() and day <= event.end_time.date():
                day_events.append (event)

        cal_day['events'] = day_events

        cal_day['in_month'] = (day.month == month)
        cal_day['is_past'] = (day < today)
        cal_day['is_today'] = (day == today)

        week.append(cal_day)

        if day.isoweekday() == 6:
            month_cal.append(week)
            week = []

        i += 1
        day += timedelta(1)

    return {'calendar': month_cal, 'headers': week_headers}

register.inclusion_tag('calendar.html')(month_cal)
<commit_msg>Revert: Week day starts on Monday<commit_after># Template tag
from datetime import date, timedelta

from django import template
from agenda.events.models import Event
from django.db.models import Q

register = template.Library()

def get_last_day_of_month(year, month):
    if (month == 12):
        year += 1
        month = 1
    else:
        month += 1
    return date(year, month, 1) - timedelta(1)

def month_cal(year, month, region = None):

    first_day_of_month = date(year, month, 1)
    last_day_of_month = get_last_day_of_month(year, month)
    first_day_of_calendar = first_day_of_month - timedelta(first_day_of_month.weekday())
    last_day_of_calendar = last_day_of_month + timedelta(7 - last_day_of_month.weekday())
    print last_day_of_month.isoweekday()
    today = date.today();

    # Filter local events for given region, include national and international events
    if region != None:
      q = Q(city__region=region,scope="L") | Q(scope="I") | Q(scope="N")
    else:
      q = Q()

    event_list = Event.objects.filter (
        start_time__gte=first_day_of_calendar,
        end_time__lte=last_day_of_calendar,
        moderated=True).filter (q)

    month_cal = []
    week = []
    week_headers = []

    i = 0
    day = first_day_of_calendar
    while day <= last_day_of_calendar:
        if i < 7:
            week_headers.append(day)

        cal_day = {}
        cal_day['day'] = day
        cal_day['event'] = False

        day_events = []
        for event in event_list:
            if day >= event.start_time.date() and day <= event.end_time.date():
                day_events.append (event)

        cal_day['events'] = day_events

        cal_day['in_month'] = (day.month == month)
        cal_day['is_past'] = (day < today)
        cal_day['is_today'] = (day == today)

        week.append(cal_day)

        if day.weekday() == 6:
            month_cal.append(week)
            week = []

        i += 1
        day += timedelta(1)

    return {'calendar': month_cal, 'headers': week_headers}

register.inclusion_tag('calendar.html')(month_cal)
<|endoftext|>


<commit_before>from django.test import TestCase
from django.core.urlresolvers import reverse

from wagtail.wagtailcore.models import Page
from wagtail.tests.models import SimplePage
from wagtail.tests.utils import login

class TestChooserBrowse(TestCase):
    def setUp(self):
        self.root_page = Page.objects.get(id=2)

        # Add child page
        self.child_page = SimplePage()
        self.child_page.title = "foobarbaz"
        self.child_page.slug = "foobarbaz"
        self.root_page.add_child(instance=self.child_page)

        login(self.client)

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page'), params)

    def test_status_code(self):
        self.assertEqual(self.get().status_code, 200)

    def test_search(self):
        response = self.get({'q': "foobarbaz"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There is one match")
        self.assertContains(response, "foobarbaz")

    def test_search_no_results(self):
        response = self.get({'q': "quux"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There are 0 matches")

    def test_get_invalid(self):
        response = self.get({'page_type': 'foo.bar'})
        self.assertEqual(response.status_code, 404)

class TestChooserBrowseChild(TestCase):
    def setUp(self):
        self.root_page = Page.objects.get(id=2)

        # Add child page
        self.child_page = SimplePage()
        self.child_page.title = "foobarbaz"
        self.child_page.slug = "foobarbaz"
        self.root_page.add_child(instance=self.child_page)

        login(self.client)

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_child',
                                       args=(self.root_page.id,)), params)

    def get_invalid(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_child',
                                       args=(9999999,)), params)

    def test_status_code(self):
        self.assertEqual(self.get().status_code, 200)

    def test_search(self):
        response = self.get({'q': "foobarbaz"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There is one match")
        self.assertContains(response, "foobarbaz")

    def test_search_no_results(self):
        response = self.get({'q': "quux"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There are 0 matches")

    def test_get_invalid(self):
        self.assertEqual(self.get_invalid().status_code, 404)

class TestChooserExternalLink(TestCase):
    def setUp(self):
        login(self.client)

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_external_link'), params)

    def post(self, post_data={}):
        return self.client.post(reverse('wagtailadmin_choose_page_external_link'), post_data)

    def test_status_code(self):
        self.assertEqual(self.get().status_code, 200)

    def test_get_with_param(self):
        self.assertEqual(self.get({'prompt_for_link_text': 'foo'}).status_code, 200)

    def test_create_link(self):
        request = self.post({'url': 'http://www.example.com'})
        self.assertContains(request, "'url': 'http://www.example.com/',")
        self.assertContains(request, "'title': 'http://www.example.com/'")

class TestChooserEmailLink(TestCase):
    def setUp(self):
        login(self.client)

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_email_link'), params)

    def post(self, post_data={}):
        return self.client.post(reverse('wagtailadmin_choose_page_email_link'), post_data)

    def test_status_code(self):
        self.assertEqual(self.get().status_code, 200)

    def test_get_with_param(self):
        self.assertEqual(self.get({'prompt_for_link_text': 'foo'}).status_code, 200)

    def test_create_link(self):
        request = self.post({'email_address': 'example@example.com'})
        self.assertContains(request, "'url': 'mailto:example@example.com',")
        self.assertContains(request, "'title': 'example@example.com'")
<commit_msg>Improvements to page chooser tests<commit_after>from django.test import TestCase
from django.core.urlresolvers import reverse

from wagtail.wagtailcore.models import Page
from wagtail.tests.models import SimplePage
from wagtail.tests.utils import WagtailTestUtils

class TestChooserBrowse(TestCase, WagtailTestUtils):
    def setUp(self):
        self.root_page = Page.objects.get(id=2)

        # Add child page
        self.child_page = SimplePage()
        self.child_page.title = "foobarbaz"
        self.child_page.slug = "foobarbaz"
        self.root_page.add_child(instance=self.child_page)

        self.login()

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page'), params)

    def test_simple(self):
        response = self.get()
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'wagtailadmin/chooser/browse.html')

    def test_search(self):
        response = self.get({'q': "foobarbaz"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There is one match")
        self.assertContains(response, "foobarbaz")

    def test_search_no_results(self):
        response = self.get({'q': "quux"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There are 0 matches")

    def test_get_invalid(self):
        response = self.get({'page_type': 'foo.bar'})
        self.assertEqual(response.status_code, 404)

class TestChooserBrowseChild(TestCase, WagtailTestUtils):
    def setUp(self):
        self.root_page = Page.objects.get(id=2)

        # Add child page
        self.child_page = SimplePage()
        self.child_page.title = "foobarbaz"
        self.child_page.slug = "foobarbaz"
        self.root_page.add_child(instance=self.child_page)

        self.login()

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_child',
                                       args=(self.root_page.id,)), params)

    def get_invalid(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_child',
                                       args=(9999999,)), params)

    def test_simple(self):
        response = self.get()
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'wagtailadmin/chooser/browse.html')

    def test_search(self):
        response = self.get({'q': "foobarbaz"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There is one match")
        self.assertContains(response, "foobarbaz")

    def test_search_no_results(self):
        response = self.get({'q': "quux"})
        self.assertEqual(response.status_code, 200)
        self.assertContains(response, "There are 0 matches")

    def test_get_invalid(self):
        self.assertEqual(self.get_invalid().status_code, 404)

class TestChooserExternalLink(TestCase, WagtailTestUtils):
    def setUp(self):
        self.login()

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_external_link'), params)

    def post(self, post_data={}):
        return self.client.post(reverse('wagtailadmin_choose_page_external_link'), post_data)

    def test_simple(self):
        response = self.get()
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'wagtailadmin/chooser/external_link.html')

    def test_get_with_param(self):
        self.assertEqual(self.get({'prompt_for_link_text': 'foo'}).status_code, 200)

    def test_create_link(self):
        request = self.post({'url': 'http://www.example.com'})
        self.assertContains(request, "'url': 'http://www.example.com/',")
        self.assertContains(request, "'title': 'http://www.example.com/'")

class TestChooserEmailLink(TestCase, WagtailTestUtils):
    def setUp(self):
        self.login()

    def get(self, params={}):
        return self.client.get(reverse('wagtailadmin_choose_page_email_link'), params)

    def post(self, post_data={}):
        return self.client.post(reverse('wagtailadmin_choose_page_email_link'), post_data)

    def test_simple(self):
        response = self.get()
        self.assertEqual(response.status_code, 200)
        self.assertTemplateUsed(response, 'wagtailadmin/chooser/email_link.html')

    def test_get_with_param(self):
        self.assertEqual(self.get({'prompt_for_link_text': 'foo'}).status_code, 200)

    def test_create_link(self):
        request = self.post({'email_address': 'example@example.com'})
        self.assertContains(request, "'url': 'mailto:example@example.com',")
        self.assertContains(request, "'title': 'example@example.com'")
<|endoftext|>


<commit_before>import asyncio
import concurrent.futures as cf
import contextlib as cl
import functools as ft
import hashlib
import json
import multiprocessing as mp
import os.path as op
import pathlib
import re
import threading

import aiohttp
import wcpan.drive.google as wdg
from wcpan.logger import DEBUG, INFO, ERROR, EXCEPTION, WARNING
import wcpan.worker as ww

from . import settings

RETRY_TIMES = 3

class DriveUploader(object):

    def __init__(self):
        self._jobs = set()
        self._sync_lock = asyncio.Lock()
        self._drive = None
        self._curl = None
        self._queue = None
        self._pool = None
        self._raii = None

    async def __aenter__(self):
        async with cl.AsyncExitStack() as stack:
            self._drive = await stack.enter_async_context(wdg.Drive())
            self._queue = await stack.enter_async_context(ww.AsyncQueue(8))
            self._curl = await stack.enter_async_context(aiohttp.ClientSession())
            self._pool = stack.enter_context(cf.ProcessPoolExecutor())
            self._raii = stack.pop_all()
        return self

    async def __aexit__(self, type_, exc, tb):
        await self._raii.aclose()
        self._drive = None
        self._curl = None
        self._queue = None
        self._pool = None
        self._raii = None

    async def upload_path(self, remote_path, local_path):
        if local_path in self._jobs:
            WARNING('duld') << local_path << 'is still uploading'
            return False

        with job_guard(self._jobs, local_path):
            await self._sync()

            node = await self._drive.get_node_by_path(remote_path)
            if not node:
                ERROR('duld') << remote_path << 'not found'
                return False

            local_path = pathlib.Path(local_path)
            ok = await self._upload(node, local_path)
            if not ok:
                ERROR('duld') << local_path << 'upload failed'
            return ok

    async def upload_torrent(self, remote_path, torrent_id, torrent_root, root_items):
        if torrent_id in self._jobs:
            WARNING('duld') << torrent_id << 'is still uploading'
            return False

        with job_guard(self._jobs, torrent_id):
            await self._sync()

            node = await self._drive.get_node_by_path(remote_path)
            if not node:
                ERROR('duld') << remote_path << 'not found'
                return False

            # files/directories to be upload
            items = map(lambda _: pathlib.Path(torrent_root, _), root_items)
            all_ok = True
            for item in items:
                ok = await self._upload(node, item)
                if not ok:
                    ERROR('duld') << item << 'upload failed'
                    all_ok = False
                    continue

            return all_ok

    async def _sync(self):
        async with self._sync_lock:
            count = 0
            async for changes in self._drive.sync():
                count += 1
            INFO('duld') << 'sync' << count

    async def _upload(self, node, local_path):
        if await self._should_exclude(local_path.name):
            INFO('duld') << 'excluded' << local_path
            return True

        if local_path.is_dir():
            ok = await self._upload_directory(node, local_path)
        else:
            ok = await self._upload_file_retry(node, local_path)
        return ok

    async def _upload_directory(self, node, local_path):
        dir_name = local_path.name

        # find or create remote directory
        child_node = await self._drive.get_node_by_name_from_parent(dir_name, node)
        if child_node and child_node.is_file:
            # is file
            path = await self._drive.get_path(child_node)
            ERROR('duld') << '(remote)' << path << 'is a file'
            return False
        elif not child_node or child_node.trashed or node.trashed:
            # not exists
            child_node = await self._drive.create_folder(node, dir_name)
            if not child_node:
                path = await self._drive.get_path(node)
                path = op.join(path, dir_name)
                ERROR('duld') << '(remote) cannot create' << path
                return False

            # Need to update local cache for the added folder.
            # In theory we should pass remote path instead of doing this.
            while True:
                try:
                    await self._drive.get_path(child_node)
                    break
                except Exception:
                    pass
                await self._sync()

        all_ok = True
        for child_path in local_path.iterdir():
            ok = await self._upload(child_node, child_path)
            if not ok:
                ERROR('duld') << '(remote) cannot upload' << child_path
                all_ok = False

        return all_ok

    async def _upload_file_retry(self, node, local_path):
        for _ in range(RETRY_TIMES):
            try:
                ok = await self._upload_file(node, local_path)
            except Exception as e:
                EXCEPTION('duld', e) << 'retry upload file'
            else:
                return ok

            await self._sync()
        else:
            ERROR('duld') << f'tried upload {RETRY_TIMES} times'
            return False

    async def _upload_file(self, node, local_path):
        file_name = local_path.name
        remote_path = await self._drive.get_path(node)
        remote_path = pathlib.Path(remote_path, file_name)

        child_node = await self._drive.get_node_by_name_from_parent(file_name, node)

        if child_node and not child_node.trashed:
            if child_node.is_folder:
                ERROR('duld') << '(remote)' << remote_path << 'is a directory'
                return False

            # check integrity
            ok = await self._verify_remote_file(local_path, remote_path, child_node.md5)
            if not ok:
                return False
            INFO('duld') << remote_path << 'already exists'

        if not child_node or child_node.trashed:
            INFO('duld') << 'uploading' << remote_path

            child_node = await wdg.upload_from_local(self._drive, node, str(local_path))

            # check integrity
            ok = await self._verify_remote_file(local_path, remote_path, child_node.md5)
            if not ok:
                return False

        return True

    async def _verify_remote_file(self, local_path, remote_path, remote_md5):
        loop = asyncio.get_running_loop()
        local_md5 = await loop.run_in_executor(self._pool, md5sum, local_path)
        if local_md5 != remote_md5:
            ERROR('duld') << '(remote)' << remote_path << 'has a different md5 ({0}, {1})'.format(local_md5, remote_md5)
            return False
        return True

    # used in exception handler, DO NOT throw another exception again
    async def _try_resolve_name_confliction(self, node, local_path):
        name = op.basename(local_path)
        node = await self._drive.get_node_by_name_from_parent(name, node)
        if not node:
            return True
        try:
            ok = await self._drive.trash_node_by_id(node.id_)
            return ok
        except Exception as e:
            EXCEPTION('duld', e)
        return False

    async def _should_exclude(self, name):
        for pattern in settings['exclude_pattern']:
            if re.match(pattern, name, re.IGNORECASE):
                return True

        if settings['exclude_url']:
            async with self._curl.get(settings['exclude_url']) as rv:
                rv = await rv.json()
                for _, pattern in rv.items():
                    if re.match(pattern, name, re.IGNORECASE):
                        return True

        return False

def md5sum(path):
    hasher = hashlib.md5()
    with path.open('rb') as fin:
        while True:
            chunk = fin.read(65536)
            if not chunk:
                break
            hasher.update(chunk)
    return hasher.hexdigest()

@cl.contextmanager
def job_guard(set_, token):
    set_.add(token)
    try:
        yield
    finally:
        set_.discard(token)
<commit_msg>slower sync<commit_after>import asyncio
import concurrent.futures as cf
import contextlib as cl
import functools as ft
import hashlib
import json
import multiprocessing as mp
import os.path as op
import pathlib
import re
import threading

import aiohttp
import wcpan.drive.google as wdg
from wcpan.logger import DEBUG, INFO, ERROR, EXCEPTION, WARNING
import wcpan.worker as ww

from . import settings

RETRY_TIMES = 3

class DriveUploader(object):

    def __init__(self):
        self._jobs = set()
        self._sync_lock = asyncio.Lock()
        self._drive = None
        self._curl = None
        self._queue = None
        self._pool = None
        self._raii = None

    async def __aenter__(self):
        async with cl.AsyncExitStack() as stack:
            self._drive = await stack.enter_async_context(wdg.Drive())
            self._queue = await stack.enter_async_context(ww.AsyncQueue(8))
            self._curl = await stack.enter_async_context(aiohttp.ClientSession())
            self._pool = stack.enter_context(cf.ProcessPoolExecutor())
            self._raii = stack.pop_all()
        return self

    async def __aexit__(self, type_, exc, tb):
        await self._raii.aclose()
        self._drive = None
        self._curl = None
        self._queue = None
        self._pool = None
        self._raii = None

    async def upload_path(self, remote_path, local_path):
        if local_path in self._jobs:
            WARNING('duld') << local_path << 'is still uploading'
            return False

        with job_guard(self._jobs, local_path):
            await self._sync()

            node = await self._drive.get_node_by_path(remote_path)
            if not node:
                ERROR('duld') << remote_path << 'not found'
                return False

            local_path = pathlib.Path(local_path)
            ok = await self._upload(node, local_path)
            if not ok:
                ERROR('duld') << local_path << 'upload failed'
            return ok

    async def upload_torrent(self, remote_path, torrent_id, torrent_root, root_items):
        if torrent_id in self._jobs:
            WARNING('duld') << torrent_id << 'is still uploading'
            return False

        with job_guard(self._jobs, torrent_id):
            await self._sync()

            node = await self._drive.get_node_by_path(remote_path)
            if not node:
                ERROR('duld') << remote_path << 'not found'
                return False

            # files/directories to be upload
            items = map(lambda _: pathlib.Path(torrent_root, _), root_items)
            all_ok = True
            for item in items:
                ok = await self._upload(node, item)
                if not ok:
                    ERROR('duld') << item << 'upload failed'
                    all_ok = False
                    continue

            return all_ok

    async def _sync(self):
        async with self._sync_lock:
            count = 0
            async for changes in self._drive.sync():
                count += 1
            INFO('duld') << 'sync' << count

    async def _upload(self, node, local_path):
        if await self._should_exclude(local_path.name):
            INFO('duld') << 'excluded' << local_path
            return True

        if local_path.is_dir():
            ok = await self._upload_directory(node, local_path)
        else:
            ok = await self._upload_file_retry(node, local_path)
        return ok

    async def _upload_directory(self, node, local_path):
        dir_name = local_path.name

        # find or create remote directory
        child_node = await self._drive.get_node_by_name_from_parent(dir_name, node)
        if child_node and child_node.is_file:
            # is file
            path = await self._drive.get_path(child_node)
            ERROR('duld') << '(remote)' << path << 'is a file'
            return False
        elif not child_node or child_node.trashed or node.trashed:
            # not exists
            child_node = await self._drive.create_folder(node, dir_name)
            if not child_node:
                path = await self._drive.get_path(node)
                path = op.join(path, dir_name)
                ERROR('duld') << '(remote) cannot create' << path
                return False

            # Need to update local cache for the added folder.
            # In theory we should pass remote path instead of doing this.
            while True:
                try:
                    await self._drive.get_path(child_node)
                    break
                except Exception:
                    pass
                await asyncio.sleep(1)
                await self._sync()

        all_ok = True
        for child_path in local_path.iterdir():
            ok = await self._upload(child_node, child_path)
            if not ok:
                ERROR('duld') << '(remote) cannot upload' << child_path
                all_ok = False

        return all_ok

    async def _upload_file_retry(self, node, local_path):
        for _ in range(RETRY_TIMES):
            try:
                ok = await self._upload_file(node, local_path)
            except Exception as e:
                EXCEPTION('duld', e) << 'retry upload file'
            else:
                return ok

            await self._sync()
        else:
            ERROR('duld') << f'tried upload {RETRY_TIMES} times'
            return False

    async def _upload_file(self, node, local_path):
        file_name = local_path.name
        remote_path = await self._drive.get_path(node)
        remote_path = pathlib.Path(remote_path, file_name)

        child_node = await self._drive.get_node_by_name_from_parent(file_name, node)

        if child_node and not child_node.trashed:
            if child_node.is_folder:
                ERROR('duld') << '(remote)' << remote_path << 'is a directory'
                return False

            # check integrity
            ok = await self._verify_remote_file(local_path, remote_path, child_node.md5)
            if not ok:
                return False
            INFO('duld') << remote_path << 'already exists'

        if not child_node or child_node.trashed:
            INFO('duld') << 'uploading' << remote_path

            child_node = await wdg.upload_from_local(self._drive, node, str(local_path))

            # check integrity
            ok = await self._verify_remote_file(local_path, remote_path, child_node.md5)
            if not ok:
                return False

        return True

    async def _verify_remote_file(self, local_path, remote_path, remote_md5):
        loop = asyncio.get_running_loop()
        local_md5 = await loop.run_in_executor(self._pool, md5sum, local_path)
        if local_md5 != remote_md5:
            ERROR('duld') << '(remote)' << remote_path << 'has a different md5 ({0}, {1})'.format(local_md5, remote_md5)
            return False
        return True

    # used in exception handler, DO NOT throw another exception again
    async def _try_resolve_name_confliction(self, node, local_path):
        name = op.basename(local_path)
        node = await self._drive.get_node_by_name_from_parent(name, node)
        if not node:
            return True
        try:
            ok = await self._drive.trash_node_by_id(node.id_)
            return ok
        except Exception as e:
            EXCEPTION('duld', e)
        return False

    async def _should_exclude(self, name):
        for pattern in settings['exclude_pattern']:
            if re.match(pattern, name, re.IGNORECASE):
                return True

        if settings['exclude_url']:
            async with self._curl.get(settings['exclude_url']) as rv:
                rv = await rv.json()
                for _, pattern in rv.items():
                    if re.match(pattern, name, re.IGNORECASE):
                        return True

        return False

def md5sum(path):
    hasher = hashlib.md5()
    with path.open('rb') as fin:
        while True:
            chunk = fin.read(65536)
            if not chunk:
                break
            hasher.update(chunk)
    return hasher.hexdigest()

@cl.contextmanager
def job_guard(set_, token):
    set_.add(token)
    try:
        yield
    finally:
        set_.discard(token)
<|endoftext|>


<commit_before>import unittest
from paas_manager.hadoop_modules import start_hadoop, exec_hadoop

class MyTestCase(unittest.TestCase):
    def test_start_hadoop(self):
        start_hadoop()
        self.assertEqual(True, False)

    def test_exec_hadoop(self):
        mock_path = "test/mock_exec_hadoop.sh"

        def callback(out, err):
            self.assertEqual(out, "fin\n")
            self.assertEqual(err, "err\n")

        command = [mock_path]
        exec_hadoop(command, callback)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix test_start_hadoop<commit_after>import unittest
from paas_manager.hadoop_modules import start_hadoop, exec_hadoop

class Test_hadoop_modules(unittest.TestCase):
    def test_start_hadoop(self):
        jar_path = ""
        args = []

        def callback(out, err):
            self.assertEqual(out, "fin\n")
            self.assertEqual(err, "err\n")

        start_hadoop(jar_path, args, callback)

    def test_exec_hadoop(self):
        mock_path = "test/mock_exec_hadoop.sh"

        def callback(out, err):
            self.assertEqual(out, "fin\n")
            self.assertEqual(err, "err\n")

        command = [mock_path]
        exec_hadoop(command, callback)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

class PyTorchvision(PythonPackage):
    """The torchvision package consists of popular datasets, model
    architectures, and common image transformations for computer vision."""

    homepage = "https://github.com/pytorch/vision"
    url      = "https://github.com/pytorch/vision/archive/v0.8.2.tar.gz"
    git      = "https://github.com/pytorch/vision.git"

    maintainers = ['adamjstewart']

    version('master', branch='master')
    version('0.9.1', sha256='79964773729880e0eee0e6af13f336041121d4cc8491a3e2c0e5f184cac8a718')
    version('0.9.0', sha256='9351ed92aded632f8c7f59dfadac13c191a834babe682f5785ea47e6fcf6b472')
    version('0.8.2', sha256='9a866c3c8feb23b3221ce261e6153fc65a98ce9ceaa71ccad017016945c178bf')
    version('0.8.1', sha256='c46734c679c99f93e5c06654f4295a05a6afe6c00a35ebd26a2cce507ae1ccbd')
    version('0.8.0', sha256='b5f040faffbfc7bac8d4687d8665bd1196937334589b3fb5fcf15bb69ca25391')
    version('0.7.0', sha256='fa0a6f44a50451115d1499b3f2aa597e0092a07afce1068750260fa7dd2c85cb')
    version('0.6.1', sha256='8173680a976c833640ecbd0d7e6f0a11047bf8833433e2147180efc905e48656')
    version('0.6.0', sha256='02de11b3abe6882de4032ce86dab9c7794cbc84369b44d04e667486580f0f1f7')
    version('0.5.0', sha256='eb9afc93df3d174d975ee0914057a9522f5272310b4d56c150b955c287a4d74d')
    version('0.4.2', sha256='1184a27eab85c9e784bacc6f9d6fec99e168ab4eda6047ef9f709e7fdb22d8f9')
    version('0.4.1', sha256='053689351272b3bd2ac3e6ba51efd284de0e4ca4a301f54674b949f1e62b7176')
    version('0.4.0', sha256='c270d74e568bad4559fed4544f6dd1e22e2eb1c60b088e04a5bd5787c4150589')
    version('0.3.0', sha256='c205f0618c268c6ed2f8abb869ef6eb83e5339c1336c243ad321a2f2a85195f0')

    # https://github.com/pytorch/vision#image-backend
    variant('backend', default='pil', description='Image backend',
            values=('pil', 'accimage', 'png', 'jpeg'), multi=False)

    # https://github.com/pytorch/vision#installation
    depends_on('python@3.6:', when='@0.7:', type=('build', 'link', 'run'))
    depends_on('python@3.5:', when='@0.6.0:0.6.999', type=('build', 'link', 'run'))
    depends_on('python@2.7:2.8,3.5:3.8', when='@0.5.0', type=('build', 'link', 'run'))
    depends_on('python@2.7:2.8,3.5:3.7', when='@:0.4', type=('build', 'link', 'run'))

    depends_on('py-setuptools', type='build')
    depends_on('ninja', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-six', when='@:0.5', type=('build', 'run'))

    # https://github.com/pytorch/vision#installation
    depends_on('py-torch@master', when='@master', type=('build', 'link', 'run'))
    depends_on('py-torch@1.8.1', when='@0.9.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.8.0', when='@0.9.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.1', when='@0.8.2', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.0', when='@0.8.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.0', when='@0.8.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.6.0', when='@0.7.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.5.1', when='@0.6.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.5.0', when='@0.6.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.4.1', when='@0.5.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.3.1', when='@0.4.2', type=('build', 'link', 'run'))
    depends_on('py-torch@1.3.0', when='@0.4.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.2.0', when='@0.4.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.1.0', when='@0.3.0', type=('build', 'link', 'run'))
    depends_on('py-torch@:1.0.1', when='@0.2.2', type=('build', 'link', 'run'))

    # https://github.com/pytorch/vision/issues/1712
    depends_on('pil@4.1.1:6', when='@:0.4 backend=pil', type=('build', 'run'))
    depends_on('pil@4.1.1:',  when='@0.5: backend=pil', type=('build', 'run'))
    depends_on('py-accimage', when='backend=accimage', type=('build', 'run'))
    depends_on('libpng@1.6.0:', when='backend=png')
    depends_on('jpeg', when='backend=jpeg')

    # Many of the datasets require additional dependencies to use.
    # These can be installed after the fact.

    depends_on('ffmpeg@3.1:', when='@0.4.2:')

    conflicts('backend=png', when='@:0.7')
    conflicts('backend=jpeg', when='@:0.7')

    def setup_build_environment(self, env):
        include = []
        library = []
        for dep in self.spec.dependencies(deptype='link'):
            query = self.spec[dep.name]
            include.extend(query.headers.directories)
            library.extend(query.libs.directories)

        # README says to use TORCHVISION_INCLUDE and TORCHVISION_LIBRARY,
        # but these do not work for older releases. Build uses a mix of
        # Spack's compiler wrapper and the actual compiler, so this is
        # needed to get parts of the build working.
        # See https://github.com/pytorch/vision/issues/2591
        env.set('TORCHVISION_INCLUDE', ':'.join(include))
        env.set('TORCHVISION_LIBRARY', ':'.join(library))
        env.set('CPATH', ':'.join(include))
        env.set('LIBRARY_PATH', ':'.join(library))

        if '+cuda' in self.spec['py-torch']:
            env.set('FORCE_CUDA', 1)
            env.set('CUDA_HOME', self.spec['cuda'].prefix)
            pytorch_cuda_arch = ';'.join(
                '{0:.1f}'.format(float(i) / 10.0) for i in
                self.spec['py-torch'].variants['cuda_arch'].value
            )
            env.set('TORCH_CUDA_ARCH_LIST', pytorch_cuda_arch)
        else:
            env.set('FORCE_CUDA', 0)
<commit_msg>py-torchvision: add v0.10.0 (#24340)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

class PyTorchvision(PythonPackage):
    """The torchvision package consists of popular datasets, model
    architectures, and common image transformations for computer vision."""

    homepage = "https://github.com/pytorch/vision"
    url      = "https://github.com/pytorch/vision/archive/v0.8.2.tar.gz"
    git      = "https://github.com/pytorch/vision.git"

    maintainers = ['adamjstewart']

    version('master', branch='master')
    version('0.10.0', sha256='82bb2c2b03d8a65f4ea74bb0ee5566b0876a1992aceefb1e11475c7b5d2e857b')
    version('0.9.1', sha256='79964773729880e0eee0e6af13f336041121d4cc8491a3e2c0e5f184cac8a718')
    version('0.9.0', sha256='9351ed92aded632f8c7f59dfadac13c191a834babe682f5785ea47e6fcf6b472')
    version('0.8.2', sha256='9a866c3c8feb23b3221ce261e6153fc65a98ce9ceaa71ccad017016945c178bf')
    version('0.8.1', sha256='c46734c679c99f93e5c06654f4295a05a6afe6c00a35ebd26a2cce507ae1ccbd')
    version('0.8.0', sha256='b5f040faffbfc7bac8d4687d8665bd1196937334589b3fb5fcf15bb69ca25391')
    version('0.7.0', sha256='fa0a6f44a50451115d1499b3f2aa597e0092a07afce1068750260fa7dd2c85cb')
    version('0.6.1', sha256='8173680a976c833640ecbd0d7e6f0a11047bf8833433e2147180efc905e48656')
    version('0.6.0', sha256='02de11b3abe6882de4032ce86dab9c7794cbc84369b44d04e667486580f0f1f7')
    version('0.5.0', sha256='eb9afc93df3d174d975ee0914057a9522f5272310b4d56c150b955c287a4d74d')
    version('0.4.2', sha256='1184a27eab85c9e784bacc6f9d6fec99e168ab4eda6047ef9f709e7fdb22d8f9')
    version('0.4.1', sha256='053689351272b3bd2ac3e6ba51efd284de0e4ca4a301f54674b949f1e62b7176')
    version('0.4.0', sha256='c270d74e568bad4559fed4544f6dd1e22e2eb1c60b088e04a5bd5787c4150589')
    version('0.3.0', sha256='c205f0618c268c6ed2f8abb869ef6eb83e5339c1336c243ad321a2f2a85195f0')

    # https://github.com/pytorch/vision#image-backend
    variant('backend', default='pil', description='Image backend',
            values=('pil', 'accimage', 'png', 'jpeg'), multi=False)

    # https://github.com/pytorch/vision#installation
    depends_on('python@3.6:', when='@0.7:', type=('build', 'link', 'run'))
    depends_on('python@3.5:', when='@0.6.0:0.6.999', type=('build', 'link', 'run'))
    depends_on('python@2.7:2.8,3.5:3.8', when='@0.5.0', type=('build', 'link', 'run'))
    depends_on('python@2.7:2.8,3.5:3.7', when='@:0.4', type=('build', 'link', 'run'))

    depends_on('py-setuptools', type='build')
    depends_on('ninja', type='build')
    depends_on('py-numpy', type=('build', 'run'))
    depends_on('py-six', when='@:0.5', type=('build', 'run'))

    # https://github.com/pytorch/vision#installation
    depends_on('py-torch@master', when='@master', type=('build', 'link', 'run'))
    depends_on('py-torch@1.9.0', when='@0.10.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.8.1', when='@0.9.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.8.0', when='@0.9.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.1', when='@0.8.2', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.0', when='@0.8.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.7.0', when='@0.8.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.6.0', when='@0.7.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.5.1', when='@0.6.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.5.0', when='@0.6.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.4.1', when='@0.5.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.3.1', when='@0.4.2', type=('build', 'link', 'run'))
    depends_on('py-torch@1.3.0', when='@0.4.1', type=('build', 'link', 'run'))
    depends_on('py-torch@1.2.0', when='@0.4.0', type=('build', 'link', 'run'))
    depends_on('py-torch@1.1.0', when='@0.3.0', type=('build', 'link', 'run'))
    depends_on('py-torch@:1.0.1', when='@0.2.2', type=('build', 'link', 'run'))

    # https://github.com/pytorch/vision/issues/1712
    depends_on('pil@4.1.1:6', when='@:0.4 backend=pil', type=('build', 'run'))
    depends_on('pil@4.1.1:',  when='@0.5: backend=pil', type=('build', 'run'))
    depends_on('pil@5.3.0:', when='@0.10: backend=pil', type=('build', 'run'))
    depends_on('py-accimage', when='backend=accimage', type=('build', 'run'))
    depends_on('libpng@1.6.0:', when='backend=png')
    depends_on('jpeg', when='backend=jpeg')

    # Many of the datasets require additional dependencies to use.
    # These can be installed after the fact.

    depends_on('ffmpeg@3.1:', when='@0.4.2:')

    conflicts('backend=png', when='@:0.7')
    conflicts('backend=jpeg', when='@:0.7')

    def setup_build_environment(self, env):
        include = []
        library = []
        for dep in self.spec.dependencies(deptype='link'):
            query = self.spec[dep.name]
            include.extend(query.headers.directories)
            library.extend(query.libs.directories)

        # README says to use TORCHVISION_INCLUDE and TORCHVISION_LIBRARY,
        # but these do not work for older releases. Build uses a mix of
        # Spack's compiler wrapper and the actual compiler, so this is
        # needed to get parts of the build working.
        # See https://github.com/pytorch/vision/issues/2591
        env.set('TORCHVISION_INCLUDE', ':'.join(include))
        env.set('TORCHVISION_LIBRARY', ':'.join(library))
        env.set('CPATH', ':'.join(include))
        env.set('LIBRARY_PATH', ':'.join(library))

        if '+cuda' in self.spec['py-torch']:
            env.set('FORCE_CUDA', 1)
            env.set('CUDA_HOME', self.spec['cuda'].prefix)
            pytorch_cuda_arch = ';'.join(
                '{0:.1f}'.format(float(i) / 10.0) for i in
                self.spec['py-torch'].variants['cuda_arch'].value
            )
            env.set('TORCH_CUDA_ARCH_LIST', pytorch_cuda_arch)
        else:
            env.set('FORCE_CUDA', 0)
<|endoftext|>


<commit_before>from csv import writer, QUOTE_MINIMAL
from pymuse.pipelinestages.pipeline_stage import PipelineStage
import time
from pymuse.signal import SignalData
from queue import Queue

class MuseCSVOutputStream(PipelineStage):
    def __init__(self):
        super().__init__()

    def execute(self): # Pour l'instant, ce sera une étape bidon du pipeline qui ne fait que mettre les résultats sur la sortie standard.
        print('MuseCSVOutputStream Forked!')

        time_val = time.time()
        for i in range(256*4):
            data = self._queue_in.get()
        print("Exec time of MuseCSVOutputStream "+ str(time.time() - time_val))

        # with open('asd.csv', 'w', newline='') as csv_file:
        #     csv_writer = writer(csv_file, delimiter=',', quotechar='|', quoting=QUOTE_MINIMAL)
        #     while(True):
        #         while(self._queue_in.qsize = )
<commit_msg>can write to csv file<commit_after>from csv import writer
from datetime import date
import time
from pymuse.pipelinestages.pipeline_stage import PipelineStage
from pymuse.signal import SignalData

DEFAULT_FILE_NAME = "MuseData%s.csv"%date.today()
DEFAULT_COLUMN_PREFIX = "electrode"
TIME_COLUMN_NAME = "time"

class MuseCSVOutputStream(PipelineStage):
    def __init__(self, file_name=DEFAULT_FILE_NAME, column_prefix=DEFAULT_COLUMN_PREFIX):
        super().__init__()
        self._file_name = file_name
        self._column_prefix = column_prefix

    def execute(self): # Pour l'instant, ce sera une étape bidon du pipeline qui ne fait que mettre les résultats sur la sortie standard.
        print('MuseCSVOutputStream Forked!')
        timeinit = time.time()
        try:
            with open(self._file_name, 'w', newline='') as csv_file:
                csv_writer = writer(csv_file)
                print("before")
                data = self._queue_in.get()
                print("data, values :", data.values)
                self._setHeaderFile(csv_writer, len(data.values))
                for i in range(256*3 - 1):
                    csv_writer.writerow([data.time] + [electrode for electrode in data.values])
                    data = self._queue_in.get()
        except PermissionError as err:
            print("MuseCSVOutputStream: Cannot open file: %s"%(err))

        print("finished, exec time: ", time.time() - timeinit)

    def _setHeaderFile(self, csv_writer, nb_samples):
        value_column_names = ["%s %i"%(self._column_prefix, x) for x in range(nb_samples)]
        column_names = [TIME_COLUMN_NAME] + value_column_names
        csv_writer.writerow(column_names)
<|endoftext|>


<commit_before>import os, time

from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.exceptions import NotConfigured
from scrapy import log, signals

class DeltaFetch(object):
    """This is a spider middleware to ignore requests to pages containing items
    seen in previous crawls of the same spider, thus producing a "delta crawl"
    containing only new items.

    This also speeds up the crawl, by reducing the number of requests that need
    to be crawled, and processed (typically, item requests are the most cpu
    intensive).

    Supported settings:

    * DELTAFETCH_ENABLED - to enable (or disable) this extension
    * DELTAFETCH_DIR - directory where to store state
    * DELTAFETCH_RESET - reset the state, clearing out all seen requests

    Supported spider arguments:

    * deltafetch_reset - same effect as DELTAFETCH_RESET setting

    Supported request meta keys:

    * deltafetch_key - used to define the lookup key for that request. by
      default it's the fingerprint, but it can be changed to contain an item
      id, for example. This requires support from the spider, but makes the
      extension more efficient for sites that many URLs for the same item.

    """

    def __init__(self, dir, stats=None, reset=False):
        dbmodule = None
        try:
            dbmodule = __import__('bsddb3').db
        except ImportError:
            try:
                dbmodule = __import__('bsddb').db
            except ImportError:
                pass
        if not dbmodule:
            raise NotConfigured('bssdb or bsddb3 is required')
        self.dbmodule = dbmodule
        self.dir = dir
        self.reset = reset
        self.stats = stats

    @classmethod
    def from_crawler(cls, crawler):
        s = crawler.settings
        if not s.getbool('DELTAFETCH_ENABLED'):
            raise NotConfigured
        dir = data_path(s.get('DELTAFETCH_DIR', 'deltafetch'))
        reset = s.getbool('DELTAFETCH_RESET')
        o = cls(dir, crawler.stats, reset)
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
        return o

    def spider_opened(self, spider):
        if not os.path.exists(self.dir):
            os.makedirs(self.dir)
        dbpath = os.path.join(self.dir, '%s.db' % spider.name)
        reset = self.reset or getattr(spider, 'deltafetch_reset', False)
        flag = self.dbmodule.DB_TRUNCATE if reset else self.dbmodule.DB_CREATE
        try:
            self.db = self.dbmodule.DB()
            self.db.open(filename=dbpath,
                         dbtype=self.dbmodule.DB_HASH,
                         flags=flag)
        except Exception:
            spider.log("Failed to open DeltaFetch database at %s, "
                       "trying to recreate it" % dbpath)
            if os.path.exists(dbpath):
                os.remove(dbpath)
            self.db = self.dbmodule.DB()
            self.db.open(filename=dbpath,
                         dbtype=self.dbmodule.DB_HASH,
                         flags=self.dbmodule.DB_CREATE)

    def spider_closed(self, spider):
        self.db.close()

    def process_spider_output(self, response, result, spider):
        for r in result:
            if isinstance(r, Request):
                key = self._get_key(r)
                if self.db.has_key(key):
                    spider.log("Ignoring already visited: %s" % r, level=log.INFO)
                    self.stats.inc_value('deltafetch/skipped', spider=spider)
                    continue
            elif isinstance(r, BaseItem):
                key = self._get_key(response.request)
                self.db[key] = str(time.time())
                self.stats.inc_value('deltafetch/stored', spider=spider)
            yield r

    def _get_key(self, request):
        return request.meta.get('deltafetch_key') or request_fingerprint(request)
<commit_msg>Log skipped urls from deltafetch at the debug level<commit_after>import os, time

from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils.request import request_fingerprint
from scrapy.utils.project import data_path
from scrapy.exceptions import NotConfigured
from scrapy import log, signals

class DeltaFetch(object):
    """This is a spider middleware to ignore requests to pages containing items
    seen in previous crawls of the same spider, thus producing a "delta crawl"
    containing only new items.

    This also speeds up the crawl, by reducing the number of requests that need
    to be crawled, and processed (typically, item requests are the most cpu
    intensive).

    Supported settings:

    * DELTAFETCH_ENABLED - to enable (or disable) this extension
    * DELTAFETCH_DIR - directory where to store state
    * DELTAFETCH_RESET - reset the state, clearing out all seen requests

    Supported spider arguments:

    * deltafetch_reset - same effect as DELTAFETCH_RESET setting

    Supported request meta keys:

    * deltafetch_key - used to define the lookup key for that request. by
      default it's the fingerprint, but it can be changed to contain an item
      id, for example. This requires support from the spider, but makes the
      extension more efficient for sites that many URLs for the same item.

    """

    def __init__(self, dir, stats=None, reset=False):
        dbmodule = None
        try:
            dbmodule = __import__('bsddb3').db
        except ImportError:
            try:
                dbmodule = __import__('bsddb').db
            except ImportError:
                pass
        if not dbmodule:
            raise NotConfigured('bssdb or bsddb3 is required')
        self.dbmodule = dbmodule
        self.dir = dir
        self.reset = reset
        self.stats = stats

    @classmethod
    def from_crawler(cls, crawler):
        s = crawler.settings
        if not s.getbool('DELTAFETCH_ENABLED'):
            raise NotConfigured
        dir = data_path(s.get('DELTAFETCH_DIR', 'deltafetch'))
        reset = s.getbool('DELTAFETCH_RESET')
        o = cls(dir, crawler.stats, reset)
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
        return o

    def spider_opened(self, spider):
        if not os.path.exists(self.dir):
            os.makedirs(self.dir)
        dbpath = os.path.join(self.dir, '%s.db' % spider.name)
        reset = self.reset or getattr(spider, 'deltafetch_reset', False)
        flag = self.dbmodule.DB_TRUNCATE if reset else self.dbmodule.DB_CREATE
        try:
            self.db = self.dbmodule.DB()
            self.db.open(filename=dbpath,
                         dbtype=self.dbmodule.DB_HASH,
                         flags=flag)
        except Exception:
            spider.log("Failed to open DeltaFetch database at %s, "
                       "trying to recreate it" % dbpath)
            if os.path.exists(dbpath):
                os.remove(dbpath)
            self.db = self.dbmodule.DB()
            self.db.open(filename=dbpath,
                         dbtype=self.dbmodule.DB_HASH,
                         flags=self.dbmodule.DB_CREATE)

    def spider_closed(self, spider):
        self.db.close()

    def process_spider_output(self, response, result, spider):
        for r in result:
            if isinstance(r, Request):
                key = self._get_key(r)
                if self.db.has_key(key):
                    spider.log("Ignoring already visited: %s" % r, level=log.DEBUG)
                    self.stats.inc_value('deltafetch/skipped', spider=spider)
                    continue
            elif isinstance(r, BaseItem):
                key = self._get_key(response.request)
                self.db[key] = str(time.time())
                self.stats.inc_value('deltafetch/stored', spider=spider)
            yield r

    def _get_key(self, request):
        return request.meta.get('deltafetch_key') or request_fingerprint(request)
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyRegex(PythonPackage):
    """Alternative regular expression module, to replace re."""

    pypi = "regex/regex-2020.11.13.tar.gz"

    version('2020.11.13', sha256='83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562')
    version('2019.11.1',  sha256='720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69')
    version('2017.07.11', sha256='dbda8bdc31a1c85445f1a1b29d04abda46e5c690f8f933a9cc3a85a358969616')

    depends_on('py-setuptools', type='build', when='@:2018,2020:')
<commit_msg>py-regex: add older versions (#23724)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyRegex(PythonPackage):
    """Alternative regular expression module, to replace re."""

    pypi = "regex/regex-2020.11.13.tar.gz"

    version('2020.11.13', sha256='83d6b356e116ca119db8e7c6fc2983289d87b27b3fac238cfe5dca529d884562')
    version('2019.11.1',  sha256='720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69')
    version('2019.02.07', sha256='4a1a1d963f462c13722b34ef1f82c4707091b0a3fb9b5fd79b6670c38b734095')
    version('2018.01.10', sha256='139678fc013b75e486e580c39b4c52d085ed7362e400960f8be1711a414f16b5')
    version('2017.07.11', sha256='dbda8bdc31a1c85445f1a1b29d04abda46e5c690f8f933a9cc3a85a358969616')

    depends_on('py-setuptools', type='build', when='@:2018,2020:')
<|endoftext|>