<commit_before>import json

from django.contrib.auth.mixins import UserPassesTestMixin
from django.core.exceptions import PermissionDenied
from django.db.models import Q, Count
from django.shortcuts import HttpResponse, get_object_or_404, reverse, render, Http404, redirect
from django.views.generic import TemplateView, View, FormView
from django.views.generic.base import ContextMixin, TemplateResponseMixin
from django.views.generic.list import ListView
from django.contrib.auth.mixins import LoginRequiredMixin
from tagging.models import Tag, TaggedItem, ContentType

from account.models import User
from account.permissions import is_admin_or_root
from submission.models import Submission, SubmissionStatus, STATUS_CHOICE
from submission.views import render_submission
from submission.statistics import get_accept_problem_list, get_attempted_problem_list, is_problem_accepted
from utils.comment import CommentForm
from utils.language import LANG_CHOICE
from utils.tagging import edit_string_for_tags
from .models import Problem
from utils.permission import is_problem_manager, get_permission_for_submission
from .statistics import (
    get_many_problem_accept_count, get_problem_accept_count, get_problem_accept_ratio, get_problem_accept_user_count,
    get_problem_accept_user_ratio, get_problem_all_count, get_problem_all_user_count, get_many_problem_difficulty,
    get_problem_difficulty, get_problem_stats
)
from .tasks import create_submission, judge_submission_on_problem

class ProblemList(ListView):
    template_name = 'problem/index.jinja2'
    paginate_by = 100
    context_object_name = 'problem_list'

    def get_queryset(self):
        source = self.request.GET.get('source')
        kw = self.request.GET.get('keyword')
        tg = self.request.GET.get('tag')
        if tg:
            queryset = TaggedItem.objects.get_by_model(Problem, get_object_or_404(Tag, name=tg))
        else:
            queryset = Problem.objects.all()
        if source:
            queryset = queryset.filter(source=source)
        if not is_admin_or_root(self.request.user):
            queryset = queryset.filter(visible=True)

        if kw:
            q = Q(title__icontains=kw) | Q(source__icontains=kw)
            if kw.isdigit():
                q |= Q(pk__exact=kw)
            queryset = queryset.filter(q)
            if not self.request.user.is_authenticated or self.request.user.show_tags:
                tag = Tag.objects.filter(name=kw)
                if tag.exists():
                    queryset = (queryset | TaggedItem.objects.get_by_model(Problem, tag.first())).distinct()

        return queryset

    def get_context_data(self, **kwargs):
        data = super(ProblemList, self).get_context_data(**kwargs)
        data['keyword'] = self.request.GET.get('keyword')
        current_problem_set = [problem.pk for problem in data['problem_list']]
        for problem in data['problem_list']:
            problem.personal_label = 0
        if self.request.user.is_authenticated:
            # Get AC / Wrong
            attempt_list = set(get_attempted_problem_list(self.request.user.id))
            accept_list = set(get_accept_problem_list(self.request.user.id))
            for problem in data['problem_list']:
                if problem.id in accept_list:
                    problem.personal_label = 1
                elif problem.id in attempt_list:
                    problem.personal_label = -1

        # Get Accepted of all users
        problem_ids = list(map(lambda x: x.id, data['problem_list']))
        accept_count = get_many_problem_accept_count(problem_ids)
        difficulties = get_many_problem_difficulty(problem_ids)
        for problem in data['problem_list']:
            problem.accept_count = accept_count[problem.id]
            problem.difficulty = difficulties[problem.id]

        # Get tags
        tagged_items = list(TaggedItem.objects.filter(content_type=ContentType.objects.get_for_model(Problem))
                            .filter(object_id__in=current_problem_set).select_related("tag"))
        for problem in data['problem_list']:
            items = list(filter(lambda x: x.object_id == problem.pk, tagged_items))
            if items:
                problem.my_tags = map(lambda x: x.tag.name, items)

        return data

class ProblemDetailMixin(TemplateResponseMixin, ContextMixin, UserPassesTestMixin):

    def dispatch(self, request, *args, **kwargs):
        self.problem = get_object_or_404(Problem, pk=kwargs.get('pk'))
        self.user = request.user
        self.privileged = is_problem_manager(self.user, self.problem)
        self.request = request
        return super(ProblemDetailMixin, self).dispatch(request, *args, **kwargs)

    def test_func(self):
        return self.privileged or self.problem.visible

    def get_context_data(self, **kwargs):
        data = super(ProblemDetailMixin, self).get_context_data(**kwargs)
        data['problem'] = self.problem
        data['is_privileged'] = self.privileged
        return data

class DiscussionView(ProblemDetailMixin, FormView):

    template_name = 'problem/detail/discussion.jinja2'
    form_class = CommentForm

    def get_context_data(self, **kwargs):
        data = super(DiscussionView, self).get_context_data(**kwargs)
        data['action_path'] = reverse('comments-post-comment')
        return data

    def get_form_kwargs(self):
        kw = super(DiscussionView, self).get_form_kwargs()
        kw['target_object'] = self.problem
        return kw

class ProblemView(ProblemDetailMixin, TemplateView):

    template_name = 'problem/detail/problem.jinja2'

    def get_context_data(self, **kwargs):
        data = super(ProblemView, self).get_context_data()
        if not is_admin_or_root(self.request.user) and not self.problem.visible:
            raise PermissionDenied("You don't have the access.")
        data['problem'] = self.problem

        show_tags = True
        if self.request.user.is_authenticated:
            show_tags = self.request.user.show_tags
        if show_tags:
            data['tags'] = self.problem.tags

        return data

class ProblemSubmitView(ProblemDetailMixin, TemplateView):

    template_name = 'problem/detail/submit.jinja2'

    def test_func(self):
        return super(ProblemSubmitView, self).test_func() and self.user.is_authenticated

    def get_context_data(self, **kwargs):
        data = super(ProblemSubmitView, self).get_context_data(**kwargs)
        submission_pk = self.request.GET.get('submission', None)
        if submission_pk:
            submission = Submission.objects.get(pk=submission_pk)
            if get_permission_for_submission(self.request.user, submission):
                data['code'] = submission.code
        data['lang_choices'] = LANG_CHOICE
        return data

    def post(self, request, pk):
        submission = create_submission(self.problem, self.user, request.POST['code'], request.POST['lang'])
        judge_submission_on_problem(submission)
        return HttpResponse(json.dumps({"url": reverse('problem:submission_api',
                                                       kwargs={'pk': self.problem.id, 'sid': submission.id})}))

class StatusList(ListView):

    template_name = 'problem/status.jinja2'
    paginate_by = 50
    context_object_name = 'submission_list'
    allow_problem_query = True
    allow_verdict_query = True
    query_number = 10000
    distinct_by_author = False  # query number should not be too large when this is true
    contest_submission_visible = False

    def get_selected_from(self):
        return Submission.objects.all()

    def reinterpret_problem_identifier(self, value):
        return value

    def get_queryset(self):
        try:
            queryset = self.get_selected_from().select_related('problem', 'author').\
                only('pk', 'contest_id', 'create_time', 'author_id', 'author__username', 'author__nickname',
                     'author__magic', 'problem_id', 'problem__title', 'lang', 'status', 'status_time', 'status_percent',
                     'code_length')
            if not self.contest_submission_visible and not is_admin_or_root(self.request.user):
                queryset = queryset.filter(contest__isnull=True, problem__visible=True)

            if 'user' in self.request.GET:
                queryset = queryset.filter(author_id=self.request.GET['user'])
            if self.allow_problem_query and 'problem' in self.request.GET:
                queryset = queryset.filter(problem_id=self.reinterpret_problem_identifier(self.request.GET['problem']))
            if 'lang' in self.request.GET:
                queryset = queryset.filter(lang=self.request.GET['lang'])
            if self.allow_verdict_query and 'verdict' in self.request.GET:
                queryset = queryset.filter(status=int(self.request.GET['verdict'][1:]))
            #
            # if kw:
            #     q = Q(author__username__iexact=kw)
            #     if kw.isdigit():
            #     q |= Q(pk__exact=kw) | Q(problem__pk__exact=kw)
            #     queryset = queryset.filter(q)

            if self.distinct_by_author:
                author_set = set()
                res = []
                for submission in queryset.all():
                    if submission.author_id not in author_set:
                        author_set.add(submission.author_id)
                        res.append(submission)
                        if len(res) >= self.query_number:
                            break
                return res
            else:
                return queryset.all()[:self.query_number]
        except:
            raise Http404

    def get_context_data(self, **kwargs):
        data = super(StatusList, self).get_context_data(**kwargs)
        user = self.request.user
        data['keyword'] = self.request.GET.get('keyword')
        data['param_verdict'], data['param_lang'], data['param_user'], data['param_problem'] = \
            self.request.GET.get('verdict', ''), self.request.GET.get('lang', ''),\
            self.request.GET.get('user', ''), self.request.GET.get('problem', '')
        data['allow_verdict_query'] = self.allow_verdict_query
        data['lang_choices'] = LANG_CHOICE
        data['verdict_choices'] = STATUS_CHOICE

        if user.is_authenticated:
            for submission in data['submission_list']:
                if is_admin_or_root(user) or submission.author == user:
                    submission.is_privileged = True
        return data

class ProblemStatisticsView(ProblemDetailMixin, StatusList):

    template_name = 'problem/detail/statistics.jinja2'
    paginate_by = None
    allow_problem_query = False
    allow_verdict_query = False
    query_number = 10
    distinct_by_author = True
    contest_submission_visible = True

    def get_selected_from(self):
        if self.request.GET.get('type') == 'shortest':
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("code_length")
        elif self.request.GET.get('type') == 'fastest':
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("status_time")
        else:
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("-create_time")

    def get_context_data(self, **kwargs):
        data = super(ProblemStatisticsView, self).get_context_data(**kwargs)
        data['user_ac_count'] = get_problem_accept_user_count(self.problem.id)
        data['user_all_count'] = get_problem_all_user_count(self.problem.id)
        data['user_ratio'] = get_problem_accept_user_ratio(self.problem.id)
        data['ac_count'] = get_problem_accept_count(self.problem.id)
        data['all_count'] = get_problem_all_count(self.problem.id)
        data['ratio'] = get_problem_accept_ratio(self.problem.id)
        data['difficulty'] = get_problem_difficulty(self.problem.id)
        data['stats'] = get_problem_stats(self.problem.id)
        data['param_type'] = self.request.GET.get('type', 'latest')
        data['tags'] = edit_string_for_tags(self.problem.tags)
        data['tags_choices'] = Tag.objects.all().values_list("name", flat=True)
        data['public_edit_access'] = is_problem_accepted(self.request.user, self.problem)
        return data

class ProblemUpdateTags(ProblemDetailMixin, View):
    @staticmethod
    def clear_tags(text):
        return ''.join(list(map(lambda x: x + ',', filter(lambda u: Tag.objects.filter(name=u).exists(),
                                                          map(lambda t: t.strip(), text.split(',')))))[:5])

    def post(self, request, *args, **kwargs):
        if not is_problem_accepted(self.request.user, self.problem):
            raise PermissionDenied
        tags = self.__class__.clear_tags(request.POST['tags'])
        if tags:
            self.problem.tags = tags
            self.problem.save()
        return redirect(request.POST['next'])

class ProblemPersonalOlderSubmissionsAPI(UserPassesTestMixin, TemplateView):

    template_name = 'components/past_submissions.jinja2'

    def test_func(self):
        return self.request.user.is_authenticated

    def get_context_data(self, **kwargs):
        submission_set = Submission.objects.only("problem_id", "id", "status", "create_time",
                                                 "author_id", "author__username", "author__magic"). \
            filter(author_id=self.request.user.pk, problem_id=kwargs.get('pk'))
        return {'submission_list': submission_set}

class ProblemSubmissionAPI(LoginRequiredMixin, View):

    def get(self, request, pk, sid):
        submission = get_object_or_404(Submission, problem_id=pk, author=self.request.user, pk=sid)
        return HttpResponse(render_submission(submission,
                                              permission=get_permission_for_submission(request.user, submission),
                                              hide_problem=True))

class ProblemSubmissionView(LoginRequiredMixin, TemplateView):

    template_name = 'submission.jinja2'

    def get_context_data(self, **kwargs):
        data = super(ProblemSubmissionView, self).get_context_data(**kwargs)
        data['submission'] = submission = get_object_or_404(Submission, pk=self.kwargs.get('sid'),
                                                                        problem_id=self.kwargs.get('pk'))
        if self.request.user.is_authenticated and (
                            submission.author == self.request.user or
                        is_problem_manager(self.request.user,
                                           submission.problem) or
                    self.request.user.submission_set.filter(
                        problem_id=self.kwargs.get('pk'),
                        status=SubmissionStatus.ACCEPTED,
                        contest__isnull=True).exists()):
            data['submission_block'] = render_submission(submission, permission=get_permission_for_submission(self.request.user,
                                                                                submission,
                                                                                special_permission=True))
        else:
            raise PermissionDenied
        data['problem'] = submission.problem
        return data

class Millionaires(ListView):
    template_name = 'problem/standings.jinja2'
    paginate_by = 100
    context_object_name = 'rank_list'

    def get_queryset(self):
        return User.objects.only("username", "magic", "score").filter(score__gt=0)

    def get_context_data(self, **kwargs):
        data = super(Millionaires, self).get_context_data(**kwargs)
        if not self.request.user.is_authenticated:
            data['my_rank'] = 'N/A'
        else:
            data['my_rank'] = User.objects.filter(score__gte=self.request.user.score).count()
        return data

class SourceList(ListView):
    template_name = 'problem/source.jinja2'
    context_object_name = 'source_list'

    def get_queryset(self):
        queryset = Problem.objects.all()
        if not is_admin_or_root(self.request.user):
            queryset = queryset.filter(visible=True)
        return queryset.only('source').exclude(source='').\
            order_by('source').values('source').annotate(count=Count('source'))
<commit_msg>fix: contest submission visible outside<commit_after>import json

from django.contrib.auth.mixins import UserPassesTestMixin
from django.core.exceptions import PermissionDenied
from django.db.models import Q, Count
from django.shortcuts import HttpResponse, get_object_or_404, reverse, render, Http404, redirect
from django.views.generic import TemplateView, View, FormView
from django.views.generic.base import ContextMixin, TemplateResponseMixin
from django.views.generic.list import ListView
from django.contrib.auth.mixins import LoginRequiredMixin
from tagging.models import Tag, TaggedItem, ContentType

from account.models import User
from account.permissions import is_admin_or_root
from submission.models import Submission, SubmissionStatus, STATUS_CHOICE
from submission.views import render_submission
from submission.statistics import get_accept_problem_list, get_attempted_problem_list, is_problem_accepted
from utils.comment import CommentForm
from utils.language import LANG_CHOICE
from utils.tagging import edit_string_for_tags
from .models import Problem
from utils.permission import is_problem_manager, get_permission_for_submission
from .statistics import (
    get_many_problem_accept_count, get_problem_accept_count, get_problem_accept_ratio, get_problem_accept_user_count,
    get_problem_accept_user_ratio, get_problem_all_count, get_problem_all_user_count, get_many_problem_difficulty,
    get_problem_difficulty, get_problem_stats
)
from .tasks import create_submission, judge_submission_on_problem

class ProblemList(ListView):
    template_name = 'problem/index.jinja2'
    paginate_by = 100
    context_object_name = 'problem_list'

    def get_queryset(self):
        source = self.request.GET.get('source')
        kw = self.request.GET.get('keyword')
        tg = self.request.GET.get('tag')
        if tg:
            queryset = TaggedItem.objects.get_by_model(Problem, get_object_or_404(Tag, name=tg))
        else:
            queryset = Problem.objects.all()
        if source:
            queryset = queryset.filter(source=source)
        if not is_admin_or_root(self.request.user):
            queryset = queryset.filter(visible=True)

        if kw:
            q = Q(title__icontains=kw) | Q(source__icontains=kw)
            if kw.isdigit():
                q |= Q(pk__exact=kw)
            queryset = queryset.filter(q)
            if not self.request.user.is_authenticated or self.request.user.show_tags:
                tag = Tag.objects.filter(name=kw)
                if tag.exists():
                    queryset = (queryset | TaggedItem.objects.get_by_model(Problem, tag.first())).distinct()

        return queryset

    def get_context_data(self, **kwargs):
        data = super(ProblemList, self).get_context_data(**kwargs)
        data['keyword'] = self.request.GET.get('keyword')
        current_problem_set = [problem.pk for problem in data['problem_list']]
        for problem in data['problem_list']:
            problem.personal_label = 0
        if self.request.user.is_authenticated:
            # Get AC / Wrong
            attempt_list = set(get_attempted_problem_list(self.request.user.id))
            accept_list = set(get_accept_problem_list(self.request.user.id))
            for problem in data['problem_list']:
                if problem.id in accept_list:
                    problem.personal_label = 1
                elif problem.id in attempt_list:
                    problem.personal_label = -1

        # Get Accepted of all users
        problem_ids = list(map(lambda x: x.id, data['problem_list']))
        accept_count = get_many_problem_accept_count(problem_ids)
        difficulties = get_many_problem_difficulty(problem_ids)
        for problem in data['problem_list']:
            problem.accept_count = accept_count[problem.id]
            problem.difficulty = difficulties[problem.id]

        # Get tags
        tagged_items = list(TaggedItem.objects.filter(content_type=ContentType.objects.get_for_model(Problem))
                            .filter(object_id__in=current_problem_set).select_related("tag"))
        for problem in data['problem_list']:
            items = list(filter(lambda x: x.object_id == problem.pk, tagged_items))
            if items:
                problem.my_tags = map(lambda x: x.tag.name, items)

        return data

class ProblemDetailMixin(TemplateResponseMixin, ContextMixin, UserPassesTestMixin):

    def dispatch(self, request, *args, **kwargs):
        self.problem = get_object_or_404(Problem, pk=kwargs.get('pk'))
        self.user = request.user
        self.privileged = is_problem_manager(self.user, self.problem)
        self.request = request
        return super(ProblemDetailMixin, self).dispatch(request, *args, **kwargs)

    def test_func(self):
        return self.privileged or self.problem.visible

    def get_context_data(self, **kwargs):
        data = super(ProblemDetailMixin, self).get_context_data(**kwargs)
        data['problem'] = self.problem
        data['is_privileged'] = self.privileged
        return data

class DiscussionView(ProblemDetailMixin, FormView):

    template_name = 'problem/detail/discussion.jinja2'
    form_class = CommentForm

    def get_context_data(self, **kwargs):
        data = super(DiscussionView, self).get_context_data(**kwargs)
        data['action_path'] = reverse('comments-post-comment')
        return data

    def get_form_kwargs(self):
        kw = super(DiscussionView, self).get_form_kwargs()
        kw['target_object'] = self.problem
        return kw

class ProblemView(ProblemDetailMixin, TemplateView):

    template_name = 'problem/detail/problem.jinja2'

    def get_context_data(self, **kwargs):
        data = super(ProblemView, self).get_context_data()
        if not is_admin_or_root(self.request.user) and not self.problem.visible:
            raise PermissionDenied("You don't have the access.")
        data['problem'] = self.problem

        show_tags = True
        if self.request.user.is_authenticated:
            show_tags = self.request.user.show_tags
        if show_tags:
            data['tags'] = self.problem.tags

        return data

class ProblemSubmitView(ProblemDetailMixin, TemplateView):

    template_name = 'problem/detail/submit.jinja2'

    def test_func(self):
        return super(ProblemSubmitView, self).test_func() and self.user.is_authenticated

    def get_context_data(self, **kwargs):
        data = super(ProblemSubmitView, self).get_context_data(**kwargs)
        submission_pk = self.request.GET.get('submission', None)
        if submission_pk:
            submission = Submission.objects.get(pk=submission_pk)
            if get_permission_for_submission(self.request.user, submission):
                data['code'] = submission.code
        data['lang_choices'] = LANG_CHOICE
        return data

    def post(self, request, pk):
        submission = create_submission(self.problem, self.user, request.POST['code'], request.POST['lang'])
        judge_submission_on_problem(submission)
        return HttpResponse(json.dumps({"url": reverse('problem:submission_api',
                                                       kwargs={'pk': self.problem.id, 'sid': submission.id})}))

class StatusList(ListView):

    template_name = 'problem/status.jinja2'
    paginate_by = 50
    context_object_name = 'submission_list'
    allow_problem_query = True
    allow_verdict_query = True
    query_number = 10000
    distinct_by_author = False  # query number should not be too large when this is true
    contest_submission_visible = False

    def get_selected_from(self):
        return Submission.objects.all()

    def reinterpret_problem_identifier(self, value):
        return value

    def get_queryset(self):
        try:
            queryset = self.get_selected_from().select_related('problem', 'author').\
                only('pk', 'contest_id', 'create_time', 'author_id', 'author__username', 'author__nickname',
                     'author__magic', 'problem_id', 'problem__title', 'lang', 'status', 'status_time', 'status_percent',
                     'code_length')
            if not self.contest_submission_visible and not is_admin_or_root(self.request.user):
                queryset = queryset.filter(contest__isnull=True, problem__visible=True)

            if 'user' in self.request.GET:
                queryset = queryset.filter(author_id=self.request.GET['user'])
            if self.allow_problem_query and 'problem' in self.request.GET:
                queryset = queryset.filter(problem_id=self.reinterpret_problem_identifier(self.request.GET['problem']))
            if 'lang' in self.request.GET:
                queryset = queryset.filter(lang=self.request.GET['lang'])
            if self.allow_verdict_query and 'verdict' in self.request.GET:
                queryset = queryset.filter(status=int(self.request.GET['verdict'][1:]))
            #
            # if kw:
            #     q = Q(author__username__iexact=kw)
            #     if kw.isdigit():
            #     q |= Q(pk__exact=kw) | Q(problem__pk__exact=kw)
            #     queryset = queryset.filter(q)

            if self.distinct_by_author:
                author_set = set()
                res = []
                for submission in queryset.all():
                    if submission.author_id not in author_set:
                        author_set.add(submission.author_id)
                        res.append(submission)
                        if len(res) >= self.query_number:
                            break
                return res
            else:
                return queryset.all()[:self.query_number]
        except:
            raise Http404

    def get_context_data(self, **kwargs):
        data = super(StatusList, self).get_context_data(**kwargs)
        user = self.request.user
        data['keyword'] = self.request.GET.get('keyword')
        data['param_verdict'], data['param_lang'], data['param_user'], data['param_problem'] = \
            self.request.GET.get('verdict', ''), self.request.GET.get('lang', ''),\
            self.request.GET.get('user', ''), self.request.GET.get('problem', '')
        data['allow_verdict_query'] = self.allow_verdict_query
        data['lang_choices'] = LANG_CHOICE
        data['verdict_choices'] = STATUS_CHOICE

        if user.is_authenticated:
            for submission in data['submission_list']:
                if is_admin_or_root(user) or submission.author == user:
                    submission.is_privileged = True
        return data

class ProblemStatisticsView(ProblemDetailMixin, StatusList):

    template_name = 'problem/detail/statistics.jinja2'
    paginate_by = None
    allow_problem_query = False
    allow_verdict_query = False
    query_number = 10
    distinct_by_author = True
    contest_submission_visible = True

    def get_selected_from(self):
        if self.request.GET.get('type') == 'shortest':
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("code_length")
        elif self.request.GET.get('type') == 'fastest':
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("status_time")
        else:
            return self.problem.submission_set.filter(status=SubmissionStatus.ACCEPTED).order_by("-create_time")

    def get_context_data(self, **kwargs):
        data = super(ProblemStatisticsView, self).get_context_data(**kwargs)
        data['user_ac_count'] = get_problem_accept_user_count(self.problem.id)
        data['user_all_count'] = get_problem_all_user_count(self.problem.id)
        data['user_ratio'] = get_problem_accept_user_ratio(self.problem.id)
        data['ac_count'] = get_problem_accept_count(self.problem.id)
        data['all_count'] = get_problem_all_count(self.problem.id)
        data['ratio'] = get_problem_accept_ratio(self.problem.id)
        data['difficulty'] = get_problem_difficulty(self.problem.id)
        data['stats'] = get_problem_stats(self.problem.id)
        data['param_type'] = self.request.GET.get('type', 'latest')
        data['tags'] = edit_string_for_tags(self.problem.tags)
        data['tags_choices'] = Tag.objects.all().values_list("name", flat=True)
        data['public_edit_access'] = is_problem_accepted(self.request.user, self.problem)
        return data

class ProblemUpdateTags(ProblemDetailMixin, View):
    @staticmethod
    def clear_tags(text):
        return ''.join(list(map(lambda x: x + ',', filter(lambda u: Tag.objects.filter(name=u).exists(),
                                                          map(lambda t: t.strip(), text.split(',')))))[:5])

    def post(self, request, *args, **kwargs):
        if not is_problem_accepted(self.request.user, self.problem):
            raise PermissionDenied
        tags = self.__class__.clear_tags(request.POST['tags'])
        if tags:
            self.problem.tags = tags
            self.problem.save()
        return redirect(request.POST['next'])

class ProblemPersonalOlderSubmissionsAPI(UserPassesTestMixin, TemplateView):

    template_name = 'components/past_submissions.jinja2'

    def test_func(self):
        return self.request.user.is_authenticated

    def get_context_data(self, **kwargs):
        submission_set = Submission.objects.only("problem_id", "id", "status", "create_time",
                                                 "author_id", "author__username", "author__magic"). \
            filter(author_id=self.request.user.pk, problem_id=kwargs.get('pk'))
        return {'submission_list': submission_set}

class ProblemSubmissionAPI(LoginRequiredMixin, View):

    def get(self, request, pk, sid):
        submission = get_object_or_404(Submission, problem_id=pk, author=self.request.user, pk=sid)
        return HttpResponse(render_submission(submission,
                                              permission=get_permission_for_submission(request.user, submission),
                                              hide_problem=True))

class ProblemSubmissionView(LoginRequiredMixin, TemplateView):

    template_name = 'submission.jinja2'

    def get_context_data(self, **kwargs):
        data = super(ProblemSubmissionView, self).get_context_data(**kwargs)
        data['submission'] = submission = get_object_or_404(Submission, pk=self.kwargs.get('sid'),
                                                                        problem_id=self.kwargs.get('pk'),
                                                                        contest__isnull=True)
        if self.request.user.is_authenticated and (
                            submission.author == self.request.user or
                        is_problem_manager(self.request.user,
                                           submission.problem) or
                    self.request.user.submission_set.filter(
                        problem_id=self.kwargs.get('pk'),
                        status=SubmissionStatus.ACCEPTED).exists()):
            data['submission_block'] = render_submission(submission, permission=get_permission_for_submission(self.request.user,
                                                                                submission,
                                                                                special_permission=True))
        else:
            raise PermissionDenied("Code is not public for users who have not got accepted yet.")
        data['problem'] = submission.problem
        return data

class Millionaires(ListView):
    template_name = 'problem/standings.jinja2'
    paginate_by = 100
    context_object_name = 'rank_list'

    def get_queryset(self):
        return User.objects.only("username", "magic", "score").filter(score__gt=0)

    def get_context_data(self, **kwargs):
        data = super(Millionaires, self).get_context_data(**kwargs)
        if not self.request.user.is_authenticated:
            data['my_rank'] = 'N/A'
        else:
            data['my_rank'] = User.objects.filter(score__gte=self.request.user.score).count()
        return data

class SourceList(ListView):
    template_name = 'problem/source.jinja2'
    context_object_name = 'source_list'

    def get_queryset(self):
        queryset = Problem.objects.all()
        if not is_admin_or_root(self.request.user):
            queryset = queryset.filter(visible=True)
        return queryset.only('source').exclude(source='').\
            order_by('source').values('source').annotate(count=Count('source'))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from hashlib import md5

from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseForbidden, Http404
from django.shortcuts import get_object_or_404, render_to_response
from django.template import loader, RequestContext
from django.template.defaultfilters import slugify
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.models import User

from tomo.problem.models import Problem, Submission, Solution, Part
from tomo.settings import SECRET_KEY

def sign(text):
    return md5(text + SECRET_KEY).hexdigest()

def verify(text, signature):
    return signature == sign(text)

def check_problem(problem, user):
    if problem.status == '10' and not user.is_staff:
        raise Http404

def problem_list(request):
    if request.user.is_staff:
        problems = Problem.objects.select_related('parts').all()
    else:
        problems = Problem.objects.select_related('parts').filter(status__gt=10)

    # solutions = {}
    # if request.user.is_authenticated():
    #     for sol in Solution.objects.filter(user=request.user).order_by('id'):
    #         solutions[sol.part] = sol

    return render_to_response("problems.html", RequestContext(request, {
        'problem_list': problems
    }))

def problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)

    solutions = {}
    if request.user.is_authenticated():
        # give the last solution for each part of the problem
        for sol in Solution.objects.filter(user=request.user, part__problem=problem).select_related('submission').order_by('id'):
            solutions[sol.part_id] = sol

    return render_to_response("problem.html", RequestContext(request, {
        'problem': problem,
        'part_ids': problem.parts.values_list('id', flat=True),
        'solutions': solutions
    }))

import datetime

# @staff_member_required
# def solutions(request, object_id):
#     problem = get_object_or_404(Problem, id=object_id)

#     solutions = {}
#     all_solutions = Solution.objects.filter(part__problem=problem).select_related('submission').order_by('-id')
#     for sol in all_solutions:
#         user_solutions = solutions.get(sol.user_id, {})
#         user_solutions[sol.part_id] = sol
#         solutions[sol.user_id] = user_solutions

#     return render_to_response("solutions.html", RequestContext(request, {
#         'problem': problem,
#         'solutions': solutions,
#         'parts': list(problem.parts.all())
#     }))

@staff_member_required
def solutions(request, object_id):
    from django.db.models import Max
    problem = get_object_or_404(Problem, id=object_id)

    parts = list(problem.parts.select_related('solutions').all())
    part0 = parts[0]

    part0sols = Solution.objects.filter(part__id=part0.id)
    users = part0sols.values('user').annotate(latest_submission=Max('submission__id')).values_list('latest_submission', flat=True)
    sols = Solution.objects.filter(submission__id__in=users).select_related('submission')
    solutions = {}

    # all_solutions = Solution.objects.filter(part__problem=problem).select_related('submission').order_by('-id')
    for s in sols:
        user_solutions = solutions.get(s.user_id, {})
        user_solutions[s.part_id] = (s.correct, s.solution())
        solutions[s.user_id] = user_solutions

    print solutions

    # for sol in all_solutions:

    return render_to_response("solutions.html", RequestContext(request, {
        'problem': problem,
        'solutions': solutions,
        'parts': list(problem.parts.all())
    }))

@login_required
def download_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)
    username = request.user.username

    solutions = {}
    if request.user.is_authenticated():
        for sol in Solution.objects.filter(user=request.user, part__problem=problem).order_by('id'):
            solutions[sol.part] = sol

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/problem.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': username,
        'signature': sign(username + str(problem.id)),
        'solutions': solutions
    })
    response.write(t.render(c))
    return response

def download_anonymous_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/problem.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': '',
        'signature': sign(str(problem.id)),
        'solutions': {}
    })
    response.write(t.render(c))
    return response

@staff_member_required
def edit_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)
    username = request.user.username

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/edit.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': username,
        'signature': sign(username + str(problem.id))
    })
    response.write(t.render(c))
    return response

@csrf_exempt
def upload_solution(request, object_id):
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    problem = get_object_or_404(Problem, id=object_id)
    username = request.POST['username']
    signature = request.POST['signature']

    if not verify(username + str(problem.id), signature):
        return HttpResponseForbidden()

    response = HttpResponse(mimetype='text/plain')
    if username:
        user = get_object_or_404(User, username=username)
        check_problem(problem, user)
        submission = Submission(user=user, source=request.POST['source'],
                            download_ip=request.POST['download_ip'],
                            upload_ip=request.META['REMOTE_ADDR'])
        submission.save()

        for part in problem.parts.all():
            label = request.POST.get('{0}_label'.format(part.id))
            if label:
                start = request.POST['{0}_start'.format(part.id)]
                end = request.POST['{0}_end'.format(part.id)]
                secret = request.POST.get('{0}_secret'.format(part.id))
                correct = secret == part.secret
                if secret and not correct:
                    response.write('Rešitev naloge {0}) je zavrnjena.'.format(label))
                    response.write('Obvestite asistenta.\n')
                s = Solution(user=user, part=part, submission=submission,
                             start=start, end=end, correct=correct, label=label)
                s.save()
        response.write('Vse rešitve so shranjene.\n')
        if problem.status == '20':
            response.write('Rešujete izpit, zato bodo vse rešitve pregledane tudi ročno.')
    else:
        response.write('Naloge rešujete kot anonimni uporabnik!\n')
        response.write('Rešitve niso bile shranjene.')

    return response

@csrf_exempt
def upload_problem(request, object_id):
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    problem = get_object_or_404(Problem, id=object_id)
    username = request.POST['username']
    signature = request.POST['signature']

    if not verify(username + str(problem.id), signature):
        return HttpResponseForbidden()

    response = HttpResponse(mimetype='text/plain')
    user = get_object_or_404(User, username=username)
    response.write('Naloge so shranjene.\n')
    ids = request.POST['problem_ids'].split(",")

    new_parts = []
    for id in map(int, ids):
        part = get_object_or_404(Part, id=id) if id > 0 else Part(problem=problem)
        part.description = request.POST['{0}_description'.format(id)]
        part.trial = request.POST['{0}_trial'.format(id)]
        part.solution = request.POST['{0}_solution'.format(id)]
        part.secret = request.POST['{0}_secret'.format(id)]
        part.save()
        new_parts.append(part)
    for p in problem.parts.all():
        if p not in new_parts:
            p.delete()
    problem.set_part_order([part.id for part in new_parts])
    problem.save()

    return response

<commit_msg>User names show at solutions page.<commit_after># -*- coding: utf-8 -*-
from hashlib import md5

from django.contrib.admin.views.decorators import staff_member_required
from django.contrib.auth.decorators import login_required
from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseForbidden, Http404
from django.shortcuts import get_object_or_404, render_to_response
from django.template import loader, RequestContext
from django.template.defaultfilters import slugify
from django.views.decorators.csrf import csrf_exempt
from django.contrib.auth.models import User

from tomo.problem.models import Problem, Submission, Solution, Part
from tomo.settings import SECRET_KEY

def sign(text):
    return md5(text + SECRET_KEY).hexdigest()

def verify(text, signature):
    return signature == sign(text)

def check_problem(problem, user):
    if problem.status == '10' and not user.is_staff:
        raise Http404

def problem_list(request):
    if request.user.is_staff:
        problems = Problem.objects.select_related('parts').all()
    else:
        problems = Problem.objects.select_related('parts').filter(status__gt=10)

    # solutions = {}
    # if request.user.is_authenticated():
    #     for sol in Solution.objects.filter(user=request.user).order_by('id'):
    #         solutions[sol.part] = sol

    return render_to_response("problems.html", RequestContext(request, {
        'problem_list': problems
    }))

def problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)

    solutions = {}
    if request.user.is_authenticated():
        # give the last solution for each part of the problem
        for sol in Solution.objects.filter(user=request.user, part__problem=problem).select_related('submission').order_by('id'):
            solutions[sol.part_id] = sol

    return render_to_response("problem.html", RequestContext(request, {
        'problem': problem,
        'part_ids': problem.parts.values_list('id', flat=True),
        'solutions': solutions
    }))

import datetime

# @staff_member_required
# def solutions(request, object_id):
#     problem = get_object_or_404(Problem, id=object_id)

#     solutions = {}
#     all_solutions = Solution.objects.filter(part__problem=problem).select_related('submission').order_by('-id')
#     for sol in all_solutions:
#         user_solutions = solutions.get(sol.user_id, {})
#         user_solutions[sol.part_id] = sol
#         solutions[sol.user_id] = user_solutions

#     return render_to_response("solutions.html", RequestContext(request, {
#         'problem': problem,
#         'solutions': solutions,
#         'parts': list(problem.parts.all())
#     }))

@staff_member_required
def solutions(request, object_id):
    from django.db.models import Max
    problem = get_object_or_404(Problem, id=object_id)

    parts = list(problem.parts.select_related('solutions').all())
    part0 = parts[0]

    part0sols = Solution.objects.filter(part__id=part0.id)
    users = part0sols.values('user').annotate(latest_submission=Max('submission__id')).values_list('latest_submission', flat=True)
    sols = Solution.objects.filter(submission__id__in=users).select_related('submission', 'user')
    solutions = {}

    # all_solutions = Solution.objects.filter(part__problem=problem).select_related('submission').order_by('-id')
    for s in sols:
        user_solutions = solutions.get(s.user, {})
        user_solutions[s.part_id] = (s.correct, s.solution())
        solutions[s.user] = user_solutions

    print solutions

    # for sol in all_solutions:

    return render_to_response("solutions.html", RequestContext(request, {
        'problem': problem,
        'solutions': solutions,
        'parts': list(problem.parts.all())
    }))

@login_required
def download_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)
    username = request.user.username

    solutions = {}
    if request.user.is_authenticated():
        for sol in Solution.objects.filter(user=request.user, part__problem=problem).order_by('id'):
            solutions[sol.part] = sol

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/problem.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': username,
        'signature': sign(username + str(problem.id)),
        'solutions': solutions
    })
    response.write(t.render(c))
    return response

def download_anonymous_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/problem.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': '',
        'signature': sign(str(problem.id)),
        'solutions': {}
    })
    response.write(t.render(c))
    return response

@staff_member_required
def edit_problem(request, object_id):
    problem = get_object_or_404(Problem, id=object_id)
    check_problem(problem, request.user)
    slug = slugify(problem.name)
    username = request.user.username

    response = HttpResponse(mimetype='text/plain')
    response['Content-Disposition'] = 'attachment; filename={0}.py'.format(slug)
    t = loader.get_template('python/edit.py')
    c = RequestContext(request, {
        'problem': problem,
        'username': username,
        'signature': sign(username + str(problem.id))
    })
    response.write(t.render(c))
    return response

@csrf_exempt
def upload_solution(request, object_id):
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    problem = get_object_or_404(Problem, id=object_id)
    username = request.POST['username']
    signature = request.POST['signature']

    if not verify(username + str(problem.id), signature):
        return HttpResponseForbidden()

    response = HttpResponse(mimetype='text/plain')
    if username:
        user = get_object_or_404(User, username=username)
        check_problem(problem, user)
        submission = Submission(user=user, source=request.POST['source'],
                            download_ip=request.POST['download_ip'],
                            upload_ip=request.META['REMOTE_ADDR'])
        submission.save()

        for part in problem.parts.all():
            label = request.POST.get('{0}_label'.format(part.id))
            if label:
                start = request.POST['{0}_start'.format(part.id)]
                end = request.POST['{0}_end'.format(part.id)]
                secret = request.POST.get('{0}_secret'.format(part.id))
                correct = secret == part.secret
                if secret and not correct:
                    response.write('Rešitev naloge {0}) je zavrnjena.'.format(label))
                    response.write('Obvestite asistenta.\n')
                s = Solution(user=user, part=part, submission=submission,
                             start=start, end=end, correct=correct, label=label)
                s.save()
        response.write('Vse rešitve so shranjene.\n')
        if problem.status == '20':
            response.write('Rešujete izpit, zato bodo vse rešitve pregledane tudi ročno.')
    else:
        response.write('Naloge rešujete kot anonimni uporabnik!\n')
        response.write('Rešitve niso bile shranjene.')

    return response

@csrf_exempt
def upload_problem(request, object_id):
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])

    problem = get_object_or_404(Problem, id=object_id)
    username = request.POST['username']
    signature = request.POST['signature']

    if not verify(username + str(problem.id), signature):
        return HttpResponseForbidden()

    response = HttpResponse(mimetype='text/plain')
    user = get_object_or_404(User, username=username)
    response.write('Naloge so shranjene.\n')
    ids = request.POST['problem_ids'].split(",")

    new_parts = []
    for id in map(int, ids):
        part = get_object_or_404(Part, id=id) if id > 0 else Part(problem=problem)
        part.description = request.POST['{0}_description'.format(id)]
        part.trial = request.POST['{0}_trial'.format(id)]
        part.solution = request.POST['{0}_solution'.format(id)]
        part.secret = request.POST['{0}_secret'.format(id)]
        part.save()
        new_parts.append(part)
    for p in problem.parts.all():
        if p not in new_parts:
            p.delete()
    problem.set_part_order([part.id for part in new_parts])
    problem.save()

    return response

<|endoftext|>


<commit_before># Copyright 2012 Jeff Trawick, http://emptyhammock.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys

LVL_SHOW_PROCESSES = 0
LVL_SHOW_THREADS = 1
LVL_SHOW_FRAMES = 2
LVL_SHOW_ARGS = 3
LVL_SHOW_VARS = 4

class thread:

    def __init__(self, tid):
        self.tid = tid
        self.frames = []
        self.name = None
        self.state = None
        self.exited = False

    def __str__(self):
        s = ''
        s += '[%s]' % self.tid
        s += ' '
        if self.name:
            s += self.name
            s += ' '
        if self.state:
            s += '(%s) ' % self.state
        s += '\n  '
        for f in self.frames:
            s += f.__str__()
            s += ', '
        return s

    def describe(self, level = 0):
        if level < LVL_SHOW_FRAMES:
            return self.__str__()
        s = ''
        s += '[%s]' % self.tid
        s += ' '
        if self.name:
            s += self.name
            s += ' '
        if self.state:
            s += '(%s) ' % self.state
        s += '\n'
        for f in self.frames:
            s += ' ' + f.describe(level)
        return s

    def set_exited(self, flag = True):
        self.exited = flag

    def add_frame(self, frame):
        self.frames.append(frame)

    def set_name(self, name):
        self.name = name

    def set_state(self, state):
        self.state = state

    def same_backtrace(self, thr2):
	if len(self.frames) != len(thr2.frames):
	    return False
        for i in range(len(self.frames)):
	    if self.frames[i].fn != thr2.frames[i].fn:
		return False
	return True

class threadgroup:
    """ group of threads with same characteristics, such as active frames """

    def __init__(self, thr):
	self.count = 0
	self.threads = [thr]

    def __str__(self):
	return self.threads[0].__str__()

    def add_thread(self, thr):
	self.threads.append(thr)

class process_group:

    def __init__(self):
        self.processes = []

    def add_process(self, proc):
        self.processes.append(proc)

    def __str__(self):
        s = ''
        for p in self.processes:
            s += p.__str__()
            s += '\n'
        return s

    def describe(self, level = 0):
        if level >= 1:
            s = ''
            for p in self.processes:
                s += p.describe(level)
                s += '\n'
            return s
        else:
            return self.__str__()

class process:

    def __init__(self, pid = None):
        self.pid = pid
        self.threads = []
	self.threadgroups = []

    def __str__(self):
        s = ''
        for tg in self.threadgroups:
	    s += '%d * ' % len(tg.threads)
            s += tg.__str__()
            s += '\n'
        return s

    def describe(self, level = 0):
        if level >= 1:
            s = ''
            for t in self.threads:
                s += t.describe(level)
                s += '\n'
        else:
            s = self.__str__()
        return s

    def add_thread(self, thr):
        self.threads.append(thr)

    def group(self):
	for t in self.threads:
            if t.exited:
                continue
	    found = False
	    for g in self.threadgroups:
		if t.same_backtrace(g.threads[0]):
		    g.add_thread(t)
		    found = True
		    break
	    if not found:
		self.threadgroups.append(threadgroup(t))

    def get_pid(self):
        return self.pid

class frame:

    def __init__(self, id, fn, args = None):
        self.id = id
        self.fn = fn
        self.args = args;
        self.vars = {}

        if '@@GLIBC' in fn:
            self.fn = fn[:fn.index('@@GLIBC')]

    def __str__(self):
        return self.fn

    def describe(self, level = 0):
        if level < LVL_SHOW_ARGS:
            return self.__str__()
        s = '#%s %s' % (self.id, self.fn)
        if level >= LVL_SHOW_ARGS and self.args:
            s += self.args
        s += '\n'
        if level >= LVL_SHOW_VARS and len(self.vars):
            for k in self.vars.keys():
                s += '  %s=%s\n' % (k, self.vars[k])
        return s

    def add_var(self, var, val):
        self.vars[var] = val

if __name__ == "__main__":
    print >> sys.stderr, "Don't run this directly."
<commit_msg>add raw description methods suitable for conversion to JSON<commit_after># Copyright 2012 Jeff Trawick, http://emptyhammock.com/
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import sys

LVL_SHOW_PROCESSES = 0
LVL_SHOW_THREADS = 1
LVL_SHOW_FRAMES = 2
LVL_SHOW_ARGS = 3
LVL_SHOW_VARS = 4

class thread:

    def __init__(self, tid):
        self.tid = tid
        self.frames = []
        self.name = None
        self.state = None
        self.exited = False

    def __str__(self):
        s = ''
        s += '[%s]' % self.tid
        s += ' '
        if self.name:
            s += self.name
            s += ' '
        if self.state:
            s += '(%s) ' % self.state
        s += '\n  '
        for f in self.frames:
            s += f.__str__()
            s += ', '
        return s

    def describe(self, level = 0):
        if level < LVL_SHOW_FRAMES:
            return self.__str__()
        s = ''
        s += '[%s]' % self.tid
        s += ' '
        if self.name:
            s += self.name
            s += ' '
        if self.state:
            s += '(%s) ' % self.state
        s += '\n'
        for f in self.frames:
            s += ' ' + f.describe(level)
        return s

    def description(self):
        return {'tid': self.tid,
                'name': self.name,
                'state': self.state}

    def set_exited(self, flag = True):
        self.exited = flag

    def add_frame(self, frame):
        self.frames.append(frame)

    def set_name(self, name):
        self.name = name

    def set_state(self, state):
        self.state = state

    def same_backtrace(self, thr2):
	if len(self.frames) != len(thr2.frames):
	    return False
        for i in range(len(self.frames)):
	    if self.frames[i].fn != thr2.frames[i].fn:
		return False
	return True

class threadgroup:
    """ group of threads with same characteristics, such as active frames """

    def __init__(self, thr):
	self.count = 0
	self.threads = [thr]

    def __str__(self):
	return self.threads[0].__str__()

    def add_thread(self, thr):
	self.threads.append(thr)

    def description(self):
        tids = []
        for t in self.threads:
            tids.append(t.tid)
        return {'thread_ids': tids}

class process_group:

    def __init__(self):
        self.processes = []

    def add_process(self, proc):
        self.processes.append(proc)

    def __str__(self):
        s = ''
        for p in self.processes:
            s += p.__str__()
            s += '\n'
        return s

    def describe(self, level = 0):
        if level >= 1:
            s = ''
            for p in self.processes:
                s += p.describe(level)
                s += '\n'
            return s
        else:
            return self.__str__()

    def description(self):
        procs = []
        for p in self.processes:
            procs.append(p.description())
        return {'processgroupname': 'no-name',
                'processes': procs}

class process:

    def __init__(self, pid = None):
        self.pid = pid
        self.threads = []
	self.threadgroups = []

    def __str__(self):
        s = ''
        for tg in self.threadgroups:
	    s += '%d * ' % len(tg.threads)
            s += tg.__str__()
            s += '\n'
        return s

    def describe(self, level = 0):
        if level >= 1:
            s = ''
            for t in self.threads:
                s += t.describe(level)
                s += '\n'
        else:
            s = self.__str__()
        return s

    def description(self):
        threads = []
        for t in self.threads:
            threads.append(t.description())
        threadgroups = []
        for t in self.threadgroups:
            threadgroups.append(t.description())
        return {'processname': 'no-name',
                'threads': threads,
                'threadgroups': threadgroups}

    def add_thread(self, thr):
        self.threads.append(thr)

    def group(self):
	for t in self.threads:
            if t.exited:
                continue
	    found = False
	    for g in self.threadgroups:
		if t.same_backtrace(g.threads[0]):
		    g.add_thread(t)
		    found = True
		    break
	    if not found:
		self.threadgroups.append(threadgroup(t))

    def get_pid(self):
        return self.pid

class frame:

    def __init__(self, id, fn, args = None):
        self.id = id
        self.fn = fn
        self.args = args;
        self.vars = {}

        if '@@GLIBC' in fn:
            self.fn = fn[:fn.index('@@GLIBC')]

    def __str__(self):
        return self.fn

    def describe(self, level = 0):
        if level < LVL_SHOW_ARGS:
            return self.__str__()
        s = '#%s %s' % (self.id, self.fn)
        if level >= LVL_SHOW_ARGS and self.args:
            s += self.args
        s += '\n'
        if level >= LVL_SHOW_VARS and len(self.vars):
            for k in self.vars.keys():
                s += '  %s=%s\n' % (k, self.vars[k])
        return s

    def add_var(self, var, val):
        self.vars[var] = val

if __name__ == "__main__":
    print >> sys.stderr, "Don't run this directly."
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Project Index -- prepares an index of WikiProjects and their in-scope pages for use by scripts
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import sys
import pymysql
import re

class WikiProjectTools:
    def metaquery(connection, sqlquery, values):
        """Carries out MySQL queries"""
        cur = connection.cursor()
        cur.execute(sqlquery, values)
        data = []
        for row in cur:
            data.append(row)
        return data

    def wikiquery(sqlquery, values):
        """Constructs a MySQL query to the Tool Labs database replica."""
        conn = pymysql.connect(host='enwiki.labsdb', port=3306, db='enwiki_p', read_default_file='~/.my.cnf', charset='utf8')
        data = WikiProjectTools.metaquery(conn, sqlquery, values)
        return data

    def indexquery(sqlquery, values):
        """Constructs a MySQL query to the WPX database."""
        conn = pymysql.connect(host='tools-db', port=3306, db='s52475__wpx', read_default_file='~/.my.cnf', charset='utf8')
        data = WikiProjectTools.metaquery(conn, sqlquery, values)
        conn.commit()
        return data

    def masterlist():
        """
        Prepares the master list of WikiProject quality assessment categories.
        Returns a dictionary of lists, with the key being the page_title of the WikiProject (or task force)
        and the value being a list of categories.
        """

        query = WikiProjectTools.wikiquery('select page_title from page where page_namespace = 14 and (page_title like "%-Class_%_articles" or page_title like "Unassessed_%_articles" or page_title like "WikiProject_%_articles") and page_title not like "%-importance_%" and page_title not like "Wikipedia_%" and page_title not like "Template-%" and page_title not like "Redirect-%" and page_title not like "Project-%" and page_title not like "Portal-%" and page_title not like "File-%" and page_title not like "FM-%" and page_title not like "Category-%" and page_title not like "Cat-%" and page_title not like "Book-%" and page_title not like "NA-%" and page_title not like "%_Operation_Majestic_Titan_%" and page_title not like "%_Version_%" and page_title not like "All_Wikipedia_%" and page_title not like "%_Wikipedia-Books_%" and page_title not like "Assessed-%" and page_title not like "%-Priority_%" and page_title not like "Unassessed_field_%" and page_title not like "Unassessed_importance_%" and page_title not like "Unassessed-Class_articles" and page_title not like "%_Article_quality_research_articles" and page_title not like "WikiProject_lists_of_encyclopedic_articles";', None)
        categories = []
        for row in query:
            categories.append(row[0].decode('utf-8'))

        # Record in a dictionary of lists? wikiprojects = {'Military history': ['Category:A', 'Category:B']}
        buckets = {}

        for category in categories:
            projectname = category
            projectname = projectname.replace('WikiProject_', '') # Some categories include "WikiProject" in the category name.
            projectname = projectname.replace('-related', '') # e.g. "Museum-related" -> "Museum"
            projectname = projectname.replace('_quality', '') # e.g. "Unassessed_quality" -> "Unassessed"
            projectname = projectname.replace('_subproject_selected_articles', '')
            projectname = projectname.replace('_automatically_assessed', '')
            projectname = re.sub(r'_task_?forces?(_by)?', '', projectname)
            projectname = re.sub(r'_work_?group', '', projectname)
            projectname = re.sub(r'_articles$', '', projectname)
            projectname = re.sub(r'_newsletter$', '', projectname)
            projectname = re.sub(r'^((.*)-Class|Unassessed)_', '', projectname)
            projectname = projectname[0].upper() + projectname[1:] # Capitalize the first letter
            try:
                buckets[projectname].append(category)
            except KeyError:
                buckets[projectname] = []
                buckets[projectname].append(category)

        # For each key in buckets, try to match it to a real WikiProject or task force name
        # Checks against the redirect table so that it can follow redirects

        pagetitles = {}
        namespaces = {2: 'User:', 3: 'User_talk:', 4: 'Wikipedia:', 5: 'Wikipedia_talk:', 100: 'Portal:', 101: 'Portal_talk:'}
        # Heavens help me if WikiProjects end up in namespaces other than those.

        for key in buckets.keys():
            project_area = key
            query = WikiProjectTools.wikiquery('select page.page_title,redirect.rd_namespace,redirect.rd_title from page left join redirect on redirect.rd_from = page.page_id where page_title = %s and page_namespace = 4;', ('WikiProject_'+key,))
            if len(query) == 0:
                query = WikiProjectTools.wikiquery('select page.page_title,redirect.rd_namespace,redirect.rd_title from page left join redirect on redirect.rd_from = page.page_id where page_title = %s and page_namespace = 4;', ('WikiProject_'+key+'s',)) # Checks for plural
                if len(query) == 0:
                    print('Warning: No project page found for key: ' + key)
                    continue

            page_title = query[0][0]
            rd_namespace = query[0][1]
            rd_title = query[0][2]

            if rd_title is not None:
                pagetitles[key] = namespaces[rd_namespace] + rd_title.decode('utf-8')
            elif rd_title is None and page_title is not None:
                pagetitles[key] = namespaces[4] + page_title.decode('utf-8')

        # At this point, each key of buckets should be tied to an actual page name
        output = {}

        for key in buckets.keys():
            for category in buckets[key]:
                try:
                    output[pagetitles[key]].append(category)
                except KeyError:
                    output[pagetitles[key]] = []
                    output[pagetitles[key]].append(category)

        return output

    def projectscope(project,categories):
        """
        Returns a list of articles in the scope of a WikiProject
        Requires the string 'project' and the list 'categories'
        """

        query_builder = 'select distinct page.page_title,page.page_namespace from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace in (1, 119) and cl_to in ('
        mastertuple = ()
        for category in categories:
            query_builder += '%s, '
            mastertuple += (category,)
        query_builder = query_builder[:-2] # Truncate extraneous "or"
        query_builder += ');' # Wrap up query

        query = WikiProjectTools.wikiquery(query_builder, mastertuple)
        namespaces = {1: "Talk:", 119: "Draft_talk:"}
        output = []

        for row in query:
            page_title = row[0].decode('utf-8')
            page_namespace = namespaces[row[1]]
            output.append(page_namespace + page_title)

        return output

    def main():
        # Prepare list of WikiProjects and their categories
        print('Preparing master list of WikiProjects...')
        wikiprojects = WikiProjectTools.masterlist()

        # Get list of pages for each WikiProject
        print('Getting list of pages for each WikiProject...')
        pages = {}
        for wikiproject in wikiprojects.keys():
            pages[wikiproject] = WikiProjectTools.projectscope(wikiproject, wikiprojects[wikiproject])

        dbinput = []
        for wikiproject in pages.keys():
            for page in pages[wikiproject]:
                dbinput.append((page, wikiproject))

        # Saves it to the database
        print('Saving to the database...')
        WikiProjectTools.indexquery('drop table if exists projectindex', None) # We are going to re-build the table
        WikiProjectTools.indexquery('create table projectindex (pi_id int(11) NOT NULL auto_increment, pi_page VARCHAR(255) character set utf8 collate utf8_unicode_ci, pi_project VARCHAR(255) character set utf8 collate utf8_unicode_ci, primary key (pi_id)) engine=innodb character set=utf8;', None)

        packages = []
        for i in range(0, len(dbinput), 10000):
            packages.append(dbinput[i:i+10000])

        counter = 0
        for package in packages:
            query_builder = 'insert into projectindex (pi_page, pi_project) values ' # seeding really long query
            mastertuple = ()
            for item in package:
                query_builder += '(%s, %s), '
                mastertuple += item
            query_builder = query_builder[:-2] # truncating the terminal comma and space
            query_builder += ';'
            counter += 1
            print('Executing batch query no. ' + str(counter))
            WikiProjectTools.indexquery(query_builder, mastertuple)

if __name__ == "__main__":
    WikiProjectTools.main()<commit_msg>Create draft table and rename after<commit_after># -*- coding: utf-8 -*-
"""
Project Index -- prepares an index of WikiProjects and their in-scope pages for use by scripts
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import sys
import pymysql
import re

class WikiProjectTools:
    def metaquery(connection, sqlquery, values):
        """Carries out MySQL queries"""
        cur = connection.cursor()
        cur.execute(sqlquery, values)
        data = []
        for row in cur:
            data.append(row)
        return data

    def wikiquery(sqlquery, values):
        """Constructs a MySQL query to the Tool Labs database replica."""
        conn = pymysql.connect(host='enwiki.labsdb', port=3306, db='enwiki_p', read_default_file='~/.my.cnf', charset='utf8')
        data = WikiProjectTools.metaquery(conn, sqlquery, values)
        return data

    def indexquery(sqlquery, values):
        """Constructs a MySQL query to the WPX database."""
        conn = pymysql.connect(host='tools-db', port=3306, db='s52475__wpx', read_default_file='~/.my.cnf', charset='utf8')
        data = WikiProjectTools.metaquery(conn, sqlquery, values)
        conn.commit()
        return data

    def masterlist():
        """
        Prepares the master list of WikiProject quality assessment categories.
        Returns a dictionary of lists, with the key being the page_title of the WikiProject (or task force)
        and the value being a list of categories.
        """

        query = WikiProjectTools.wikiquery('select page_title from page where page_namespace = 14 and (page_title like "%-Class_%_articles" or page_title like "Unassessed_%_articles" or page_title like "WikiProject_%_articles") and page_title not like "%-importance_%" and page_title not like "Wikipedia_%" and page_title not like "Template-%" and page_title not like "Redirect-%" and page_title not like "Project-%" and page_title not like "Portal-%" and page_title not like "File-%" and page_title not like "FM-%" and page_title not like "Category-%" and page_title not like "Cat-%" and page_title not like "Book-%" and page_title not like "NA-%" and page_title not like "%_Operation_Majestic_Titan_%" and page_title not like "%_Version_%" and page_title not like "All_Wikipedia_%" and page_title not like "%_Wikipedia-Books_%" and page_title not like "Assessed-%" and page_title not like "%-Priority_%" and page_title not like "Unassessed_field_%" and page_title not like "Unassessed_importance_%" and page_title not like "Unassessed-Class_articles" and page_title not like "%_Article_quality_research_articles" and page_title not like "WikiProject_lists_of_encyclopedic_articles";', None)
        categories = []
        for row in query:
            categories.append(row[0].decode('utf-8'))

        # Record in a dictionary of lists? wikiprojects = {'Military history': ['Category:A', 'Category:B']}
        buckets = {}

        for category in categories:
            projectname = category
            projectname = projectname.replace('WikiProject_', '') # Some categories include "WikiProject" in the category name.
            projectname = projectname.replace('-related', '') # e.g. "Museum-related" -> "Museum"
            projectname = projectname.replace('_quality', '') # e.g. "Unassessed_quality" -> "Unassessed"
            projectname = projectname.replace('_subproject_selected_articles', '')
            projectname = projectname.replace('_automatically_assessed', '')
            projectname = re.sub(r'_task_?forces?(_by)?', '', projectname)
            projectname = re.sub(r'_work_?group', '', projectname)
            projectname = re.sub(r'_articles$', '', projectname)
            projectname = re.sub(r'_newsletter$', '', projectname)
            projectname = re.sub(r'^((.*)-Class|Unassessed)_', '', projectname)
            projectname = projectname[0].upper() + projectname[1:] # Capitalize the first letter
            try:
                buckets[projectname].append(category)
            except KeyError:
                buckets[projectname] = []
                buckets[projectname].append(category)

        # For each key in buckets, try to match it to a real WikiProject or task force name
        # Checks against the redirect table so that it can follow redirects

        pagetitles = {}
        namespaces = {2: 'User:', 3: 'User_talk:', 4: 'Wikipedia:', 5: 'Wikipedia_talk:', 100: 'Portal:', 101: 'Portal_talk:'}
        # Heavens help me if WikiProjects end up in namespaces other than those.

        for key in buckets.keys():
            project_area = key
            query = WikiProjectTools.wikiquery('select page.page_title,redirect.rd_namespace,redirect.rd_title from page left join redirect on redirect.rd_from = page.page_id where page_title = %s and page_namespace = 4;', ('WikiProject_'+key,))
            if len(query) == 0:
                query = WikiProjectTools.wikiquery('select page.page_title,redirect.rd_namespace,redirect.rd_title from page left join redirect on redirect.rd_from = page.page_id where page_title = %s and page_namespace = 4;', ('WikiProject_'+key+'s',)) # Checks for plural
                if len(query) == 0:
                    print('Warning: No project page found for key: ' + key)
                    continue

            page_title = query[0][0]
            rd_namespace = query[0][1]
            rd_title = query[0][2]

            if rd_title is not None:
                pagetitles[key] = namespaces[rd_namespace] + rd_title.decode('utf-8')
            elif rd_title is None and page_title is not None:
                pagetitles[key] = namespaces[4] + page_title.decode('utf-8')

        # At this point, each key of buckets should be tied to an actual page name
        output = {}

        for key in buckets.keys():
            for category in buckets[key]:
                try:
                    output[pagetitles[key]].append(category)
                except KeyError:
                    output[pagetitles[key]] = []
                    output[pagetitles[key]].append(category)

        return output

    def projectscope(project,categories):
        """
        Returns a list of articles in the scope of a WikiProject
        Requires the string 'project' and the list 'categories'
        """

        query_builder = 'select distinct page.page_title,page.page_namespace from categorylinks join page on categorylinks.cl_from=page.page_id where page_namespace in (1, 119) and cl_to in ('
        mastertuple = ()
        for category in categories:
            query_builder += '%s, '
            mastertuple += (category,)
        query_builder = query_builder[:-2] # Truncate extraneous "or"
        query_builder += ');' # Wrap up query

        query = WikiProjectTools.wikiquery(query_builder, mastertuple)
        namespaces = {1: "Talk:", 119: "Draft_talk:"}
        output = []

        for row in query:
            page_title = row[0].decode('utf-8')
            page_namespace = namespaces[row[1]]
            output.append(page_namespace + page_title)

        return output

    def main():
        # Prepare list of WikiProjects and their categories
        print('Preparing master list of WikiProjects...')
        wikiprojects = WikiProjectTools.masterlist()

        # Get list of pages for each WikiProject
        print('Getting list of pages for each WikiProject...')
        pages = {}
        for wikiproject in wikiprojects.keys():
            pages[wikiproject] = WikiProjectTools.projectscope(wikiproject, wikiprojects[wikiproject])

        dbinput = []
        for wikiproject in pages.keys():
            for page in pages[wikiproject]:
                dbinput.append((page, wikiproject))

        # Saves it to the database
        print('Saving to the database...')
        WikiProjectTools.indexquery('create table projectindex_draft (pi_id int(11) NOT NULL auto_increment, pi_page VARCHAR(255) character set utf8 collate utf8_unicode_ci, pi_project VARCHAR(255) character set utf8 collate utf8_unicode_ci, primary key (pi_id)) engine=innodb character set=utf8;', None)

        packages = []
        for i in range(0, len(dbinput), 10000):
            packages.append(dbinput[i:i+10000])

        counter = 0
        for package in packages:
            query_builder = 'insert into projectindex_draft (pi_page, pi_project) values ' # seeding really long query
            mastertuple = ()
            for item in package:
                query_builder += '(%s, %s), '
                mastertuple += item
            query_builder = query_builder[:-2] # truncating the terminal comma and space
            query_builder += ';'
            counter += 1
            print('Executing batch query no. ' + str(counter))
            WikiProjectTools.indexquery(query_builder, mastertuple)

        WikiProjectTools.indexquery('drop table if exists projectindex', None)
        WikiProjectTools.indexquery('rename table projectindex_draft to projectindex', None) # Moving draft table over to live table

if __name__ == "__main__":
    WikiProjectTools.main()<|endoftext|>


<commit_before>import numpy as np
from flow.core.traffic_light_utils import color_vehicles, get_id_within_dist

ID_IDX = 1

class PressureDecentLightGridEnv:

    def __init__(self, params_obj):

        self.benchmark_params = params_obj
        self.look_ahead = self.benchmark_params.look_ahead
        self.num_of_emergency_stops = dict()
        self.delays = dict()
        self.num_of_switch_actions = dict()
        self.current_state = dict()
        self.prev_state = dict()
        self.edge_pressure_dict = dict()
        self.waiting_times = dict()

    def obs_shape_func(self, num_traffic_lights):
        """Define the shape of the observation space for the PressLight benchmark"""

        obs_shape = 14
        return obs_shape

    def get_state(self,
                  kernel,
                  network,
                  _get_relative_node,
                  direction,
                  currently_yellow,
                  step_counter):

        """See parent class.

        Returns edge pressures of an intersection, edge_numbers,
         traffic light state. This is partially observed
         TODO: Explian in detail
        """

        # Traffic light information
        direction = direction.flatten()
        currently_yellow = currently_yellow.flatten()
        # This is a catch-all for when the relative_node method returns a -1
        # (when there is no node in the direction sought). We add a last
        # item to the lists here, which will serve as a default value.
        # TODO(cathywu) are these values reasonable?
        direction = np.append(direction, [0])
        currently_yellow = np.append(currently_yellow, [1])

        obs = {}
        # TODO(cathywu) allow differentiation between rl and non-rl lights

        # collect list of names of inner edges
        internal_edges = []
        for i in kernel.network.rts:
            if kernel.network.rts[i]:
                if kernel.network.rts[i][0][0][1:-1]:
                    internal_edges += [kernel.network.rts[i][0][0][1:]]

        node_to_edges = network.node_mapping

        all_ids_incoming = dict()
        for rl_id in kernel.traffic_light.get_ids():
            all_ids_incoming[rl_id] = []

            # collect observations for each traffic light
            rl_id_num = int(rl_id.split("center")[ID_IDX])
            local_edges = node_to_edges[rl_id_num][1]
            local_edge_numbers = [kernel.network.get_edge_list().index(e)
                                  for e in local_edges]
            local_id_nums = [rl_id_num, _get_relative_node(rl_id, "top"),
                             _get_relative_node(rl_id, "bottom"),
                             _get_relative_node(rl_id, "left"),
                             _get_relative_node(rl_id, "right")]

            # define incoming edges
            incoming = local_edges
            outgoing_edges = []
            edge_pressure = []
            for edge_ in incoming:
                if kernel.network.rts[edge_]:
                    # if edge is an outer(global) incoming edge,
                    # outgoing edge is the next edge in the route
                    index_ = kernel.network.rts[edge_][0][0].index(edge_)
                    outgoing_edges = kernel.network.rts[edge_][0][0][index_ + 1]
                else:
                    for lst in internal_edges:
                        # if edge is an inner edges, outgoing is the next edge in the list
                        if len(lst) > 1 and edge_ in lst:
                            index_ = lst.index(edge_)
                            outgoing_edges = lst[index_ + 1]

                # get vehicle ids in incoming edge
                observed_ids = \
                    get_id_within_dist(edge_, "ahead", kernel, self.benchmark_params)
                all_ids_incoming[rl_id] += observed_ids

                # get ids in outgoing edge
                observed_ids_behind = \
                    get_id_within_dist(outgoing_edges, "behind", kernel, self.benchmark_params)

                # get edge pressures
                edge_pressure += [len(observed_ids) - len(observed_ids_behind)]

                # color incoming and outgoing vehicles
                color_vehicles(observed_ids, self.benchmark_params.CYAN, kernel)
                color_vehicles(observed_ids_behind, self.benchmark_params.RED, kernel)

            # for each incoming edge, store the pressure terms to be used in compute reward
            self.edge_pressure_dict[rl_id] = edge_pressure

            light_states = kernel.traffic_light.get_state(rl_id)

            if light_states == "GrGr":
                light_states_ = [1]
            elif light_states == ["yryr"]:
                light_states_ = [0.6]
            else:
                light_states_ = [0.2]

            observation = np.array(np.concatenate(
                [self.edge_pressure_dict[rl_id],
                 local_edge_numbers,
                 direction[local_id_nums],
                 light_states_
                 ]))
            obs.update({rl_id: observation})

        return obs

    def compute_reward(self, rl_actions, step_counter, action_dict=None,
                       rl_id=None, **kwargs):
        """See class definition."""

        return -sum(self.edge_pressure_dict[rl_id])

class PressureCentLightGridEnv(PressureDecentLightGridEnv):
    """TODO: cite paper"""

    def obs_shape_func(self, num_traffic_lights):
        """Define the shape of the observation space for the PressLight benchmark"""

        obs_shape = super().obs_shape_func(num_traffic_lights)
        return obs_shape * num_traffic_lights

    def get_state(self,
                  kernel,
                  network,
                  _get_relative_node,
                  direction,
                  currently_yellow,
                  step_counter):
        """ see parent class"""

        obs = super().get_state(kernel,
                                network,
                                _get_relative_node,
                                direction,
                                currently_yellow,
                                step_counter)

        final_obs = np.concatenate(list((obs.values())))
        return final_obs
<commit_msg>broke up the get_state function<commit_after>import numpy as np
from flow.core.traffic_light_utils import color_vehicles, get_id_within_dist

ID_IDX = 1

class PressureDecentLightGridEnv:

    def __init__(self, params_obj):

        self.benchmark_params = params_obj
        self.look_ahead = self.benchmark_params.look_ahead
        self.num_of_emergency_stops = dict()
        self.delays = dict()
        self.num_of_switch_actions = dict()
        self.current_state = dict()
        self.prev_state = dict()
        self.edge_pressure_dict = dict()
        self.waiting_times = dict()

    def obs_shape_func(self, num_traffic_lights):
        """Define the shape of the observation space for the PressLight benchmark"""

        obs_shape = 14
        return obs_shape

    def get_state(self,
                  kernel,
                  network,
                  _get_relative_node,
                  direction,
                  currently_yellow,
                  step_counter):

        """See parent class.

        Returns edge pressures of an intersection, edge_numbers,
         traffic light state. This is partially observed
         TODO: Explian in detail
        """

        # Traffic light information
        direction = direction.flatten()
        # This is a catch-all for when the relative_node method returns a -1
        # (when there is no node in the direction sought). We add a last
        # item to the lists here, which will serve as a default value.
        # TODO(cathywu) are these values reasonable?
        direction = np.append(direction, [0])

        obs = {}

        # collect list of names of inner edges
        internal_edges = get_internal_edges(kernel)
        node_to_edges = network.node_mapping

        for rl_id in kernel.traffic_light.get_ids():
            # collect observations for each traffic light

            # collect and set edge names and ids
            incoming_edges, local_edge_numbers, local_id_nums = get_traffic_params(rl_id,
                                                                                node_to_edges,
                                                                                kernel,
                                                                                _get_relative_node)

            # define incoming edges
            edge_pressure_state = []
            for edge_ in incoming_edges:

                # get outgoing edge
                outgoing_edge = get_outgoing_edge(kernel, edge_, internal_edges)

                # collect observed ids within distances and color them
                observed_ids_ahead, observed_ids_behind = get_observed_ids(kernel,
                                                                           edge_,
                                                                           outgoing_edge,
                                                                           self.benchmark_params)

                # get edge pressures of those edges
                edge_pressure_state += [len(observed_ids_ahead) - len(observed_ids_behind)]

            # for each incoming edge, store the pressure terms to be used in compute reward
            self.edge_pressure_dict[rl_id] = edge_pressure_state

            # for each  intersection, collect traffic light state
            light_states_ = get_light_states(kernel, rl_id)

            observation = np.array(np.concatenate(
                [self.edge_pressure_dict[rl_id],
                 local_edge_numbers,
                 direction[local_id_nums],
                 light_states_
                 ]))
            obs.update({rl_id: observation})

        return obs

    def compute_reward(self, rl_actions, step_counter, action_dict=None,
                       rl_id=None, **kwargs):
        """See class definition."""

        return -sum(self.edge_pressure_dict[rl_id])

class PressureCentLightGridEnv(PressureDecentLightGridEnv):
    """TODO: cite paper"""

    def obs_shape_func(self, num_traffic_lights):
        """Define the shape of the observation space for the PressLight benchmark"""

        obs_shape = super().obs_shape_func(num_traffic_lights)
        return obs_shape * num_traffic_lights

    def get_state(self,
                  kernel,
                  network,
                  _get_relative_node,
                  direction,
                  currently_yellow,
                  step_counter):
        """ see parent class"""

        obs = super().get_state(kernel,
                                network,
                                _get_relative_node,
                                direction,
                                currently_yellow,
                                step_counter)

        final_obs = np.concatenate(list((obs.values())))
        return final_obs

def get_internal_edges(kernel):
    """Collect all internal edges in network
    ie. Inner edges are Internal edges
                  (outer)         (outer)       (outer)
                     |              |              |
        (outer) x----|----Inner-----|----Inner-----|----x (outer)
                   Inner          Inner          Inner
        (outer) x----|----Inner-----|----Inner-----|----x (outer)
                     |              |              |
                 (outer)         (outer)       (outer)

    """
    internal_edges = []
    for i in kernel.network.rts:
        if kernel.network.rts[i]:
            if kernel.network.rts[i][0][0][1:-1]:
                internal_edges += [kernel.network.rts[i][0][0][1:]]
    return internal_edges

def get_outgoing_edge(kernel, edge_, internal_edges):
    """Collect the next edge for vehicles given the incoming edge id""
    ie.
                    incoming
                        |
        -> --incoming---|---outgoing--->
                        |
                     outgoing
    """
    if kernel.network.rts[edge_]:
        # if edge is an outer(global) incoming edge,
        # outgoing edge is the next edge in the route
        index_ = kernel.network.rts[edge_][0][0].index(edge_)
        outgoing_edge = kernel.network.rts[edge_][0][0][index_ + 1]
    else:
        for lst in internal_edges:
            # if edge is an inner edges, outgoing is the next edge in the list
            if len(lst) > 1 and edge_ in lst:
                index_ = lst.index(edge_)
                outgoing_edge = lst[index_ + 1]

    return outgoing_edge

def get_light_states(kernel, rl_id):
    """Return current traffic light stats as number:
    ie either GrGrGr, yryryr, ryryry, rGrGrG"""

    light_states = kernel.traffic_light.get_state(rl_id)

    if light_states == "GrGr":
        light_states__ = [1]
    elif light_states == ["yryr"]:
        light_states__ = [0.6]
    else:
        light_states__ = [0.2]

    return light_states__

def get_observed_ids(kernel, edge_, outgoing_edge, benchmark_params):
    """Return lists of observed vehicles within look ahead and behind distances"""

    # get vehicle ids in incoming edge
    observed_ids_ahead = \
        get_id_within_dist(edge_, "ahead", kernel, benchmark_params)

    # get ids in outgoing edge
    observed_ids_behind = \
        get_id_within_dist(outgoing_edge, "behind", kernel, benchmark_params)

    # color incoming and outgoing vehicles
    color_vehicles(observed_ids_ahead, benchmark_params.CYAN, kernel)
    color_vehicles(observed_ids_behind, benchmark_params.RED, kernel)

    return observed_ids_ahead, observed_ids_behind

def get_traffic_params(rl_id, node_to_edges, kernel, _get_relative_node):

    """Collect ids and names of edges"""

    rl_id_num = int(rl_id.split("center")[ID_IDX])
    local_edges = node_to_edges[rl_id_num][1]
    local_edge_numbers = [kernel.network.get_edge_list().index(e)
                          for e in local_edges]
    local_id_nums = [rl_id_num, _get_relative_node(rl_id, "top"),
                     _get_relative_node(rl_id, "bottom"),
                     _get_relative_node(rl_id, "left"),
                     _get_relative_node(rl_id, "right")]

    return local_edges, local_edge_numbers, local_id_nums<|endoftext|>


<commit_before>#
#     Kay Hayen, mailto:kayhayen@gmx.de
#
#     Python test originally created or extracted from other peoples work. The
#     parts from me are in the public domain. It is at least Free Software
#     where it's copied from other people. In these cases, it will normally be
#     indicated.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     Please leave the whole of this copyright notice intact.
#

def starImporterFunction():
    from sys import *

    print "Version", version.split()[0]

starImporterFunction()

def deepExec():
    for_closure = 3

    def execFunction():
        code = "f=2"

        # Can fool it to nest
        exec code in None, None

        print "Locals now", locals()

        # print "Closure was taken", for_closure
        print "Globals still work", starImporterFunction
        print "Added local from code", f

    execFunction()

deepExec()
<commit_msg>Cover an even deeper nesting of closures for the overflow function, still commented out though.<commit_after>#
#     Kay Hayen, mailto:kayhayen@gmx.de
#
#     Python test originally created or extracted from other peoples work. The
#     parts from me are in the public domain. It is at least Free Software
#     where it's copied from other people. In these cases, it will normally be
#     indicated.
#
#     If you submit Kay Hayen patches to this software in either form, you
#     automatically grant him a copyright assignment to the code, or in the
#     alternative a BSD license to the code, should your jurisdiction prevent
#     this. Obviously it won't affect code that comes to him indirectly or
#     code you don't submit to him.
#
#     This is to reserve my ability to re-license the code at any time, e.g.
#     the PSF. With this version of Nuitka, using it for Closed Source will
#     not be allowed.
#
#     Please leave the whole of this copyright notice intact.
#

def starImporterFunction():
    from sys import *

    print "Version", version.split()[0]

starImporterFunction()

def deepExec():
    for_closure = 3

    def deeper():
        for_closure_as_well = 4

        def execFunction():
            code = "f=2"

            # Can fool it to nest
            exec code in None, None

            print "Locals now", locals()

            # print "Closure one level up was taken", for_closure_as_well
            # print "Closure two levels up was taken", for_closure
            print "Globals still work", starImporterFunction
            print "Added local from code", f

        execFunction()

    deeper()

deepExec()
<|endoftext|>