<commit_before>from abandoned.githubapi import get_project_data, AbandonedException, GithubException
from abandoned.models import Project, Tag, Author, Reason, Language
from abandoned.serializers import TagSerializer, ProjectSerializer, ReasonSerializer, AuthorSerializer, LanguageSerializer
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db.models import Count, Sum
from django.http import HttpResponse, HttpResponseBadRequest, Http404
from django.shortcuts import render
from django.views.decorators.csrf import csrf_protect
from json import loads
from rest_framework import viewsets, status
from rest_framework.decorators import list_route, api_view
from rest_framework.response import Response

def votes_generic(model_obj, class_instance):
    votes = model_obj.objects.annotate(votes_total=Sum('projects__upvotes')).order_by('votes_total').reverse()
    page = class_instance.paginate_queryset(votes)
    serializer = class_instance.get_pagination_serializer(page)
    return Response(serializer.data)

def projects_generic(model_obj, class_instance):
    votes = model_obj.objects.annotate(Count('projects')).order_by('projects__count').reverse()
    page = class_instance.paginate_queryset(votes)
    serializer = class_instance.get_pagination_serializer(page)
    return Response(serializer.data)

def pagination_generic(page, sorting, model_obj, sql_lowercase, lowercase_name):
    if sorting == 'projects':
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).\
            order_by('projects_total').reverse()
    elif sorting == 'votes':
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).order_by('votes_total').reverse()
    else:
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).\
            extra(select={lowercase_name: sql_lowercase}).order_by(lowercase_name)
    paginator = Paginator(result_list, 10)
    try:
        result = paginator.page(page)
    except PageNotAnInteger:
        result = paginator.page(1)
    except EmptyPage:
        result = paginator.page(paginator.num_pages)
    return result

class AuthorViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Author.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = AuthorSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Author, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Author, self)

class TagViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Tag.objects.extra(select={'lowercase_text': 'lower(text)'}).order_by('lowercase_text')
    serializer_class = TagSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Tag, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Tag, self)

class ReasonViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Reason.objects.extra(select={'lowercase_reason': 'lower(reason)'}).order_by('lowercase_reason')
    serializer_class = ReasonSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Reason, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Reason, self)

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Language.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = LanguageSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Language, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Language, self)

class ProjectViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Project.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = ProjectSerializer

    @list_route()
    def votes(self, request):
        top = Project.objects.all().order_by('upvotes').reverse()
        page = self.paginate_queryset(top)
        serializer = self.get_pagination_serializer(page)
        return Response(serializer.data)

    @list_route()
    def latest(self, request):
        latest = Project.objects.all().order_by('date_added').reverse()
        page = self.paginate_queryset(latest)
        serializer = self.get_pagination_serializer(page)
        return Response(serializer.data)

@csrf_protect
@api_view(['POST'])
def handle_submit(request):
    request_data = loads(request.body.decode('utf-8'))
    try:
        project_data = get_project_data(request_data['repo_url'])
    except AbandonedException as e:
        return HttpResponseBadRequest(str(e))
    except GithubException as e:
        return HttpResponse(str(e), status=503)
    repo_name, author_name, language = project_data
    if language is None:
        language = 'Language not specified'
    author_url = 'https://github.com/' + author_name
    repo_url = 'https://github.com/' + author_name + '/' + repo_name

    if not Project.objects.filter(link=repo_url).exists():

        curr_reason = Reason.objects.get(id=request_data['reason_id'])
        curr_author, author_created = Author.objects.get_or_create(name=author_name, link=author_url)
        curr_language, language_created = Language.objects.get_or_create(name=language)

        tags_list = []
        for tag_text in request_data['tags']:
            curr_tag, tag_created = Tag.objects.get_or_create(text=tag_text)
            tags_list.append(curr_tag)

        curr_project = Project.objects.create(name=repo_name, link=repo_url, author=curr_author,
                                              description=request_data['description'],
                                              reason=curr_reason, language=curr_language)
        for tag in tags_list:
            curr_project.tags.add(tag)
        curr_project.save()
        serializer = ProjectSerializer(curr_project)
        return Response(serializer.data, status=status.HTTP_201_CREATED)
    else:
        return HttpResponseBadRequest("This project is already in the database")

def projects_view(request, sorting='latest'):
    page = request.GET.get('page')
    if sorting == 'latest':
        projects_list = Project.objects.order_by('date_added').reverse()
    elif sorting == 'votes':
        projects_list = Project.objects.order_by('upvotes').reverse()
    else:
        projects_list = Project.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    paginator = Paginator(projects_list, 10)
    try:
        projects = paginator.page(page)
    except PageNotAnInteger:
        projects = paginator.page(1)
    except EmptyPage:
        projects = paginator.page(paginator.num_pages)
    return render(request, 'projects.html', {'projects_list': projects})

def single_project_view(request, project_id):
    try:
        project = Project.objects.get(id=project_id)
    except Project.DoesNotExist:
        raise Http404
    return render(request, 'single_project.html', {'project': project})

def single_author_view(request, author_id):
    try:
        author = Author.objects.annotate(votes_total=Sum('projects__upvotes'),
                                         projects_total=Count('projects')).get(id=author_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_author.html', {'author': author})

def single_language_view(request, language_id):
    try:
        language = Language.objects.annotate(votes_total=Sum('projects__upvotes'),
                                             projects_total=Count('projects')).get(id=language_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_language.html', {'language': language})

def single_tag_view(request, tag_id):
    try:
        tag = Tag.objects.annotate(votes_total=Sum('projects__upvotes'),
                                   projects_total=Count('projects')).get(id=tag_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_tag.html', {'tag': tag})

def languages_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'languages.html', {'languages_list': pagination_generic(page, sorting, Language,
                                                                                'lower(language_name)',
                                                                                'lowercase_name')})

def tags_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'tags.html', {'tags_list': pagination_generic(page, sorting, Tag, 'lower(text)',
                                                                         'lowercase_text')})

def authors_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'authors.html', {'authors_list': pagination_generic(page, sorting,
                                                                               Author, 'lower(author_name)',
                                                                               'lowercase_name')})

def main_page(request):
    return render(request, 'index.html')<commit_msg>basic submit form handling<commit_after>from abandoned.githubapi import get_project_data, AbandonedException, GithubException
from abandoned.forms import ProjectForm
from abandoned.models import Project, Tag, Author, Reason, Language
from abandoned.serializers import TagSerializer, ProjectSerializer, ReasonSerializer, AuthorSerializer, LanguageSerializer
from django.core.urlresolvers import reverse
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db.models import Count, Sum
from django.http import HttpResponse, HttpResponseBadRequest, Http404, HttpResponseRedirect
from django.shortcuts import render
from django.views.decorators.csrf import csrf_protect
from json import loads
from rest_framework import viewsets
from rest_framework.decorators import list_route
from rest_framework.response import Response

def votes_generic(model_obj, class_instance):
    votes = model_obj.objects.annotate(votes_total=Sum('projects__upvotes')).order_by('votes_total').reverse()
    page = class_instance.paginate_queryset(votes)
    serializer = class_instance.get_pagination_serializer(page)
    return Response(serializer.data)

def projects_generic(model_obj, class_instance):
    votes = model_obj.objects.annotate(Count('projects')).order_by('projects__count').reverse()
    page = class_instance.paginate_queryset(votes)
    serializer = class_instance.get_pagination_serializer(page)
    return Response(serializer.data)

def pagination_generic(page, sorting, model_obj, sql_lowercase, lowercase_name):
    if sorting == 'projects':
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).\
            order_by('projects_total').reverse()
    elif sorting == 'votes':
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).order_by('votes_total').reverse()
    else:
        result_list = model_obj.objects.annotate(votes_total=Sum('projects__upvotes'),
                                                 projects_total=Count('projects')).\
            extra(select={lowercase_name: sql_lowercase}).order_by(lowercase_name)
    paginator = Paginator(result_list, 10)
    try:
        result = paginator.page(page)
    except PageNotAnInteger:
        result = paginator.page(1)
    except EmptyPage:
        result = paginator.page(paginator.num_pages)
    return result

class AuthorViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Author.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = AuthorSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Author, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Author, self)

class TagViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Tag.objects.extra(select={'lowercase_text': 'lower(text)'}).order_by('lowercase_text')
    serializer_class = TagSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Tag, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Tag, self)

class ReasonViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Reason.objects.extra(select={'lowercase_reason': 'lower(reason)'}).order_by('lowercase_reason')
    serializer_class = ReasonSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Reason, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Reason, self)

class LanguageViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Language.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = LanguageSerializer

    @list_route()
    def votes(self, request):
        return votes_generic(Language, self)

    @list_route()
    def projects(self, request):
        return projects_generic(Language, self)

class ProjectViewSet(viewsets.ReadOnlyModelViewSet):
    queryset = Project.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    serializer_class = ProjectSerializer

    @list_route()
    def votes(self, request):
        top = Project.objects.all().order_by('upvotes').reverse()
        page = self.paginate_queryset(top)
        serializer = self.get_pagination_serializer(page)
        return Response(serializer.data)

    @list_route()
    def latest(self, request):
        latest = Project.objects.all().order_by('date_added').reverse()
        page = self.paginate_queryset(latest)
        serializer = self.get_pagination_serializer(page)
        return Response(serializer.data)

def handle_submit(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST)
        # print(form.)
        if form.is_valid():
            repo_url = form.cleaned_data['link']
            project_data = get_project_data(repo_url)
            repo_name, author_name, language = project_data
            if language is None:
                language = 'Language not specified'
            author_url = 'https://github.com/' + author_name
            repo_url = 'https://github.com/' + author_name + '/' + repo_name

            if not Project.objects.filter(link=repo_url).exists():
                curr_author, author_created = Author.objects.get_or_create(author_name=author_name,
                                                                           author_link=author_url)
                curr_language, language_created = Language.objects.get_or_create(language_name=language)

                tags_list = form.cleaned_data['tags_textfield'].split(',')

                for i, tag_text in enumerate(tags_list):
                    tag_text = tag_text.strip()
                    tags_list[i] = tag_text
                    curr_tag, tag_created = Tag.objects.get_or_create(text=tag_text)
                    tags_list[i] = curr_tag

                curr_project = Project.objects.create(name=repo_name, link=repo_url, author=curr_author,
                                                      description=form.cleaned_data['description'],
                                                      reason=form.cleaned_data['reason'], language=curr_language)
                for tag in tags_list:
                    curr_project.tags.add(tag)
                return HttpResponseRedirect(reverse('abandoned.views.single_project_view', args=(curr_project.id,)))
    else:
        form = ProjectForm()
        return render(request, 'submit.html', {'form': form})
    # request_data = loads(request.body.decode('utf-8'))
    # try:
    #     project_data = get_project_data(request_data['repo_url'])
    # except AbandonedException as e:
    #     return HttpResponseBadRequest(str(e))
    # except GithubException as e:
    #     return HttpResponse(str(e), status=503)
    # repo_name, author_name, language = project_data
    # if language is None:
    #     language = 'Language not specified'
    # author_url = 'https://github.com/' + author_name
    # repo_url = 'https://github.com/' + author_name + '/' + repo_name
    #
    # if not Project.objects.filter(link=repo_url).exists():
    #
    #     curr_reason = Reason.objects.get(id=request_data['reason_id'])
    #     curr_author, author_created = Author.objects.get_or_create(name=author_name, link=author_url)
    #     curr_language, language_created = Language.objects.get_or_create(name=language)
    #
    #     tags_list = []
    #     for tag_text in request_data['tags']:
    #         curr_tag, tag_created = Tag.objects.get_or_create(text=tag_text)
    #         tags_list.append(curr_tag)
    #
    #     curr_project = Project.objects.create(name=repo_name, link=repo_url, author=curr_author,
    #                                           description=request_data['description'],
    #                                           reason=curr_reason, language=curr_language)
    #     for tag in tags_list:
    #         curr_project.tags.add(tag)
    #     curr_project.save()
    #     serializer = ProjectSerializer(curr_project)
    #     return Response(serializer.data, status=status.HTTP_201_CREATED)
    # else:
    #     return HttpResponseBadRequest("This project is already in the database")

def projects_view(request, sorting='latest'):
    page = request.GET.get('page')
    if sorting == 'latest':
        projects_list = Project.objects.order_by('date_added').reverse()
    elif sorting == 'votes':
        projects_list = Project.objects.order_by('upvotes').reverse()
    else:
        projects_list = Project.objects.extra(select={'lowercase_name': 'lower(name)'}).order_by('lowercase_name')
    paginator = Paginator(projects_list, 10)
    try:
        projects = paginator.page(page)
    except PageNotAnInteger:
        projects = paginator.page(1)
    except EmptyPage:
        projects = paginator.page(paginator.num_pages)
    return render(request, 'projects.html', {'projects_list': projects})

def single_project_view(request, project_id):
    try:
        project = Project.objects.get(id=project_id)
    except Project.DoesNotExist:
        raise Http404
    return render(request, 'single_project.html', {'project': project})

def single_author_view(request, author_id):
    try:
        author = Author.objects.annotate(votes_total=Sum('projects__upvotes'),
                                         projects_total=Count('projects')).get(id=author_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_author.html', {'author': author})

def single_language_view(request, language_id):
    try:
        language = Language.objects.annotate(votes_total=Sum('projects__upvotes'),
                                             projects_total=Count('projects')).get(id=language_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_language.html', {'language': language})

def single_tag_view(request, tag_id):
    try:
        tag = Tag.objects.annotate(votes_total=Sum('projects__upvotes'),
                                   projects_total=Count('projects')).get(id=tag_id)
    except Author.DoesNotExist:
        raise Http404
    return render(request, 'single_tag.html', {'tag': tag})

def languages_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'languages.html', {'languages_list': pagination_generic(page, sorting, Language,
                                                                                'lower(language_name)',
                                                                                'lowercase_name')})

def tags_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'tags.html', {'tags_list': pagination_generic(page, sorting, Tag, 'lower(text)',
                                                                         'lowercase_text')})

def authors_view(request, sorting='alphabetical'):
    page = request.GET.get('page')
    return render(request, 'authors.html', {'authors_list': pagination_generic(page, sorting,
                                                                               Author, 'lower(author_name)',
                                                                               'lowercase_name')})

def main_page(request):
    return render(request, 'index.html')<|endoftext|>


<commit_before># Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import multiprocessing
import random

from oslo_log import log as logging

from act.engine import config
from act.engine import utils

LOG = logging.getLogger(__name__)

OPERATION_ACT = 'act'
OPERATION_RESPONSE = 'response'

class World(object):
    def __init__(self):
        self.storage = {}

    def put(self, item):
        self.storage[item.id] = item

    def pop(self, item):
        del self.storage[item.id]

    def get_item_types(self):
        item_types = set()
        for item in self.storage.values():
            if item.can_be_used():
                item_types.add(item.item_type)
        return item_types

    def pick_items(self, item_types):
        if not item_types:
            return

        items_by_types = collections.defaultdict(list)
        for item in self.storage.values():
            if item.can_be_used():
                items_by_types[item.item_type].append(item)

        items = [random.choice(v) for v in items_by_types.values()]
        for item in items:
            item.lock()

        return items

class Action(object):
    weight = 0.1
    meta_type = None
    depends_on = set()

    def __init__(self):
        super(Action, self).__init__()

        # if type(self).meta_type:
        #     type(self).depends_on.add(type(self).meta_type)

    def get_weight(self):
        return self.weight

    def get_meta_type(self):
        return self.meta_type

    def get_depends_on(self):
        return self.depends_on

    def act(self, items):
        LOG.info('act(%s)', items)

class CreateAction(Action):
    weight = 0.9

    def __init__(self):
        super(CreateAction, self).__init__()

class CreateNet(CreateAction):
    meta_type = 'meta_net'
    depends_on = {'meta_net'}

    def act(self, items):
        LOG.info('Create Net is called! %s', items)
        return 'some net created'

class DoNothing(Action):

    def act(self, items):
        LOG.info('Do Nothing is called! %s', items)

REGISTRY = [CreateNet(), DoNothing()]

class Item(object):
    def __init__(self, item_type, ref_count_limit=1000):
        self.item_type = item_type
        self.id = utils.make_id()
        self.refs = set()
        self.ref_count_limit = ref_count_limit
        self.is_locked = False

    def __repr__(self):
        return str(dict(id=self.id, item_type=self.item_type))

    def lock(self):
        self.is_locked = True

    def unlock(self):
        self.is_locked = False

    def can_be_used(self):
        return (not self.is_locked) and len(self.refs) < self.ref_count_limit

    def add_ref(self, other_id):
        self.refs.add(other_id)

    def del_ref(self, other_id):
        self.refs.pop(other_id)

class Worker(multiprocessing.Process):

    def __init__(self, task_queue, result_queue):
        multiprocessing.Process.__init__(self)
        self.task_queue = task_queue
        self.result_queue = result_queue

    def run(self):
        proc_name = self.name
        while True:
            next_task = self.task_queue.get()
            if next_task is None:
                # Poison pill means shutdown
                LOG.info('%s: Exiting', proc_name)
                self.task_queue.task_done()
                break
            LOG.info('%s executing action %s', proc_name, next_task)
            result = next_task.action.act(next_task.items)
            LOG.info('%s put result %s into queue', proc_name, result)
            self.task_queue.task_done()
            self.result_queue.put(result)
        return

Task = collections.namedtuple('Task', ['action', 'items'])

def produce_task(world, actions, task_queue):
    available_item_types = world.get_item_types()
    available_actions = [a for a in actions
                         if a.get_depends_on() <= available_item_types]

    action = random.choice(available_actions)
    items = world.pick_items(action.get_depends_on())

    task_queue.put(Task(action=action, items=items))

def handle_operation(op):
    LOG.info('Handle: %s', op)

def process():
    # Establish communication queues
    task_queue = multiprocessing.JoinableQueue()
    result_queue = multiprocessing.Queue()

    # Start workers
    workers_count = 2  # multiprocessing.cpu_count() * 2
    LOG.info('Creating %d workers' % workers_count)
    workers = [Worker(task_queue, result_queue) for i in range(workers_count)]
    for w in workers:
        w.start()

    default_items = []
    for action_klazz in REGISTRY:
        meta_type = action_klazz.get_meta_type()
        if meta_type:
            item = Item(meta_type)
            default_items.append(item)

    world = World()
    for item in default_items:
        world.put(item)

    for i in range(workers_count):
        result_queue.put('dummy')

    steps = 4

    for i in range(steps):
        res = result_queue.get()
        handle_operation(res)

        produce_task(world, REGISTRY, task_queue)

    # Add a poison pill for each worker
    for i in range(workers_count):
        task_queue.put(None)

    # Wait for all of the tasks to finish
    task_queue.join()

def main():
    utils.init_config_and_logging(config.MAIN_OPTS)

    process()

if __name__ == "__main__":
    main()
<commit_msg>Create Operation<commit_after># Copyright (c) 2016 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import multiprocessing
import random

from oslo_log import log as logging

from act.engine import config
from act.engine import utils

LOG = logging.getLogger(__name__)

OPERATION_ACT = 'act'
OPERATION_RESPONSE = 'response'

class World(object):
    def __init__(self):
        self.storage = {}

    def put(self, item, dependencies=None):
        self.storage[item.id] = item

        if dependencies:
            for dep in dependencies:
                # dependencies are copies of storage objects!
                real_dep = self.storage[dep.id]
                real_dep.add_ref(item.id)
                real_dep.unlock()

    def pop(self, item):
        del self.storage[item.id]

    def get_item_types(self):
        item_types = set()
        for item in self.storage.values():
            if item.can_be_used():
                item_types.add(item.item_type)
        return item_types

    def pick_items(self, item_types):
        if not item_types:
            return

        items_by_types = collections.defaultdict(list)
        for item in self.storage.values():
            if item.item_type in item_types and item.can_be_used():
                items_by_types[item.item_type].append(item)

        items = [random.choice(v) for v in items_by_types.values()]
        for item in items:
            item.lock()

        return items

    def __repr__(self):
        return str(self.storage)

class Action(object):
    weight = 0.1
    meta_type = None
    depends_on = set()

    def __init__(self):
        super(Action, self).__init__()

    def get_weight(self):
        return self.weight

    def get_meta_type(self):
        return self.meta_type

    def get_depends_on(self):
        return self.depends_on

    def act(self, items):
        LOG.info('act(%s)', items)

    def get_operation_class(self):
        return Operation

class CreateAction(Action):
    weight = 0.9

    def __init__(self):
        super(CreateAction, self).__init__()

    def get_operation_class(self):
        return CreateOperation

class CreateNet(CreateAction):
    meta_type = 'meta_net'
    depends_on = {'meta_net'}

    def act(self, items):
        LOG.info('Create Net is called! %s', items)
        net = dict(name='foo', id='1234')
        return Item('net', net, ref_count_limit=10)

class DoNothing(Action):

    def act(self, items):
        LOG.info('Do Nothing is called! %s', items)

REGISTRY = [CreateNet(), DoNothing()]

class Operation(object):
    def __init__(self, item, dependencies):
        self.item = item
        self.dependencies = dependencies

    def do(self, world):
        pass

class CreateOperation(Operation):
    def do(self, world):
        LOG.info('Add something to world')
        world.put(self.item, self.dependencies)

class Item(object):
    def __init__(self, item_type, payload, ref_count_limit=1000):
        self.item_type = item_type
        self.payload = payload
        self.id = utils.make_id()
        self.refs = set()
        self.ref_count_limit = ref_count_limit
        self.lock_count = 0

    def __repr__(self):
        return str(dict(id=self.id, item_type=self.item_type,
                        payload=self.payload, ref_count=len(self.refs)))

    def lock(self):
        self.lock_count += 1

    def unlock(self):
        self.lock_count -= 1

    def can_be_used(self):
        return len(self.refs) + self.lock_count < self.ref_count_limit

    def add_ref(self, other_id):
        self.refs.add(other_id)

    def del_ref(self, other_id):
        self.refs.pop(other_id)

class Worker(multiprocessing.Process):

    def __init__(self, task_queue, result_queue):
        multiprocessing.Process.__init__(self)
        self.task_queue = task_queue
        self.result_queue = result_queue

    def run(self):
        proc_name = self.name
        while True:
            task = self.task_queue.get()

            if task is None:
                # Poison pill means shutdown
                LOG.info('%s: Exiting', proc_name)
                self.task_queue.task_done()
                break

            LOG.info('%s executing action %s', proc_name, task)

            action = task.action
            action_result = action.act(task.items)
            operation_class = action.get_operation_class()
            operation = operation_class(item=action_result,
                                        dependencies=task.items)

            LOG.info('%s put operation %s into queue', proc_name, operation)
            self.task_queue.task_done()
            self.result_queue.put(operation)
        return

Task = collections.namedtuple('Task', ['action', 'items'])

def produce_task(world, actions, task_queue):
    available_item_types = world.get_item_types()
    available_actions = [a for a in actions
                         if a.get_depends_on() <= available_item_types]

    action = random.choice(available_actions)
    items = world.pick_items(action.get_depends_on())

    task_queue.put(Task(action=action, items=items))

def handle_operation(op, world):
    LOG.info('Handle: %s', op)
    op.do(world)

def process():
    # Establish communication queues
    task_queue = multiprocessing.JoinableQueue()
    result_queue = multiprocessing.Queue()

    # Start workers
    workers_count = 2  # multiprocessing.cpu_count() * 2
    LOG.info('Creating %d workers' % workers_count)
    workers = [Worker(task_queue, result_queue) for i in range(workers_count)]
    for w in workers:
        w.start()

    default_items = []
    for action_klazz in REGISTRY:
        meta_type = action_klazz.get_meta_type()
        if meta_type:
            item = Item(meta_type, None)
            default_items.append(item)

    world = World()
    for item in default_items:
        world.put(item)

    for i in range(workers_count):
        result_queue.put(Operation(None, None))

    steps = 10

    for i in range(steps):
        operation = result_queue.get()
        handle_operation(operation, world)

        produce_task(world, REGISTRY, task_queue)

    # todo pick remainders from result_queue

    # Add a poison pill for each worker
    for i in range(workers_count):
        task_queue.put(None)

    # Wait for all of the tasks to finish
    task_queue.join()

    LOG.info('World: %s', world)

def main():
    utils.init_config_and_logging(config.MAIN_OPTS)

    process()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import os
import sys
from _ssl import PROTOCOL_SSLv23
from collections import Mapping
from ssl import SSLContext
from tempfile import NamedTemporaryFile

import flask
from flask import Flask
from werkzeug.contrib.fixers import HeaderRewriterFix
from werkzeug.wrappers import Response

import active_data
from active_data import record_request, cors_wrapper
from active_data.actions import save_query
from active_data.actions.json import get_raw_json
from active_data.actions.jx import jx_query
from active_data.actions.save_query import SaveQueries, find_query
from active_data.actions.sql import sql_query
from active_data.actions.static import download
from jx_base import container
from mo_files import File
from mo_logs import Log
from mo_logs import constants, startup
from mo_threads import Thread
from pyLibrary import convert
from pyLibrary.env import elasticsearch

OVERVIEW = File("active_data/public/index.html").read()

app = Flask(__name__)
config = None

@app.route('/', defaults={'path': ''}, methods=['OPTIONS', 'HEAD'])
@app.route('/<path:path>', methods=['OPTIONS', 'HEAD'])
@cors_wrapper
def _head(path):
    return Response(b'', status=200)

app.add_url_rule('/tools/<path:filename>', None, download)
app.add_url_rule('/find/<path:hash>', None, find_query)
app.add_url_rule('/query', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql/', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/<path:path>', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/json/<path:path>', None, get_raw_json, methods=['GET'])

@app.route('/', defaults={'path': ''}, methods=['GET', 'POST'])
@cors_wrapper
def _default(path):
    record_request(flask.request, None, flask.request.get_data(), None)

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=200,
        headers={
            "Content-Type": "text/html"
        }
    )

def setup():
    global config

    try:
        config = startup.read_settings(
            defs=[
                {
                    "name": ["--process_num", "--process"],
                    "help": "Additional port offset (for multiple Flask processes",
                    "type": int,
                    "dest": "process_num",
                    "default": 0,
                    "required": False
                },
                {
                    "name": "app_name",
                    "help": "gunicorn supplied argument",
                    "type": str
                }
            ],
            env_filename=os.environ.get('ACTIVEDATA_CONFIG')
        )

        constants.set(config.constants)
        Log.start(config.debug)

        # PIPE REQUEST LOGS TO ES DEBUG
        if config.request_logs:
            request_logger = elasticsearch.Cluster(config.request_logs).get_or_create_index(config.request_logs)
            active_data.request_log_queue = request_logger.threaded_queue(max_size=2000)

        # SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
        container.config.default = {
            "type": "elasticsearch",
            "settings": config.elasticsearch.copy()
        }

        # TRIGGER FIRST INSTANCE
        if config.saved_queries:
            setattr(save_query, "query_finder", SaveQueries(config.saved_queries))
        HeaderRewriterFix(app, remove_headers=['Date', 'Server'])

    except BaseException as e:  # MUST CATCH BaseException BECAUSE gunicorn LIKES TO EXIT THAT WAY, AND NOT REPORT
        Log.error("Serious problem with ActiveData service construction!  Shutdown!", cause=e)

def run_flask():
    if config.flask.port and config.args.process_num:
        config.flask.port += config.args.process_num

    # TURN ON /exit FOR WINDOWS DEBUGGING
    if config.flask.debug or config.flask.allow_exit:
        config.flask.allow_exit = None
        Log.warning("ActiveData is in debug mode")
        app.add_url_rule('/exit', 'exit', _exit)

    if config.flask.ssl_context:
        if config.args.process_num:
            Log.error("can not serve ssl and multiple Flask instances at once")
        setup_flask_ssl()

    app.run(**config.flask)

def run_gunicorn():
    from gunicorn.app.base import BaseApplication

    class GunicornApp(BaseApplication):

        def load(self):
            return app

    GunicornApp().run()

def setup_flask_ssl():
    config.flask.ssl_context = None

    if not config.flask.ssl_context:
        return

    ssl_flask = config.flask.copy()
    ssl_flask.debug = False
    ssl_flask.port = 443

    if isinstance(config.flask.ssl_context, Mapping):
        # EXPECTED PEM ENCODED FILE NAMES
        # `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS
        tempfile = NamedTemporaryFile(delete=False, suffix=".pem")
        try:
            tempfile.write(File(ssl_flask.ssl_context.certificate_file).read_bytes())
            if ssl_flask.ssl_context.certificate_chain_file:
                tempfile.write(File(ssl_flask.ssl_context.certificate_chain_file).read_bytes())
            tempfile.flush()
            tempfile.close()

            context = SSLContext(PROTOCOL_SSLv23)
            context.load_cert_chain(tempfile.name, keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath)

            ssl_flask.ssl_context = context
        except Exception, e:
            Log.error("Could not handle ssl context construction", cause=e)
        finally:
            try:
                tempfile.delete()
            except Exception:
                pass

    def runner(please_stop):
        Log.warning("ActiveData listening on encrypted port {{port}}", port=ssl_flask.port)
        app.run(**ssl_flask)

    Thread.run("SSL Server", runner)

    if config.flask.ssl_context and config.flask.port != 80:
        Log.warning("ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}", port=config.flask.port)

    config.flask.ssl_context = None

def _exit():
    Log.note("Got request to shutdown")
    shutdown = flask.request.environ.get('werkzeug.server.shutdown')
    if shutdown:
        shutdown()
    else:
        Log.warning("werkzeug.server.shutdown does not exist")

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=400,
        headers={
            "Content-Type": "text/html"
        }
    )

if __name__ in ("__main__", "active_data.app"):
    try:
        setup()
        if config.flask:
            run_flask()
        else:
            run_gunicorn()
    finally:
        Log.stop()

    sys.exit(0)

<commit_msg>git commit<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (kyle@lahnakoski.com)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import os
import sys
from _ssl import PROTOCOL_SSLv23
from collections import Mapping
from ssl import SSLContext
from tempfile import NamedTemporaryFile

import flask
from flask import Flask
from werkzeug.contrib.fixers import HeaderRewriterFix
from werkzeug.wrappers import Response

import active_data
from active_data import record_request, cors_wrapper
from active_data.actions import save_query
from active_data.actions.json import get_raw_json
from active_data.actions.jx import jx_query
from active_data.actions.save_query import SaveQueries, find_query
from active_data.actions.sql import sql_query
from active_data.actions.static import download
from jx_base import container
from mo_files import File
from mo_logs import Log
from mo_logs import constants, startup
from mo_threads import Thread
from pyLibrary import convert
from pyLibrary.env import elasticsearch

OVERVIEW = File("active_data/public/index.html").read()

app = Flask(__name__)
config = None

@app.route('/', defaults={'path': ''}, methods=['OPTIONS', 'HEAD'])
@app.route('/<path:path>', methods=['OPTIONS', 'HEAD'])
@cors_wrapper
def _head(path):
    return Response(b'', status=200)

app.add_url_rule('/tools/<path:filename>', None, download)
app.add_url_rule('/find/<path:hash>', None, find_query)
app.add_url_rule('/query', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/sql/', None, sql_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/query/<path:path>', None, jx_query, defaults={'path': ''}, methods=['GET', 'POST'])
app.add_url_rule('/json/<path:path>', None, get_raw_json, methods=['GET'])

@app.route('/', defaults={'path': ''}, methods=['GET', 'POST'])
@cors_wrapper
def _default(path):
    record_request(flask.request, None, flask.request.get_data(), None)

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=200,
        headers={
            "Content-Type": "text/html"
        }
    )

def setup():
    global config

    config = startup.read_settings(
        defs=[
            {
                "name": ["--process_num", "--process"],
                "help": "Additional port offset (for multiple Flask processes",
                "type": int,
                "dest": "process_num",
                "default": 0,
                "required": False
            },
            {
                "name": "app_name",
                "help": "gunicorn supplied argument",
                "type": str
            }
        ],
        env_filename=os.environ.get('ACTIVEDATA_CONFIG')
    )

    constants.set(config.constants)
    Log.start(config.debug)

    # PIPE REQUEST LOGS TO ES DEBUG
    if config.request_logs:
        request_logger = elasticsearch.Cluster(config.request_logs).get_or_create_index(config.request_logs)
        active_data.request_log_queue = request_logger.threaded_queue(max_size=2000)

    # SETUP DEFAULT CONTAINER, SO THERE IS SOMETHING TO QUERY
    container.config.default = {
        "type": "elasticsearch",
        "settings": config.elasticsearch.copy()
    }

    # TRIGGER FIRST INSTANCE
    if config.saved_queries:
        setattr(save_query, "query_finder", SaveQueries(config.saved_queries))

    HeaderRewriterFix(app, remove_headers=['Date', 'Server'])

def run_flask():
    if config.flask.port and config.args.process_num:
        config.flask.port += config.args.process_num

    # TURN ON /exit FOR WINDOWS DEBUGGING
    if config.flask.debug or config.flask.allow_exit:
        config.flask.allow_exit = None
        Log.warning("ActiveData is in debug mode")
        app.add_url_rule('/exit', 'exit', _exit)

    if config.flask.ssl_context:
        if config.args.process_num:
            Log.error("can not serve ssl and multiple Flask instances at once")
        setup_flask_ssl()

    app.run(**config.flask)

def run_gunicorn():
    from gunicorn.app.base import BaseApplication

    class GunicornApp(BaseApplication):

        def load(self):
            return app

    GunicornApp().run()

def setup_flask_ssl():
    config.flask.ssl_context = None

    if not config.flask.ssl_context:
        return

    ssl_flask = config.flask.copy()
    ssl_flask.debug = False
    ssl_flask.port = 443

    if isinstance(config.flask.ssl_context, Mapping):
        # EXPECTED PEM ENCODED FILE NAMES
        # `load_cert_chain` REQUIRES CONCATENATED LIST OF CERTS
        tempfile = NamedTemporaryFile(delete=False, suffix=".pem")
        try:
            tempfile.write(File(ssl_flask.ssl_context.certificate_file).read_bytes())
            if ssl_flask.ssl_context.certificate_chain_file:
                tempfile.write(File(ssl_flask.ssl_context.certificate_chain_file).read_bytes())
            tempfile.flush()
            tempfile.close()

            context = SSLContext(PROTOCOL_SSLv23)
            context.load_cert_chain(tempfile.name, keyfile=File(ssl_flask.ssl_context.privatekey_file).abspath)

            ssl_flask.ssl_context = context
        except Exception, e:
            Log.error("Could not handle ssl context construction", cause=e)
        finally:
            try:
                tempfile.delete()
            except Exception:
                pass

    def runner(please_stop):
        Log.warning("ActiveData listening on encrypted port {{port}}", port=ssl_flask.port)
        app.run(**ssl_flask)

    Thread.run("SSL Server", runner)

    if config.flask.ssl_context and config.flask.port != 80:
        Log.warning("ActiveData has SSL context, but is still listening on non-encrypted http port {{port}}", port=config.flask.port)

    config.flask.ssl_context = None

def _exit():
    Log.note("Got request to shutdown")
    shutdown = flask.request.environ.get('werkzeug.server.shutdown')
    if shutdown:
        shutdown()
    else:
        Log.warning("werkzeug.server.shutdown does not exist")

    return Response(
        convert.unicode2utf8(OVERVIEW),
        status=400,
        headers={
            "Content-Type": "text/html"
        }
    )

if __name__ in ("__main__", "active_data.app"):
    try:
        print("setup")
        setup()
        if config.flask:
            run_flask()
        else:
            print("run gunicorn")
            run_gunicorn()
    except BaseException as e:  # MUST CATCH BaseException BECAUSE argparse LIKES TO EXIT THAT WAY, AND gunicorn WILL NOT REPORT
        Log.warning("Serious problem with ActiveData service construction!  Shutdown!", cause=e)
    finally:
        Log.stop()

    sys.exit(0)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging
from collections import namedtuple

import click
import trollius as asyncio
from trollius import From

logger = logging.getLogger(__name__)

ClientTuple = namedtuple('ClientTuple', 'timeout, reader, writer')

class TypedServer(object):

    def __init__(self, host, port, timeout, corrector):
        self.server = None
        self.loop = None
        self.corrector = corrector
        self.connections = {}
        self.timers = {}
        self.listen_host = host
        self.listen_port = port
        self.timeout = timeout / 1000

    def on_connect(self, reader, writer):
        client = ClientTuple(timeout=None, reader=reader, writer=writer)
        task = asyncio.Task(self.process(client))
        task.add_done_callback(self.on_disconnect)
        self.connections[task] = client
        timer = self.loop.call_later(self.timeout, self.on_disconnect, task)
        self.timers[task] = timer
        client_ip = self.client_ip(client)
        logger.debug("{client}:connect".format(client=client_ip))

    def client_ip(self, client):
        transport = client.writer.transport
        (remote_ip, remote_port) = transport.get_extra_info('peername')
        return '{ip}:{port}'.format(ip=remote_ip, port=remote_port)

    def on_disconnect(self, task):
        if task in self.timers:
            timer = self.timers[task]
            del self.timers[task]
            timer.cancel()
        if task in self.connections:
            client = self.connections[task]
            client_ip = self.client_ip(client)
            client.writer.close()
            del self.connections[task]
            logger.debug("{client}:disconnect".format(client=client_ip))

    @asyncio.coroutine
    def process(self, client):
        query = (yield From(asyncio.wait_for(client.reader.readline(),
                                             timeout=0.01)))
        if not query:
            return
        query = query.strip()
        client_ip = self.client_ip(client)
        cmd = query.split(' ', 1)
        if cmd[0] == 'RELOAD':
            self.corrector.reload()
            result = 'DONE'
        elif cmd[0] == 'QUERY' and len(cmd) > 1:
            data = cmd[1]
            typo = unicode(data, "utf-8")

            corrected, is_success = self.corrector.suggestion(typo)
            result = corrected.encode('utf-8') if is_success else data
        else:
            result = 'ERROR'

        client.writer.write('{}\n'.format(result))
        yield From(client.writer.drain())
        logger.info("{client}:request:{request}:{result}"
                    .format(client=client_ip, request=query, result=result))

    def start(self, loop):
        server = asyncio.streams.start_server(
            self.on_connect, self.listen_host, self.listen_port, loop=loop
        )
        self.loop = loop
        self.server = loop.run_until_complete(server)

    def stop(self, loop):
        if self.server is not None:
            self.server.close()
            loop.run_until_complete(self.server.wait_closed())
            self.server = None

@click.group()
def server_group():
    pass

@server_group.command()
@click.option('--host', type=str, default='0.0.0.0', required=True)
@click.option('--port', type=click.IntRange(1, 65535), default=3333,
              required=True)
@click.option('--timeout', type=click.IntRange(1, 5000), default=1000,
              required=True)
@click.pass_context
def server(ctx, host, port, timeout):
    """Typod server"""

    corrector_index = ctx.obj['corrector_index']
    corrector_cls = ctx.obj['corrector']
    inst = corrector_cls(corrector_index)
    server = TypedServer(host=host,
                         port=port,
                         timeout=timeout,
                         corrector=inst)

    loop = asyncio.get_event_loop()
    logger.info('Run server on {}:{}, using {} corrector'
                .format(host, port, corrector_cls.typo_name))
    server.start(loop)
    try:
        loop.run_forever()
    finally:
        loop.close()
<commit_msg>always return corrected data<commit_after># -*- coding: utf-8 -*-
import logging
from collections import namedtuple

import click
import trollius as asyncio
from trollius import From

logger = logging.getLogger(__name__)

ClientTuple = namedtuple('ClientTuple', 'timeout, reader, writer')

class TypedServer(object):

    def __init__(self, host, port, timeout, corrector):
        self.server = None
        self.loop = None
        self.corrector = corrector
        self.connections = {}
        self.timers = {}
        self.listen_host = host
        self.listen_port = port
        self.timeout = timeout / 1000

    def on_connect(self, reader, writer):
        client = ClientTuple(timeout=None, reader=reader, writer=writer)
        task = asyncio.Task(self.process(client))
        task.add_done_callback(self.on_disconnect)
        self.connections[task] = client
        timer = self.loop.call_later(self.timeout, self.on_disconnect, task)
        self.timers[task] = timer
        client_ip = self.client_ip(client)
        logger.debug("{client}:connect".format(client=client_ip))

    def client_ip(self, client):
        transport = client.writer.transport
        (remote_ip, remote_port) = transport.get_extra_info('peername')
        return '{ip}:{port}'.format(ip=remote_ip, port=remote_port)

    def on_disconnect(self, task):
        if task in self.timers:
            timer = self.timers[task]
            del self.timers[task]
            timer.cancel()
        if task in self.connections:
            client = self.connections[task]
            client_ip = self.client_ip(client)
            client.writer.close()
            del self.connections[task]
            logger.debug("{client}:disconnect".format(client=client_ip))

    @asyncio.coroutine
    def process(self, client):
        query = (yield From(asyncio.wait_for(client.reader.readline(),
                                             timeout=0.01)))
        if not query:
            return
        query = query.strip()
        client_ip = self.client_ip(client)
        cmd = query.split(' ', 1)
        if cmd[0] == 'RELOAD':
            self.corrector.reload()
            result = 'DONE'
        elif cmd[0] == 'QUERY' and len(cmd) > 1:
            data = cmd[1]
            typo = unicode(data, "utf-8")

            corrected, is_success = self.corrector.suggestion(typo)
            result = corrected.encode('utf-8')
        else:
            result = 'ERROR'

        client.writer.write('{}\n'.format(result))
        yield From(client.writer.drain())
        logger.info("{client}:request:{request}:{result}"
                    .format(client=client_ip, request=query, result=result))

    def start(self, loop):
        server = asyncio.streams.start_server(
            self.on_connect, self.listen_host, self.listen_port, loop=loop
        )
        self.loop = loop
        self.server = loop.run_until_complete(server)

    def stop(self, loop):
        if self.server is not None:
            self.server.close()
            loop.run_until_complete(self.server.wait_closed())
            self.server = None

@click.group()
def server_group():
    pass

@server_group.command()
@click.option('--host', type=str, default='0.0.0.0', required=True)
@click.option('--port', type=click.IntRange(1, 65535), default=3333,
              required=True)
@click.option('--timeout', type=click.IntRange(1, 5000), default=1000,
              required=True)
@click.pass_context
def server(ctx, host, port, timeout):
    """Typod server"""

    corrector_index = ctx.obj['corrector_index']
    corrector_cls = ctx.obj['corrector']
    inst = corrector_cls(corrector_index)
    server = TypedServer(host=host,
                         port=port,
                         timeout=timeout,
                         corrector=inst)

    loop = asyncio.get_event_loop()
    logger.info('Run server on {}:{}, using {} corrector'
                .format(host, port, corrector_cls.typo_name))
    server.start(loop)
    try:
        loop.run_forever()
    finally:
        loop.close()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""tzwhere.py - time zone computation from latitude/longitude.

Ordinarily this is loaded as a module and instances of the tzwhere
class are instantiated and queried directly, but the module can be run
as a script too (this requires the docopt package to be installed).
Run it with the -h option to see usage.

"""

import csv
import collections
import datetime
try:
    import json
except ImportError:
    import simplejson as json
import math
import os
import pickle

# We can speed up things by about 200 times if we use shapely
try:
    from shapely.geometry import Polygon, Point
    from shapely.prepared import prep
    SHAPELY_IMPORT = True
except ImportError:
    SHAPELY_IMPORT = False

# We can save about 222MB of RAM by turning our polygon lists into
# numpy arrays rather than tuples, if numpy is installed.
try:
    import numpy
    WRAP = numpy.array
except ImportError:
    WRAP = tuple

class tzwhere(object):

    SHORTCUT_DEGREES_LATITUDE = 1
    SHORTCUT_DEGREES_LONGITUDE = 1
    # By default, use the data file in our package directory
    DEFAULT_JSON = os.path.join(os.path.dirname(__file__),
                                'tz_world_compact.json')
    DEFAULT_PICKLE = os.path.join(os.path.dirname(__file__),
                                  'tz_world.pickle')
    DEFAULT_CSV = os.path.join(os.path.dirname(__file__),
                               'tz_world.csv')

    def __init__(self, input_kind='json', path=None,
                 shapely=False, forceTZ=False):
        '''
        Initializes the tzwhere class.
        @input_kind: Which filetype you want to read from
        @path: Where you want to read the input file from
        @shapely: Whether you want to use shapely to represent the geometry.
        Lookups become much faster at the cost of a slower initialization
        @forceTZ: If you want to force the lookup method to a return a
        timezone even if the point you are looking up is slightly outside it's
        bounds, you need to specify this during initialization arleady

        '''

        # Make sure people have an idea why things are not working later down
        # the line and rather tell them now
        if (shapely or forceTZ) and not SHAPELY_IMPORT:
            raise ValueError('You need to have shapley installed for this '
                             'feature, but we can\'t find it')
        if not shapely and forceTZ:
            raise ValueError('The lookup \'hack\' depends on shapely. Try using'
                             ' shapely=True when initializing the class')

        self.forceTZ = forceTZ
        self.shapely = shapely and SHAPELY_IMPORT

        # Construct appropriate generator for (tz, polygon) pairs.
        if input_kind in ['json', 'pickle']:
            featureCollection = tzwhere.read_tzworld(input_kind, path)
            pgen = tzwhere._feature_collection_polygons(featureCollection)
        elif input_kind == 'csv':
            pgen = tzwhere._read_polygons_from_csv(path)
        else:
            raise ValueError(input_kind)

        # Turn that into an internal mapping
        if self.shapely:
            self._construct_shapely_map(pgen, forceTZ)
        else:
            self._construct_polygon_map(pgen)

        # Convert polygon lists to numpy arrays or (failing that)
        # tuples to save memory.
        for tzname in self.timezoneNamesToPolygons.keys():
            self.timezoneNamesToPolygons[tzname] = \
                WRAP(self.timezoneNamesToPolygons[tzname])

        # And construct lookup shortcuts.
        self._construct_shortcuts()

    def _construct_shapely_map(self, polygon_generator, forceTZ):
        """Turn a (tz, polygon) generator, into our internal shapely mapping."""
        self.timezoneNamesToPolygons = collections.defaultdict(list)
        self.unprepTimezoneNamesToPolygons = collections.defaultdict(list)

        for (tzname, raw_poly) in polygon_generator:
            poly = Polygon(tzwhere._raw_poly_to_poly(raw_poly))
            self.timezoneNamesToPolygons[tzname].append(
                prep(poly))
            if forceTZ:
                self.unprepTimezoneNamesToPolygons[tzname].append(
                    poly)

    def _construct_polygon_map(self, polygon_generator):
        """Turn a (tz, polygon) generator, into our internal mapping."""
        self.timezoneNamesToPolygons = collections.defaultdict(list)
        for (tzname, raw_poly) in polygon_generator:
            self.timezoneNamesToPolygons[tzname].append(
                WRAP(tzwhere._raw_poly_to_poly(raw_poly)))

    def _construct_shortcuts(self):
        ''' Construct our shortcuts for looking up polygons. Much faster
        than using an r-tree '''
        self.timezoneLongitudeShortcuts = {}
        self.timezoneLatitudeShortcuts = {}

        for tzname in self.timezoneNamesToPolygons:
            for polyIndex, poly in enumerate(self.timezoneNamesToPolygons[tzname]):
                if self.shapely:
                    lats = [x[0] for x in poly.context.exterior.coords]
                    lngs = [x[1] for x in poly.context.exterior.coords]
                else:
                    lats = [x[0] for x in poly]
                    lngs = [x[1] for x in poly]
                minLng = (math.floor(min(lngs) / self.SHORTCUT_DEGREES_LONGITUDE)
                          * self.SHORTCUT_DEGREES_LONGITUDE)
                maxLng = (math.floor(max(lngs) / self.SHORTCUT_DEGREES_LONGITUDE)
                          * self.SHORTCUT_DEGREES_LONGITUDE)
                minLat = (math.floor(min(lats) / self.SHORTCUT_DEGREES_LATITUDE)
                          * self.SHORTCUT_DEGREES_LATITUDE)
                maxLat = (math.floor(max(lats) / self.SHORTCUT_DEGREES_LATITUDE)
                          * self.SHORTCUT_DEGREES_LATITUDE)
                degree = minLng
                while degree <= maxLng:
                    if degree not in self.timezoneLongitudeShortcuts:
                        self.timezoneLongitudeShortcuts[degree] = {}

                    if tzname not in self.timezoneLongitudeShortcuts[degree]:
                        self.timezoneLongitudeShortcuts[degree][tzname] = []

                    self.timezoneLongitudeShortcuts[degree][tzname].append(polyIndex)
                    degree = degree + self.SHORTCUT_DEGREES_LONGITUDE

                degree = minLat
                while degree <= maxLat:
                    if degree not in self.timezoneLatitudeShortcuts:
                        self.timezoneLatitudeShortcuts[degree] = {}

                    if tzname not in self.timezoneLatitudeShortcuts[degree]:
                        self.timezoneLatitudeShortcuts[degree][tzname] = []

                    self.timezoneLatitudeShortcuts[degree][tzname].append(polyIndex)
                    degree = degree + self.SHORTCUT_DEGREES_LATITUDE

        # Convert things to tuples to save memory
        for degree in self.timezoneLatitudeShortcuts:
            for tzname in self.timezoneLatitudeShortcuts[degree].keys():
                self.timezoneLatitudeShortcuts[degree][tzname] = \
                    tuple(self.timezoneLatitudeShortcuts[degree][tzname])
        for degree in self.timezoneLongitudeShortcuts.keys():
            for tzname in self.timezoneLongitudeShortcuts[degree].keys():
                self.timezoneLongitudeShortcuts[degree][tzname] = \
                    tuple(self.timezoneLongitudeShortcuts[degree][tzname])

    def tzNameAt(self, latitude, longitude, forceTZ=False):
        '''
        Let's you lookup for a given latitude and longitude the appropriate
        timezone.
        @latitude: latitude
        @longitude: longitude
        @forceTZ: If forceTZ is true and you can't find a valid timezone return
        the closest timezone you can find instead. Only works if the point is
        reasonable close to a timezone already. Consider this a somewhat of a
        'hack'. Introduces potential errors, be warned.

        '''

        if forceTZ and not self.forceTZ:
            raise ValueError('You need to initialize the class with forceTZ='
                             'True if you want to use it later on during the'
                             ' lookup')

        latTzOptions = self.timezoneLatitudeShortcuts[
            (math.floor(latitude / self.SHORTCUT_DEGREES_LATITUDE)
             * self.SHORTCUT_DEGREES_LATITUDE)
        ]
        latSet = set(latTzOptions.keys())
        lngTzOptions = self.timezoneLongitudeShortcuts[
            (math.floor(longitude / self.SHORTCUT_DEGREES_LONGITUDE)
             * self.SHORTCUT_DEGREES_LONGITUDE)
        ]
        lngSet = set(lngTzOptions.keys())
        possibleTimezones = lngSet.intersection(latSet)

        if self.shapely:
            queryPoint = Point(latitude, longitude)

        if possibleTimezones:
            for tzname in possibleTimezones:
                polyIndices = set(latTzOptions[tzname]).intersection(set(lngTzOptions[tzname]))
                for polyIndex in polyIndices:
                    poly = self.timezoneNamesToPolygons[tzname][polyIndex]
                    if self.shapely:
                        if poly.contains_properly(queryPoint):
                            return tzname
                    else:
                        if self._point_inside_polygon(longitude, latitude, poly):
                            return tzname
        distances = []
        if forceTZ:
            if possibleTimezones:
                if len(possibleTimezones) == 1:
                    return possibleTimezones[0]
                else:
                    for tzname in possibleTimezones:
                        polyIndices = set(latTzOptions[tzname]).intersection(set(lngTzOptions[tzname]))
                        for polyIndex in polyIndices:
                            poly = self.unprepTimezoneNamesToPolygons[tzname][polyIndex]
                            d = poly.distance(queryPoint)
                            distances.append((d, tzname))
            if len(distances) > 0:
                return sorted(distances, key=lambda x: x[1])[0][1]

    @staticmethod
    def _point_inside_polygon(x, y, poly):
        n = len(poly)
        inside = False

        p1x, p1y = poly[0][1], poly[0][0]
        for i in range(n + 1):
            p2x, p2y = poly[i % n][1], poly[i % n][0]
            if y > min(p1y, p2y):
                if y <= max(p1y, p2y):
                    if x <= max(p1x, p2x):
                        if p1y != p2y:
                            xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
                        if p1x == p2x or x <= xinters:
                            inside = not inside
            p1x, p1y = p2x, p2y

        return inside

    @staticmethod
    def read_tzworld(input_kind='json', path=None):
        reader = tzwhere.read_json if input_kind == 'json' else tzwhere.read_pickle
        return reader(path)

    @staticmethod
    def read_json(path=None):
        if path is None:
            path = tzwhere.DEFAULT_JSON
        print('Reading json input file: %s' % path)
        with open(path, 'r') as f:
            featureCollection = json.load(f)
        return featureCollection

    @staticmethod
    def read_pickle(path=None):
        if path is None:
            path = tzwhere.DEFAULT_PICKLE
        print('Reading pickle input file: %s' % path)
        with open(path, 'rb') as f:
            featureCollection = pickle.load(f)
        return featureCollection

    @staticmethod
    def write_pickle(featureCollection, path=DEFAULT_PICKLE):
        print('Writing pickle output file: %s' % path)
        with open(path, 'wb') as f:
            pickle.dump(featureCollection, f, pickle.HIGHEST_PROTOCOL)

    @staticmethod
    def _read_polygons_from_csv(path=None):
        if path is None:
            path = tzwhere.DEFAULT_CSV
        print('Reading from CSV input file: %s' % path)
        with open(path, 'r') as f:
            for row in f:
                row = row.split(',')
                yield(row[0], [float(x) for x in row[1:]])

    @staticmethod
    def write_csv(featureCollection, path=DEFAULT_CSV):
        print('Writing csv output file: %s' % path)
        with open(path, 'w') as f:
            writer = csv.writer(f)
            for (tzname, polygon) in tzwhere._feature_collection_polygons(
                    featureCollection):
                writer.writerow([tzname] + polygon)

    @staticmethod
    def _feature_collection_polygons(featureCollection):
        """Turn a feature collection into an iterator over polygons.

        Given a featureCollection of the kind loaded from the json
        input, unpack it to an iterator which produces a series of
        (tzname, polygon) pairs, one for every polygon in the
        featureCollection.  Here tzname is a string and polygon is a
        list of floats.

        """
        for feature in featureCollection['features']:
            tzname = feature['properties']['TZID']
            if feature['geometry']['type'] == 'Polygon':
                polys = feature['geometry']['coordinates']
                for poly in polys:
                    yield (tzname, poly)

    @staticmethod
    def _raw_poly_to_poly(raw_poly):
        # WPS84 coordinates are [long, lat], while many conventions
        # are [lat, long]. Our data is in WPS84. Convert to an
        # explicit format which geolib likes.
        assert len(raw_poly) % 2 == 0
        poly = []
        while raw_poly:
            lat = raw_poly.pop()
            lng = raw_poly.pop()
            poly.append((lat, lng))
        return poly

HELP = """tzwhere.py - time zone computation from latitude/longitude.

Usage:
  tzwhere.py [options] write_pickle [<input_path>] [<output_path>]
  tzwhere.py [options] write_csv [<input_path>] [<output_path>]

Modes:

  write_pickle - write out a pickle file of a feature collection;
                 <input_path> is as with test.  <output_path> is also
                 optional, and defaults to {default_pickle}.
                 N.b.: don't do this with -k csv

  write_csv - write out a CSV file.  Each line contains the time zone
              name and a list of floats for a single polygon in that
              time zone.  <input_path> is as with test.  <output_path>
              is also optional, and defaults to {default_csv}.
              N.b.: don't do this with -k csv

Options:
  -k <kind>, --kind=<kind>  Input kind. Should be json or csv or pickle
                            [default: json].
  -m, --memory              Report on memory usage before, during, and
                            after operation.
  -h, --help                Show this help.

""".format(**{
    'default_json': tzwhere.DEFAULT_JSON,
    'default_pickle': tzwhere.DEFAULT_PICKLE,
    'default_csv': tzwhere.DEFAULT_CSV
})

report_memory = False

def main():
    try:
        import docopt
    except ImportError:
        print("Please install the docopt package to use tzwhere.py as a script.")
        import sys
        sys.exit(1)

    args = docopt.docopt(HELP)

    global report_memory
    report_memory = args['--memory']

    if args['write_pickle']:
        if args['--kind'] not in ('json', 'pickle'):
            print("Can't write pickle output from CSV input")
            return
        if args['<output_path>'] is None:
            args['<output_path>'] = tzwhere.DEFAULT_PICKLE
        write_pickle(args['--kind'], args['<input_path>'],
                     args['<output_path>'])
    elif args['write_csv']:
        if args['--kind'] not in ('json', 'pickle'):
            print("Can't write CSV output from CSV input")
            return
        if args['<output_path>'] is None:
            args['<output_path>'] = tzwhere.DEFAULT_CSV
        write_csv(args['--kind'], args['<input_path>'],
                  args['<output_path>'])

def write_pickle(input_kind, input_path, output_path):
    memuse()
    features = tzwhere.read_tzworld(input_kind, input_path)
    memuse()
    tzwhere.write_pickle(features, output_path)
    memuse()

def write_csv(input_kind, input_path, output_path):
    memuse()
    features = tzwhere.read_tzworld(input_kind, input_path)
    memuse()
    tzwhere.write_csv(features, output_path)
    memuse()

def memuse():
    global report_memory
    if not report_memory:
        return

    import subprocess
    import resource

    import sys
    if sys.version_info >= (3, 0):
        sep = '\\n'
    else:
        sep = '\n'

    free = int(str(subprocess.check_output(['free', '-m']
                                           )).split(sep)[2].split()[-1])
    maxrss = int(resource.getrusage(
        resource.RUSAGE_SELF).ru_maxrss / 1000)
    print()
    print('Memory:')
    print('{0:6d} MB free'.format(free))
    print('{0:6d} MB maxrss'.format(maxrss))
    print()

if __name__ == "__main__":
    main()
<commit_msg>Fixed return forceTZ<commit_after>#!/usr/bin/env python

"""tzwhere.py - time zone computation from latitude/longitude.

Ordinarily this is loaded as a module and instances of the tzwhere
class are instantiated and queried directly, but the module can be run
as a script too (this requires the docopt package to be installed).
Run it with the -h option to see usage.

"""

import csv
import collections
import datetime
try:
    import json
except ImportError:
    import simplejson as json
import math
import os
import pickle

# We can speed up things by about 200 times if we use shapely
try:
    from shapely.geometry import Polygon, Point
    from shapely.prepared import prep
    SHAPELY_IMPORT = True
except ImportError:
    SHAPELY_IMPORT = False

# We can save about 222MB of RAM by turning our polygon lists into
# numpy arrays rather than tuples, if numpy is installed.
try:
    import numpy
    WRAP = numpy.array
except ImportError:
    WRAP = tuple

class tzwhere(object):

    SHORTCUT_DEGREES_LATITUDE = 1
    SHORTCUT_DEGREES_LONGITUDE = 1
    # By default, use the data file in our package directory
    DEFAULT_JSON = os.path.join(os.path.dirname(__file__),
                                'tz_world_compact.json')
    DEFAULT_PICKLE = os.path.join(os.path.dirname(__file__),
                                  'tz_world.pickle')
    DEFAULT_CSV = os.path.join(os.path.dirname(__file__),
                               'tz_world.csv')

    def __init__(self, input_kind='json', path=None,
                 shapely=False, forceTZ=False):
        '''
        Initializes the tzwhere class.
        @input_kind: Which filetype you want to read from
        @path: Where you want to read the input file from
        @shapely: Whether you want to use shapely to represent the geometry.
        Lookups become much faster at the cost of a slower initialization
        @forceTZ: If you want to force the lookup method to a return a
        timezone even if the point you are looking up is slightly outside it's
        bounds, you need to specify this during initialization arleady

        '''

        # Make sure people have an idea why things are not working later down
        # the line and rather tell them now
        if (shapely or forceTZ) and not SHAPELY_IMPORT:
            raise ValueError('You need to have shapley installed for this '
                             'feature, but we can\'t find it')
        if not shapely and forceTZ:
            raise ValueError('The lookup \'hack\' depends on shapely. Try using'
                             ' shapely=True when initializing the class')

        self.forceTZ = forceTZ
        self.shapely = shapely and SHAPELY_IMPORT

        # Construct appropriate generator for (tz, polygon) pairs.
        if input_kind in ['json', 'pickle']:
            featureCollection = tzwhere.read_tzworld(input_kind, path)
            pgen = tzwhere._feature_collection_polygons(featureCollection)
        elif input_kind == 'csv':
            pgen = tzwhere._read_polygons_from_csv(path)
        else:
            raise ValueError(input_kind)

        # Turn that into an internal mapping
        if self.shapely:
            self._construct_shapely_map(pgen, forceTZ)
        else:
            self._construct_polygon_map(pgen)

        # Convert polygon lists to numpy arrays or (failing that)
        # tuples to save memory.
        for tzname in self.timezoneNamesToPolygons.keys():
            self.timezoneNamesToPolygons[tzname] = \
                WRAP(self.timezoneNamesToPolygons[tzname])

        # And construct lookup shortcuts.
        self._construct_shortcuts()

    def _construct_shapely_map(self, polygon_generator, forceTZ):
        """Turn a (tz, polygon) generator, into our internal shapely mapping."""
        self.timezoneNamesToPolygons = collections.defaultdict(list)
        self.unprepTimezoneNamesToPolygons = collections.defaultdict(list)

        for (tzname, raw_poly) in polygon_generator:
            poly = Polygon(tzwhere._raw_poly_to_poly(raw_poly))
            self.timezoneNamesToPolygons[tzname].append(
                prep(poly))
            if forceTZ:
                self.unprepTimezoneNamesToPolygons[tzname].append(
                    poly)

    def _construct_polygon_map(self, polygon_generator):
        """Turn a (tz, polygon) generator, into our internal mapping."""
        self.timezoneNamesToPolygons = collections.defaultdict(list)
        for (tzname, raw_poly) in polygon_generator:
            self.timezoneNamesToPolygons[tzname].append(
                WRAP(tzwhere._raw_poly_to_poly(raw_poly)))

    def _construct_shortcuts(self):
        ''' Construct our shortcuts for looking up polygons. Much faster
        than using an r-tree '''
        self.timezoneLongitudeShortcuts = {}
        self.timezoneLatitudeShortcuts = {}

        for tzname in self.timezoneNamesToPolygons:
            for polyIndex, poly in enumerate(self.timezoneNamesToPolygons[tzname]):
                if self.shapely:
                    lats = [x[0] for x in poly.context.exterior.coords]
                    lngs = [x[1] for x in poly.context.exterior.coords]
                else:
                    lats = [x[0] for x in poly]
                    lngs = [x[1] for x in poly]
                minLng = (math.floor(min(lngs) / self.SHORTCUT_DEGREES_LONGITUDE)
                          * self.SHORTCUT_DEGREES_LONGITUDE)
                maxLng = (math.floor(max(lngs) / self.SHORTCUT_DEGREES_LONGITUDE)
                          * self.SHORTCUT_DEGREES_LONGITUDE)
                minLat = (math.floor(min(lats) / self.SHORTCUT_DEGREES_LATITUDE)
                          * self.SHORTCUT_DEGREES_LATITUDE)
                maxLat = (math.floor(max(lats) / self.SHORTCUT_DEGREES_LATITUDE)
                          * self.SHORTCUT_DEGREES_LATITUDE)
                degree = minLng
                while degree <= maxLng:
                    if degree not in self.timezoneLongitudeShortcuts:
                        self.timezoneLongitudeShortcuts[degree] = {}

                    if tzname not in self.timezoneLongitudeShortcuts[degree]:
                        self.timezoneLongitudeShortcuts[degree][tzname] = []

                    self.timezoneLongitudeShortcuts[degree][tzname].append(polyIndex)
                    degree = degree + self.SHORTCUT_DEGREES_LONGITUDE

                degree = minLat
                while degree <= maxLat:
                    if degree not in self.timezoneLatitudeShortcuts:
                        self.timezoneLatitudeShortcuts[degree] = {}

                    if tzname not in self.timezoneLatitudeShortcuts[degree]:
                        self.timezoneLatitudeShortcuts[degree][tzname] = []

                    self.timezoneLatitudeShortcuts[degree][tzname].append(polyIndex)
                    degree = degree + self.SHORTCUT_DEGREES_LATITUDE

        # Convert things to tuples to save memory
        for degree in self.timezoneLatitudeShortcuts:
            for tzname in self.timezoneLatitudeShortcuts[degree].keys():
                self.timezoneLatitudeShortcuts[degree][tzname] = \
                    tuple(self.timezoneLatitudeShortcuts[degree][tzname])
        for degree in self.timezoneLongitudeShortcuts.keys():
            for tzname in self.timezoneLongitudeShortcuts[degree].keys():
                self.timezoneLongitudeShortcuts[degree][tzname] = \
                    tuple(self.timezoneLongitudeShortcuts[degree][tzname])

    def tzNameAt(self, latitude, longitude, forceTZ=False):
        '''
        Let's you lookup for a given latitude and longitude the appropriate
        timezone.
        @latitude: latitude
        @longitude: longitude
        @forceTZ: If forceTZ is true and you can't find a valid timezone return
        the closest timezone you can find instead. Only works if the point is
        reasonable close to a timezone already. Consider this a somewhat of a
        'hack'. Introduces potential errors, be warned.

        '''

        if forceTZ and not self.forceTZ:
            raise ValueError('You need to initialize the class with forceTZ='
                             'True if you want to use it later on during the'
                             ' lookup')

        latTzOptions = self.timezoneLatitudeShortcuts[
            (math.floor(latitude / self.SHORTCUT_DEGREES_LATITUDE)
             * self.SHORTCUT_DEGREES_LATITUDE)
        ]
        latSet = set(latTzOptions.keys())
        lngTzOptions = self.timezoneLongitudeShortcuts[
            (math.floor(longitude / self.SHORTCUT_DEGREES_LONGITUDE)
             * self.SHORTCUT_DEGREES_LONGITUDE)
        ]
        lngSet = set(lngTzOptions.keys())
        possibleTimezones = lngSet.intersection(latSet)

        if self.shapely:
            queryPoint = Point(latitude, longitude)

        if possibleTimezones:
            for tzname in possibleTimezones:
                polyIndices = set(latTzOptions[tzname]).intersection(set(lngTzOptions[tzname]))
                for polyIndex in polyIndices:
                    poly = self.timezoneNamesToPolygons[tzname][polyIndex]
                    if self.shapely:
                        if poly.contains_properly(queryPoint):
                            return tzname
                    else:
                        if self._point_inside_polygon(longitude, latitude, poly):
                            return tzname
        distances = []
        if forceTZ:
            if possibleTimezones:
                if len(possibleTimezones) == 1:
                    return possibleTimezones.pop()
                else:
                    for tzname in possibleTimezones:
                        polyIndices = set(latTzOptions[tzname]).intersection(set(lngTzOptions[tzname]))
                        for polyIndex in polyIndices:
                            poly = self.unprepTimezoneNamesToPolygons[tzname][polyIndex]
                            d = poly.distance(queryPoint)
                            distances.append((d, tzname))
            if len(distances) > 0:
                return sorted(distances, key=lambda x: x[1])[0][1]

    @staticmethod
    def _point_inside_polygon(x, y, poly):
        n = len(poly)
        inside = False

        p1x, p1y = poly[0][1], poly[0][0]
        for i in range(n + 1):
            p2x, p2y = poly[i % n][1], poly[i % n][0]
            if y > min(p1y, p2y):
                if y <= max(p1y, p2y):
                    if x <= max(p1x, p2x):
                        if p1y != p2y:
                            xinters = (y - p1y) * (p2x - p1x) / (p2y - p1y) + p1x
                        if p1x == p2x or x <= xinters:
                            inside = not inside
            p1x, p1y = p2x, p2y

        return inside

    @staticmethod
    def read_tzworld(input_kind='json', path=None):
        reader = tzwhere.read_json if input_kind == 'json' else tzwhere.read_pickle
        return reader(path)

    @staticmethod
    def read_json(path=None):
        if path is None:
            path = tzwhere.DEFAULT_JSON
        print('Reading json input file: %s' % path)
        with open(path, 'r') as f:
            featureCollection = json.load(f)
        return featureCollection

    @staticmethod
    def read_pickle(path=None):
        if path is None:
            path = tzwhere.DEFAULT_PICKLE
        print('Reading pickle input file: %s' % path)
        with open(path, 'rb') as f:
            featureCollection = pickle.load(f)
        return featureCollection

    @staticmethod
    def write_pickle(featureCollection, path=DEFAULT_PICKLE):
        print('Writing pickle output file: %s' % path)
        with open(path, 'wb') as f:
            pickle.dump(featureCollection, f, pickle.HIGHEST_PROTOCOL)

    @staticmethod
    def _read_polygons_from_csv(path=None):
        if path is None:
            path = tzwhere.DEFAULT_CSV
        print('Reading from CSV input file: %s' % path)
        with open(path, 'r') as f:
            for row in f:
                row = row.split(',')
                yield(row[0], [float(x) for x in row[1:]])

    @staticmethod
    def write_csv(featureCollection, path=DEFAULT_CSV):
        print('Writing csv output file: %s' % path)
        with open(path, 'w') as f:
            writer = csv.writer(f)
            for (tzname, polygon) in tzwhere._feature_collection_polygons(
                    featureCollection):
                writer.writerow([tzname] + polygon)

    @staticmethod
    def _feature_collection_polygons(featureCollection):
        """Turn a feature collection into an iterator over polygons.

        Given a featureCollection of the kind loaded from the json
        input, unpack it to an iterator which produces a series of
        (tzname, polygon) pairs, one for every polygon in the
        featureCollection.  Here tzname is a string and polygon is a
        list of floats.

        """
        for feature in featureCollection['features']:
            tzname = feature['properties']['TZID']
            if feature['geometry']['type'] == 'Polygon':
                polys = feature['geometry']['coordinates']
                for poly in polys:
                    yield (tzname, poly)

    @staticmethod
    def _raw_poly_to_poly(raw_poly):
        # WPS84 coordinates are [long, lat], while many conventions
        # are [lat, long]. Our data is in WPS84. Convert to an
        # explicit format which geolib likes.
        assert len(raw_poly) % 2 == 0
        poly = []
        while raw_poly:
            lat = raw_poly.pop()
            lng = raw_poly.pop()
            poly.append((lat, lng))
        return poly

HELP = """tzwhere.py - time zone computation from latitude/longitude.

Usage:
  tzwhere.py [options] write_pickle [<input_path>] [<output_path>]
  tzwhere.py [options] write_csv [<input_path>] [<output_path>]

Modes:

  write_pickle - write out a pickle file of a feature collection;
                 <input_path> is as with test.  <output_path> is also
                 optional, and defaults to {default_pickle}.
                 N.b.: don't do this with -k csv

  write_csv - write out a CSV file.  Each line contains the time zone
              name and a list of floats for a single polygon in that
              time zone.  <input_path> is as with test.  <output_path>
              is also optional, and defaults to {default_csv}.
              N.b.: don't do this with -k csv

Options:
  -k <kind>, --kind=<kind>  Input kind. Should be json or csv or pickle
                            [default: json].
  -m, --memory              Report on memory usage before, during, and
                            after operation.
  -h, --help                Show this help.

""".format(**{
    'default_json': tzwhere.DEFAULT_JSON,
    'default_pickle': tzwhere.DEFAULT_PICKLE,
    'default_csv': tzwhere.DEFAULT_CSV
})

report_memory = False

def main():
    try:
        import docopt
    except ImportError:
        print("Please install the docopt package to use tzwhere.py as a script.")
        import sys
        sys.exit(1)

    args = docopt.docopt(HELP)

    global report_memory
    report_memory = args['--memory']

    if args['write_pickle']:
        if args['--kind'] not in ('json', 'pickle'):
            print("Can't write pickle output from CSV input")
            return
        if args['<output_path>'] is None:
            args['<output_path>'] = tzwhere.DEFAULT_PICKLE
        write_pickle(args['--kind'], args['<input_path>'],
                     args['<output_path>'])
    elif args['write_csv']:
        if args['--kind'] not in ('json', 'pickle'):
            print("Can't write CSV output from CSV input")
            return
        if args['<output_path>'] is None:
            args['<output_path>'] = tzwhere.DEFAULT_CSV
        write_csv(args['--kind'], args['<input_path>'],
                  args['<output_path>'])

def write_pickle(input_kind, input_path, output_path):
    memuse()
    features = tzwhere.read_tzworld(input_kind, input_path)
    memuse()
    tzwhere.write_pickle(features, output_path)
    memuse()

def write_csv(input_kind, input_path, output_path):
    memuse()
    features = tzwhere.read_tzworld(input_kind, input_path)
    memuse()
    tzwhere.write_csv(features, output_path)
    memuse()

def memuse():
    global report_memory
    if not report_memory:
        return

    import subprocess
    import resource

    import sys
    if sys.version_info >= (3, 0):
        sep = '\\n'
    else:
        sep = '\n'

    free = int(str(subprocess.check_output(['free', '-m']
                                           )).split(sep)[2].split()[-1])
    maxrss = int(resource.getrusage(
        resource.RUSAGE_SELF).ru_maxrss / 1000)
    print()
    print('Memory:')
    print('{0:6d} MB free'.format(free))
    print('{0:6d} MB maxrss'.format(maxrss))
    print()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#Importing libraries
import discord
from discord.ext import commands
from sys import argv

class Lockdown:
    """
    Channel lockdown commands.
    """
    def __init__(self, bot):
        self.bot = bot
        print('Addon "{}" loaded'.format(self.__class__.__name__))

    @commands.has_permissions(manage_messages=True)
    @commands.command(pass_context=True, name="lockdown")
    async def lockdown(self, ctx):
       """Lock message sending in the channel. Staff only."""
       try:
            everyone_role = discord.utils.get(ctx.message.server.roles, name="@everyone")
            overwrites = ctx.message.channel.overwrites_for(everyone_role)
            if overwrites.send_messages == False:
                await self.bot.say("🔒 Channel is already locked down. Use `.unlock` to unlock.")
                return
            overwrites.send_messages = False
            await self.bot.edit_channel_permissions(ctx.message.channel, everyone_role, overwrites)
            await self.bot.say("🔒 Channel locked down. Only staff members may speak. Do not bring the topic to other channels or risk disciplinary actions.")
            msg = "🔒 **Lockdown**: {0} by {1} | {2}#{3}".format(ctx.message.channel.mention, ctx.message.author.mention, ctx.message.author.name, ctx.message.author.discriminator)
            await self.bot.send_message(discord.utils.get(ctx.message.server.channels, name="mod-logs"), msg)
       except discord.errors.Forbidden:
            await self.bot.say("💢 I don't have permission to do this.")

    @commands.has_permissions(manage_messages=True)
    @commands.command(pass_context=True, name="unlock")
    async def unlock(self, ctx):
       """Unock message sending in the channel. Staff only."""
       try:
            everyone_role = discord.utils.get(ctx.message.server.roles, name="@everyone")
            overwrites = ctx.message.channel.overwrites_for(everyone_role)
            if overwrites.send_messages == None:
                await self.bot.say("🔓 Channel is already unlocked.")
                return
            overwrites.send_messages = None
            await self.bot.edit_channel_permissions(ctx.message.channel, everyone_role, overwrites)
            await self.bot.say("🔓 Channel unlocked.")
            msg = "🔓 **Unlock**: {0} by {1} | {2}#{3}".format(ctx.message.channel.mention, ctx.message.author.mention, ctx.message.author.name, ctx.message.author.discriminator)
            await self.bot.send_message(discord.utils.get(ctx.message.server.channels, name="mod-logs"), msg)
       except discord.errors.Forbidden:
            await self.bot.say("💢 I don't have permission to do this.")

def setup(bot):
    bot.add_cog(Lockdown(bot))
<commit_msg>.softlock<commit_after>#Importing libraries
import discord
from discord.ext import commands
from sys import argv

class Lockdown:
    """
    Channel lockdown commands.
    """
    def __init__(self, bot):
        self.bot = bot
        print('Addon "{}" loaded'.format(self.__class__.__name__))

    @commands.has_permissions(manage_messages=True)
    @commands.command(pass_context=True, name="lockdown")
    async def lockdown(self, ctx):
       """Lock message sending in the channel. Staff only."""
       try:
            everyone_role = discord.utils.get(ctx.message.server.roles, name="@everyone")
            overwrites = ctx.message.channel.overwrites_for(everyone_role)
            if overwrites.send_messages == False:
                await self.bot.say("🔒 Channel is already locked down. Use `.unlock` to unlock.")
                return
            overwrites.send_messages = False
            await self.bot.edit_channel_permissions(ctx.message.channel, everyone_role, overwrites)
            await self.bot.say("🔒 Channel locked down. Only staff members may speak. Do not bring the topic to other channels or risk disciplinary actions.")
            msg = "🔒 **Lockdown**: {0} by {1} | {2}#{3}".format(ctx.message.channel.mention, ctx.message.author.mention, ctx.message.author.name, ctx.message.author.discriminator)
            await self.bot.send_message(discord.utils.get(ctx.message.server.channels, name="mod-logs"), msg)
       except discord.errors.Forbidden:
            await self.bot.say("💢 I don't have permission to do this.")

    @commands.has_permissions(manage_messages=True)
    @commands.command(pass_context=True, name="softlock")
    async def softlock(self, ctx):
       """Lock message sending in the channel, without the "disciplinary action" note. Staff only."""
       try:
            everyone_role = discord.utils.get(ctx.message.server.roles, name="@everyone")
            overwrites = ctx.message.channel.overwrites_for(everyone_role)
            if overwrites.send_messages == False:
                await self.bot.say("🔒 Channel is already locked down. Use `.unlock` to unlock.")
                return
            overwrites.send_messages = False
            await self.bot.edit_channel_permissions(ctx.message.channel, everyone_role, overwrites)
            await self.bot.say("🔒 Channel locked.")
            msg = "🔒 **Lockdown**: {0} by {1} | {2}#{3}".format(ctx.message.channel.mention, ctx.message.author.mention, ctx.message.author.name, ctx.message.author.discriminator)
            await self.bot.send_message(discord.utils.get(ctx.message.server.channels, name="mod-logs"), msg)
       except discord.errors.Forbidden:
            await self.bot.say("💢 I don't have permission to do this.")

    @commands.has_permissions(manage_messages=True)
    @commands.command(pass_context=True, name="unlock")
    async def unlock(self, ctx):
       """Unock message sending in the channel. Staff only."""
       try:
            everyone_role = discord.utils.get(ctx.message.server.roles, name="@everyone")
            overwrites = ctx.message.channel.overwrites_for(everyone_role)
            if overwrites.send_messages == None:
                await self.bot.say("🔓 Channel is already unlocked.")
                return
            overwrites.send_messages = None
            await self.bot.edit_channel_permissions(ctx.message.channel, everyone_role, overwrites)
            await self.bot.say("🔓 Channel unlocked.")
            msg = "🔓 **Unlock**: {0} by {1} | {2}#{3}".format(ctx.message.channel.mention, ctx.message.author.mention, ctx.message.author.name, ctx.message.author.discriminator)
            await self.bot.send_message(discord.utils.get(ctx.message.server.channels, name="mod-logs"), msg)
       except discord.errors.Forbidden:
            await self.bot.say("💢 I don't have permission to do this.")

def setup(bot):
    bot.add_cog(Lockdown(bot))
<|endoftext|>