<commit_before>#!/usr/bin/env python
import redis
import json
import argparse
import pprint
import mturk_vision
import base64
import uuid
import pickle
from hadoop_parse import scrape_hadoop_jobs

class UnauthorizedException(Exception):
    """User is not authorized to make this call"""

class NotFoundException(Exception):
    """Task was not found"""

class Jobs(object):

    def __init__(self, host, port, db, annotation_redis_host, annotation_redis_port):
        self.args = (host, port, db, annotation_redis_host, annotation_redis_port)
        self.redis_host = host
        self.redis_port = port
        self.db = redis.StrictRedis(host=host, port=port, db=db)
        self.annotation_cache = {}
        self._owner_prefix = 'owner:'
        self._task_prefix = 'task:'
        self._lock_prefix = 'lock:'
        self.annotation_redis_host = annotation_redis_host
        self.annotation_redis_port = annotation_redis_port
        self.hadoop_completed_jobs_cache = set()

    def __reduce__(self):
        return (Jobs, self.args)

    def add_task(self, type, owner, params, secret_params):
        task = base64.urlsafe_b64encode(uuid.uuid4().bytes)[:-2]
        data = {'owner': owner, '_params': json.dumps(secret_params),
                'params': json.dumps(params), 'type': type}
        if not self.db.set(self._lock_prefix + task, '', nx=True):
            raise UnauthorizedException
        # TODO: Do these atomically
        self.db.hmset(self._task_prefix + task, data)
        self.db.sadd(self._owner_prefix + owner, task)
        return task

    def _check_owner(self, task, owner):
        if self.db.hget(self._task_prefix + task, 'owner') != owner:
            raise UnauthorizedException

    def _get_task_type(self, task):
        out = self.db.hgetall(self._task_prefix + task)
        out = self.db.hget(self._task_prefix + task, 'type')
        if out is None:
            raise NotFoundException
        return out

    def _check_type(self, task, type):
        if self._get_task_type(task) != type:
            raise NotFoundException

    def _exists(self, task):
        if not self.db.exists(self._lock_prefix + task):
            raise NotFoundException

    def get_task(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        out = self.db.hgetall(self._task_prefix + task)
        out = {k: v for k, v in out.items() if not k.startswith('_')}
        return out

    def get_task_secret(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        return json.loads(self.db.hget(self._task_prefix + task, '_params'))

    def delete_task(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        task_type = self._get_task_type(task)
        if task_type == 'annotation':
            manager = self.get_annotation_manager(task)
        # TODO: Do these atomically
        self.db.delete(self._task_prefix + task, self._lock_prefix + task)
        self.db.srem(self._owner_prefix + owner, task)
        if task_type == 'annotation':
            manager.destroy()  # TODO: MTurk specific
            try:
                del self.annotation_cache[task]
            except KeyError:
                pass
        # TODO: For Hadoop jobs kill the task if it is running
        # TODO: For worker/crawl/model jobs kill the worker process or send it a signal

    def update_job(self, row, columns):
        self.db.hmset(self._task_prefix + row, columns)

    def update_hadoop_jobs(self, hadoop_jobtracker):
        for row, columns in scrape_hadoop_jobs(hadoop_jobtracker, self.hadoop_completed_jobs_cache).items():
            # NOTE: We do this at this point as a job may not exist but is finished completed/failed in hadoop
            if columns.get('status', '') in ('completed', 'failed'):
                self.hadoop_completed_jobs_cache.add(row)
            try:
                self._exists(row)
                self._check_type(row, 'process')
            except NotFoundException:
                continue
            # TODO: Need to do this atomically with the exists check
            self.update_job(row, columns)

    def get_tasks(self, owner):
        outs = {}
        for job_key in self.db.smembers(self._owner_prefix + owner):
            # TODO: Error check if something gets removed while we are accumulating
            task = self._task_prefix + job_key
            if self.db.hget(task, 'owner') == owner:
                out = self.db.hgetall(task)
                out = {k: v for k, v in out.items() if not k.startswith('_')}
                outs[task.split(':', 1)[1]] = out
        return outs

    def get_annotation_manager(self, task, sync=False):
        self._exists(task)
        self._check_type(task, 'annotation')
        try:
            return self.annotation_cache[task]
        except KeyError:
            data = self.db.hgetall(self._task_prefix + task)
            p = json.loads(data['params'])
            ps = json.loads(data['_params'])
            p['sync'] = sync
            p['secret'] = str(ps['secret'])
            p['redis_address'] = self.annotation_redis_host
            p['redis_port'] = int(self.annotation_redis_port)
            p['task_key'] = task
            self.annotation_cache[task] = mturk_vision.manager(data=str(ps['data']), **p)
            return self.annotation_cache[task]

    def get_annotation_manager_check(self, task, owner):
        self._exists(task)
        self._check_type(task, 'annotation')
        self._check_owner(task, owner)
        return self.get_annotation_manager(task)

    def add_work(self, db, func, method_args, method_kwargs, queue):
        self.db.lpush('queue:' + queue, pickle.dumps({'db': db, 'func': func, 'method_args': method_args, 'method_kwargs': method_kwargs}, -1))

    def get_work(self, queues, timeout=0):
        return self.db.brpop(['queue:' + x for x in queues], timeout=timeout)

def main():

    def _get_all_tasks(jobs):
        outs = []
        for job_key in jobs.db.keys('task:*'):
            out = jobs.db.hgetall(job_key)
            outs.append(out)
        return outs

    def _info(args, jobs):
        pprint.pprint(_get_all_tasks(jobs))

    def _destroy(args, jobs):
        jobs.db.flushall()

    def job_worker(db, func, method_args, method_kwargs):
        getattr(db, func)(*method_args, **method_kwargs)

    def _work(args, jobs):
        job_worker(**jobs.get_work(args.queues))

    parser = argparse.ArgumentParser(description='Picarus job operations')
    parser.add_argument('--redis_host', help='Redis Host', default='localhost')
    parser.add_argument('--redis_port', type=int, help='Redis Port', default=3)
    parser.add_argument('--annotations_redis_host', help='Annotations Host', default='localhost')
    parser.add_argument('--annotations_redis_port', type=int, help='Annotations Port', default=6380)
    subparsers = parser.add_subparsers(help='Commands')

    subparser = subparsers.add_parser('info', help='Display info about jobs')
    subparser.set_defaults(func=_info)

    subparser = subparsers.add_parser('destroy', help='Delete everything in the jobs DB')
    subparser.set_defaults(func=_destroy)

    subparser = subparsers.add_parser('work', help='Do background work')
    parser.add_argument('queues', nargs='+', help='Queues to do work on')
    subparser.set_defaults(func=_work)

    args = parser.parse_args()
    jobs = Jobs(args.redis_host, args.redis_port, 3, args.annotations_redis_host, args.annotations_redis_port)
    args.func(args, jobs)

if __name__ == '__main__':
    main()
<commit_msg>Updated jobs to unpickle<commit_after>#!/usr/bin/env python
import redis
import json
import argparse
import pprint
import mturk_vision
import base64
import uuid
import pickle
from hadoop_parse import scrape_hadoop_jobs

class UnauthorizedException(Exception):
    """User is not authorized to make this call"""

class NotFoundException(Exception):
    """Task was not found"""

class Jobs(object):

    def __init__(self, host, port, db, annotation_redis_host, annotation_redis_port):
        self.args = (host, port, db, annotation_redis_host, annotation_redis_port)
        self.redis_host = host
        self.redis_port = port
        self.db = redis.StrictRedis(host=host, port=port, db=db)
        self.annotation_cache = {}
        self._owner_prefix = 'owner:'
        self._task_prefix = 'task:'
        self._lock_prefix = 'lock:'
        self.annotation_redis_host = annotation_redis_host
        self.annotation_redis_port = annotation_redis_port
        self.hadoop_completed_jobs_cache = set()

    def __reduce__(self):
        return (Jobs, self.args)

    def add_task(self, type, owner, params, secret_params):
        task = base64.urlsafe_b64encode(uuid.uuid4().bytes)[:-2]
        data = {'owner': owner, '_params': json.dumps(secret_params),
                'params': json.dumps(params), 'type': type}
        if not self.db.set(self._lock_prefix + task, '', nx=True):
            raise UnauthorizedException
        # TODO: Do these atomically
        self.db.hmset(self._task_prefix + task, data)
        self.db.sadd(self._owner_prefix + owner, task)
        return task

    def _check_owner(self, task, owner):
        if self.db.hget(self._task_prefix + task, 'owner') != owner:
            raise UnauthorizedException

    def _get_task_type(self, task):
        out = self.db.hgetall(self._task_prefix + task)
        out = self.db.hget(self._task_prefix + task, 'type')
        if out is None:
            raise NotFoundException
        return out

    def _check_type(self, task, type):
        if self._get_task_type(task) != type:
            raise NotFoundException

    def _exists(self, task):
        if not self.db.exists(self._lock_prefix + task):
            raise NotFoundException

    def get_task(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        out = self.db.hgetall(self._task_prefix + task)
        out = {k: v for k, v in out.items() if not k.startswith('_')}
        return out

    def get_task_secret(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        return json.loads(self.db.hget(self._task_prefix + task, '_params'))

    def delete_task(self, task, owner):
        self._exists(task)
        self._check_owner(task, owner)
        task_type = self._get_task_type(task)
        if task_type == 'annotation':
            manager = self.get_annotation_manager(task)
        # TODO: Do these atomically
        self.db.delete(self._task_prefix + task, self._lock_prefix + task)
        self.db.srem(self._owner_prefix + owner, task)
        if task_type == 'annotation':
            manager.destroy()  # TODO: MTurk specific
            try:
                del self.annotation_cache[task]
            except KeyError:
                pass
        # TODO: For Hadoop jobs kill the task if it is running
        # TODO: For worker/crawl/model jobs kill the worker process or send it a signal

    def update_job(self, row, columns):
        self.db.hmset(self._task_prefix + row, columns)

    def update_hadoop_jobs(self, hadoop_jobtracker):
        for row, columns in scrape_hadoop_jobs(hadoop_jobtracker, self.hadoop_completed_jobs_cache).items():
            # NOTE: We do this at this point as a job may not exist but is finished completed/failed in hadoop
            if columns.get('status', '') in ('completed', 'failed'):
                self.hadoop_completed_jobs_cache.add(row)
            try:
                self._exists(row)
                self._check_type(row, 'process')
            except NotFoundException:
                continue
            # TODO: Need to do this atomically with the exists check
            self.update_job(row, columns)

    def get_tasks(self, owner):
        outs = {}
        for job_key in self.db.smembers(self._owner_prefix + owner):
            # TODO: Error check if something gets removed while we are accumulating
            task = self._task_prefix + job_key
            if self.db.hget(task, 'owner') == owner:
                out = self.db.hgetall(task)
                out = {k: v for k, v in out.items() if not k.startswith('_')}
                outs[task.split(':', 1)[1]] = out
        return outs

    def get_annotation_manager(self, task, sync=False):
        self._exists(task)
        self._check_type(task, 'annotation')
        try:
            return self.annotation_cache[task]
        except KeyError:
            data = self.db.hgetall(self._task_prefix + task)
            p = json.loads(data['params'])
            ps = json.loads(data['_params'])
            p['sync'] = sync
            p['secret'] = str(ps['secret'])
            p['redis_address'] = self.annotation_redis_host
            p['redis_port'] = int(self.annotation_redis_port)
            p['task_key'] = task
            self.annotation_cache[task] = mturk_vision.manager(data=str(ps['data']), **p)
            return self.annotation_cache[task]

    def get_annotation_manager_check(self, task, owner):
        self._exists(task)
        self._check_type(task, 'annotation')
        self._check_owner(task, owner)
        return self.get_annotation_manager(task)

    def add_work(self, db, func, method_args, method_kwargs, queue):
        self.db.lpush('queue:' + queue, pickle.dumps({'db': db, 'func': func, 'method_args': method_args, 'method_kwargs': method_kwargs}, -1))

    def get_work(self, queues, timeout=0):
        return pickle.loads(self.db.brpop(['queue:' + x for x in queues], timeout=timeout))

def main():

    def _get_all_tasks(jobs):
        outs = []
        for job_key in jobs.db.keys('task:*'):
            out = jobs.db.hgetall(job_key)
            outs.append(out)
        return outs

    def _info(args, jobs):
        pprint.pprint(_get_all_tasks(jobs))

    def _destroy(args, jobs):
        jobs.db.flushall()

    def job_worker(db, func, method_args, method_kwargs):
        getattr(db, func)(*method_args, **method_kwargs)

    def _work(args, jobs):
        job_worker(**jobs.get_work(args.queues))

    parser = argparse.ArgumentParser(description='Picarus job operations')
    parser.add_argument('--redis_host', help='Redis Host', default='localhost')
    parser.add_argument('--redis_port', type=int, help='Redis Port', default=3)
    parser.add_argument('--annotations_redis_host', help='Annotations Host', default='localhost')
    parser.add_argument('--annotations_redis_port', type=int, help='Annotations Port', default=6380)
    subparsers = parser.add_subparsers(help='Commands')

    subparser = subparsers.add_parser('info', help='Display info about jobs')
    subparser.set_defaults(func=_info)

    subparser = subparsers.add_parser('destroy', help='Delete everything in the jobs DB')
    subparser.set_defaults(func=_destroy)

    subparser = subparsers.add_parser('work', help='Do background work')
    parser.add_argument('queues', nargs='+', help='Queues to do work on')
    subparser.set_defaults(func=_work)

    args = parser.parse_args()
    jobs = Jobs(args.redis_host, args.redis_port, 3, args.annotations_redis_host, args.annotations_redis_port)
    args.func(args, jobs)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from __future__ import absolute_import
from ..scan import Dispatcher
from ..spec.v1_2.objects import Operation, Model
from ..utils import scope_compose

class TypeReduce(object):
    """ Type Reducer, collect Operation & Model
    spreaded in Resources put in a global accessible place.
    """
    class Disp(Dispatcher): pass

    def __init__(self):
        self.op = {}
        self.model = {}

    @staticmethod
    def __insert(target, scope, name, obj):
        new_scope = scope_compose(scope, name)
        if new_scope in target.keys():
            raise ValueError('duplicated key found: ' + new_scope)

        target[new_scope] = obj

    @Disp.register([Model])
    def _model(self, scope, name, obj, _):
        self.__insert(self.model, scope, name, obj)

    @Disp.register([Operation])
    def _op(self, scope, name, obj, _):
        self.__insert(self.op, scope, name, obj)

<commit_msg>fix TypeReducer<commit_after>from __future__ import absolute_import
from ..scan import Dispatcher
from ..spec.v2_0.objects import Operation
from ..utils import scope_compose

class TypeReduce(object):
    """ Type Reducer, collect Operation & Model
    spreaded in Resources put in a global accessible place.
    """
    class Disp(Dispatcher): pass

    def __init__(self):
        self.op = {}

    @Disp.register([Operation])
    def _op(self, path, obj, _):
        scope = obj.tags[0] if obj.tags and len(obj.tags) > 0 else None
        name = obj.operationId if obj.operationId else None

        if scope and name:
            new_scope = scope_compose(scope, name)
            if new_scope in self.op.keys():
                raise ValueError('duplicated key found: ' + new_scope)

            self.op[new_scope] = obj

<|endoftext|>


<commit_before>"""Tests concerning S3 interaction."""
import os
from . import base
from buildercore import s3

class SimpleCases(base.BaseCase):
    def setUp(self):
        s3.delete_contents("test/")
        os.system("mkdir -p /tmp/builder/")
        self.envname = base.generate_environment_name()

    def tearDown(self):
        s3.delete_contents("test/")
        os.system("rm -f /tmp/builder/*")

    def test_exists(self):
        key = "test/boo"
        self.assertFalse(s3.exists(key))

    def test_writable(self):
        key = "test/foo-%s" % self.envname
        self.assertFalse(s3.exists(key))
        s3.write(key, "asdf")
        self.assertTrue(s3.exists(key))

    def test_overwrite(self):
        key = "test/foo-%s" % self.envname
        self.assertFalse(s3.exists(key))
        s3.write(key, "asdf")
        self.assertRaises(KeyError, s3.write, key, "fdsa")
        s3.write(key, "fdsa", overwrite=True)

        # TODO: test content was actually overwritten

    def test_delete(self):
        key = "test/foo-%s" % self.envname
        s3.write(key, "asdf")
        self.assertTrue(s3.exists(key))
        s3.delete(key)
        self.assertFalse(s3.exists(key))

    def test_list(self):
        keys = [
            "test/foo-%s" % base.generate_environment_name(),
            "test/bar-%s" % base.generate_environment_name(),
            "test/baz-%s" % base.generate_environment_name(),
        ]
        for key in keys:
            s3.write(key, 'asdf')
        self.assertEqual(sorted(keys), sorted(s3.simple_listing("test/")))

    def test_download(self):
        key = "test/foo-%s" % self.envname
        expected_contents = "test content"
        expected_output = '/tmp/builder/baz'
        s3.write(key, expected_contents)
        s3.download(key, expected_output)
        self.assertTrue(os.path.exists(expected_output))
        self.assertEqual(open(expected_output, 'r').read(), expected_contents)
<commit_msg>Do not rely on pre-existing keys in test_list<commit_after>"""Tests concerning S3 interaction."""
import os
from . import base
from buildercore import s3

class SimpleCases(base.BaseCase):
    def setUp(self):
        s3.delete_contents("test/")
        os.system("mkdir -p /tmp/builder/")
        self.envname = base.generate_environment_name()

    def tearDown(self):
        s3.delete_contents("test/")
        os.system("rm -f /tmp/builder/*")

    def test_exists(self):
        key = "test/boo"
        self.assertFalse(s3.exists(key))

    def test_writable(self):
        key = "test/foo-%s" % self.envname
        self.assertFalse(s3.exists(key))
        s3.write(key, "asdf")
        self.assertTrue(s3.exists(key))

    def test_overwrite(self):
        key = "test/foo-%s" % self.envname
        self.assertFalse(s3.exists(key))
        s3.write(key, "asdf")
        self.assertRaises(KeyError, s3.write, key, "fdsa")
        s3.write(key, "fdsa", overwrite=True)

        # TODO: test content was actually overwritten

    def test_delete(self):
        key = "test/foo-%s" % self.envname
        s3.write(key, "asdf")
        self.assertTrue(s3.exists(key))
        s3.delete(key)
        self.assertFalse(s3.exists(key))

    def test_list(self):
        keys = [
            "test/foo-%s" % base.generate_environment_name(),
            "test/bar-%s" % base.generate_environment_name(),
            "test/baz-%s" % base.generate_environment_name(),
        ]
        for key in keys:
            s3.write(key, 'asdf')
        list_result = sorted(s3.simple_listing("test/"))
        for key in keys:
            self.assertIn(key, list_result)

    def test_download(self):
        key = "test/foo-%s" % self.envname
        expected_contents = "test content"
        expected_output = '/tmp/builder/baz'
        s3.write(key, expected_contents)
        s3.download(key, expected_output)
        self.assertTrue(os.path.exists(expected_output))
        self.assertEqual(open(expected_output, 'r').read(), expected_contents)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from django.conf import settings
from django.shortcuts import render_to_response
import json
from django.utils.translation import ugettext as _
from django.utils.translation import get_language
from proso.django.config import get_global_config
from proso_flashcards.models import Category
from django.views.decorators.csrf import ensure_csrf_cookie
from proso_models.models import get_environment
from proso.django.config import get_config
from proso_user.models import migrate_google_openid_user
import django.contrib.auth as auth
from proso.django.request import is_user_id_overridden
import os
import random
from django.template import RequestContext
from django.core import management
from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse
import base64

JS_FILES = (
    "dist/js/bower-libs.min.js",
    "dist/js/proso-apps-all.js",
    "dist/js/geography.min.js",
    "dist/js/geography.html.js",
)
CSS_FILES = (
    "dist/css/bower-libs.css",
    "dist/css/app.css",
    "dist/css/map.css"
)

@ensure_csrf_cookie
def home(request, hack=None):
    if not hasattr(request.user, "userprofile") or request.user.userprofile is None:
        environment = get_environment()
        user = json.dumps({
            'user': {},
            'number_of_answers': environment.number_of_answers(user=request.user.id) if request.user.id is not None else 0,
            'number_of_correct_answers': environment.number_of_correct_answers(user=request.user.id) if request.user.id is not None else 0,
        })
        email = ''
    else:
        if get_config('proso_user', 'google.openid.migration', default=True) and not is_user_id_overridden(request):
            migrated_user = migrate_google_openid_user(request.user)
            if migrated_user is not None:
                auth.logout(request)
                migrated_user.backend = 'social_auth.backends.google.GoogleOAuth2Backend'
                auth.login(request, migrated_user)
        user = json.dumps(request.user.userprofile.to_json(stats=True))
        email = request.user.email
    c = {
        'title': _(u'Slepé mapy') + ' - ' + _(u'inteligentní aplikace na procvičování zeměpisu'),
        'map': get_map_from_url(hack),
        'is_production': settings.ON_PRODUCTION,
        'css_files': CSS_FILES,
        'map_files': get_map_files(),
        'js_files': JS_FILES,
        'continents': Category.objects.filter(
            lang=get_language(), type='continent').order_by('name'),
        'states': Category.objects.filter(
            lang=get_language(), type='state').order_by('name'),
        'user_json': user,
        'email': email,
        'LANGUAGE_CODE': get_language(),
        'LANGUAGES': settings.LANGUAGES,
        'LANGUAGE_DOMAINS': settings.LANGUAGE_DOMAINS if hasattr(
            settings, 'LANGUAGE_DOMAINS') else {},
        'is_homepage': hack is None,
        'hack': hack or '',
        'config_json': json.dumps(get_global_config()),
        'DOMAIN': request.build_absolute_uri('/')[:-1],
        'screenshot_files': get_screenshot_files(request, hack),
    }
    return render_to_response('home.html', c)

def get_map_files():
    path = os.path.join(settings.STATICFILES_DIRS[0], 'map')
    dirs = os.listdir(path)
    map_files = ['map/' + file for file in dirs]
    return map_files

def get_screenshot_files(request, hack):
    screenshot_files = [
        "/static/img/practice-example-" + get_language() + ".png",
        "/static/img/knowledge-map.png",
        "/static/img/knowledge-map-africa.png",
        "/static/img/knowledge-map-asia.png",
        "/static/img/knowledge-map-cz-mountains.png",
    ]

    path = os.path.join(settings.STATICFILES_DIRS[0], 'img', 'thumb')
    dirs = os.listdir(path)

    for file in dirs:
        if get_language() + '.png' in file:
            screenshot_files.append('/static/img/thumb/' + file)
    random.shuffle(screenshot_files)

    if hack is not None:
        thumb_file_name = hack.replace('/', '-').replace(
            'practice-', '').replace('view-', '') + '.png'
        thumb_file = os.path.join(
            settings.MEDIA_ROOT, 'thumbs', thumb_file_name)
        if os.path.exists(thumb_file):
            screenshot_files[0] = "/media/thumbs/" + thumb_file_name

        thumb_file_name = hack.replace('/', '-') + '-' + get_language() +'.png'
        thumb_file = os.path.join(settings.STATICFILES_DIRS[0], 'img', 'thumb', thumb_file_name)
        if os.path.exists(thumb_file):
            screenshot_files[0] = "/static/img/thumb/" + thumb_file_name

    if request.GET.get('thumb', None) is not None:
        screenshot_files[0] = "/static/img/thumb/" + request.GET['thumb'] + "-" + get_language() + ".png"
    return screenshot_files[:5]

def get_map_from_url(hack):
    map_string = ""
    if hack:
        url = hack.split('/')
        if url[0] == 'view' or url[0] == 'practice':
            map_code = url[1]
            try:
                map_category = Category.objects.get(lang=get_language(), identifier=map_code)
                map_string = map_category.name
                if len(url) >= 3 and url[2] != '':
                    map_string = map_string + ' - ' + resolve_map_type(url[2])
            except Category.DoesNotExist:
                pass
    return map_string

def resolve_map_type(code):
    types = {
        'state': _(u'Státy'),
        'region': _('Regiony'),
        'province': _('Provincie'),
        'region_cz': _('Kraje'),
        'region_it': _('Oblasti'),
        'autonomous_Comunity': _(u'Autonomní společenství'),
        'bundesland': _(u'Spolkové země'),
        'city': _(u'Města'),
        'river': _(u'Řeky'),
        'lake': _('Jezera'),
        'sea': _(u'Moře'),
        'mountains': _(u'Pohoří'),
        'island': _('Ostrovy'),
    }
    return types.get(code, '')

@staff_member_required
def load_data(request):
    c = RequestContext(request, {
        'css_files': CSS_FILES,
        'alerts': [],
    })

    if request.method == 'POST':
        if 'svg' not in request.FILES:
            c['alerts'].append({
                'text': 'Chybí SVG soubor',
                'type': 'danger',
            })
        if 'json' not in request.FILES:
            c['alerts'].append({
                'text': 'Chybí JSON soubor',
                'type': 'danger',
            })

        if len(c['alerts']) == 0:
            json_file = request.FILES['json']
            filepath = os.path.join(settings.DATA_DIR, json_file.name)
            handle_uploaded_file(json_file, filepath)

            management.call_command(
                'load_flashcards',
                filepath,
                ignored_flashcards='disable',
                skip_language_check=True,
                verbosity=0,
                interactive=False)

            svg_file = request.FILES['svg']
            filepath = os.path.join(settings.STATICFILES_DIRS[0], 'map', svg_file.name)
            handle_uploaded_file(svg_file, filepath)

            management.call_command(
                'collectstatic',
                verbosity=0,
                interactive=False)
            c['alerts'].append({
                'type': 'success',
                'text': 'Mapa byla úspěšně nahrána',
            })

    return render_to_response('load_data.html', c)

def handle_uploaded_file(f, filepath):
    destination = open(filepath, 'wb+')
    for chunk in f.chunks():
        destination.write(chunk)
    destination.close()

def save_screenshot(request):
    if request.body:
        data = json.loads(request.body.decode("utf-8"))
        image = data['image']
        filename = os.path.join(
            settings.MEDIA_ROOT, 'thumbs', data['name'] + '.png')
        save_base64_to_file(filename, image)
        if hasattr(request.user, "username"):
            filename = os.path.join(
                settings.MEDIA_ROOT,
                'userthumbs',
                request.user.username + '--' + data['name'] + '.png')
            save_base64_to_file(filename, image)

        response = """{
            "type": "success",
            "msg" : "Obrázek byl úspěšně nahrán"
        }"""
        return HttpResponse(response, content_type='application/javascript')

def save_base64_to_file(filename, image):
    directory = os.path.dirname(filename)
    if not os.path.exists(directory):
        os.makedirs(directory)
    head = 'data:image/png;base64,'
    if head in image:
        image = image[len(head):]
        fh = open(filename, "wb")
        fh.write(base64.b64decode(image))
        fh.close()
<commit_msg>Redirect registered users to maps overview<commit_after># -*- coding: utf-8 -*-
from django.conf import settings
from django.shortcuts import render_to_response, redirect
import json
from django.utils.translation import ugettext as _
from django.utils.translation import get_language
from proso.django.config import get_global_config
from proso_flashcards.models import Category
from django.views.decorators.csrf import ensure_csrf_cookie
from proso_models.models import get_environment
from proso.django.config import get_config
from proso_user.models import migrate_google_openid_user
import django.contrib.auth as auth
from proso.django.request import is_user_id_overridden
import os
import random
from django.template import RequestContext
from django.core import management
from django.contrib.admin.views.decorators import staff_member_required
from django.http import HttpResponse
import base64

JS_FILES = (
    "dist/js/bower-libs.min.js",
    "dist/js/proso-apps-all.js",
    "dist/js/geography.min.js",
    "dist/js/geography.html.js",
)
CSS_FILES = (
    "dist/css/bower-libs.css",
    "dist/css/app.css",
    "dist/css/map.css"
)

@ensure_csrf_cookie
def home(request, hack=None):
    if not hasattr(request.user, "userprofile") or request.user.userprofile is None:
        environment = get_environment()
        user = json.dumps({
            'user': {},
            'number_of_answers': environment.number_of_answers(user=request.user.id) if request.user.id is not None else 0,
            'number_of_correct_answers': environment.number_of_correct_answers(user=request.user.id) if request.user.id is not None else 0,
        })
        email = ''
    else:
        if hack is None:
            return redirect('/overview/')
        if get_config('proso_user', 'google.openid.migration', default=True) and not is_user_id_overridden(request):
            migrated_user = migrate_google_openid_user(request.user)
            if migrated_user is not None:
                auth.logout(request)
                migrated_user.backend = 'social_auth.backends.google.GoogleOAuth2Backend'
                auth.login(request, migrated_user)
        user = json.dumps(request.user.userprofile.to_json(stats=True))
        email = request.user.email
    c = {
        'title': _(u'Slepé mapy') + ' - ' + _(u'inteligentní aplikace na procvičování zeměpisu'),
        'map': get_map_from_url(hack),
        'is_production': settings.ON_PRODUCTION,
        'css_files': CSS_FILES,
        'map_files': get_map_files(),
        'js_files': JS_FILES,
        'continents': Category.objects.filter(
            lang=get_language(), type='continent').order_by('name'),
        'states': Category.objects.filter(
            lang=get_language(), type='state').order_by('name'),
        'user_json': user,
        'email': email,
        'LANGUAGE_CODE': get_language(),
        'LANGUAGES': settings.LANGUAGES,
        'LANGUAGE_DOMAINS': settings.LANGUAGE_DOMAINS if hasattr(
            settings, 'LANGUAGE_DOMAINS') else {},
        'is_homepage': hack is None,
        'hack': hack or '',
        'config_json': json.dumps(get_global_config()),
        'DOMAIN': request.build_absolute_uri('/')[:-1],
        'screenshot_files': get_screenshot_files(request, hack),
    }
    return render_to_response('home.html', c)

def get_map_files():
    path = os.path.join(settings.STATICFILES_DIRS[0], 'map')
    dirs = os.listdir(path)
    map_files = ['map/' + file for file in dirs]
    return map_files

def get_screenshot_files(request, hack):
    screenshot_files = [
        "/static/img/practice-example-" + get_language() + ".png",
        "/static/img/knowledge-map.png",
        "/static/img/knowledge-map-africa.png",
        "/static/img/knowledge-map-asia.png",
        "/static/img/knowledge-map-cz-mountains.png",
    ]

    path = os.path.join(settings.STATICFILES_DIRS[0], 'img', 'thumb')
    dirs = os.listdir(path)

    for file in dirs:
        if get_language() + '.png' in file:
            screenshot_files.append('/static/img/thumb/' + file)
    random.shuffle(screenshot_files)

    if hack is not None:
        thumb_file_name = hack.replace('/', '-').replace(
            'practice-', '').replace('view-', '') + '.png'
        thumb_file = os.path.join(
            settings.MEDIA_ROOT, 'thumbs', thumb_file_name)
        if os.path.exists(thumb_file):
            screenshot_files[0] = "/media/thumbs/" + thumb_file_name

        thumb_file_name = hack.replace('/', '-') + '-' + get_language() +'.png'
        thumb_file = os.path.join(settings.STATICFILES_DIRS[0], 'img', 'thumb', thumb_file_name)
        if os.path.exists(thumb_file):
            screenshot_files[0] = "/static/img/thumb/" + thumb_file_name

    if request.GET.get('thumb', None) is not None:
        screenshot_files[0] = "/static/img/thumb/" + request.GET['thumb'] + "-" + get_language() + ".png"
    return screenshot_files[:5]

def get_map_from_url(hack):
    map_string = ""
    if hack:
        url = hack.split('/')
        if url[0] == 'view' or url[0] == 'practice':
            map_code = url[1]
            try:
                map_category = Category.objects.get(lang=get_language(), identifier=map_code)
                map_string = map_category.name
                if len(url) >= 3 and url[2] != '':
                    map_string = map_string + ' - ' + resolve_map_type(url[2])
            except Category.DoesNotExist:
                pass
    return map_string

def resolve_map_type(code):
    types = {
        'state': _(u'Státy'),
        'region': _('Regiony'),
        'province': _('Provincie'),
        'region_cz': _('Kraje'),
        'region_it': _('Oblasti'),
        'autonomous_Comunity': _(u'Autonomní společenství'),
        'bundesland': _(u'Spolkové země'),
        'city': _(u'Města'),
        'river': _(u'Řeky'),
        'lake': _('Jezera'),
        'sea': _(u'Moře'),
        'mountains': _(u'Pohoří'),
        'island': _('Ostrovy'),
    }
    return types.get(code, '')

@staff_member_required
def load_data(request):
    c = RequestContext(request, {
        'css_files': CSS_FILES,
        'alerts': [],
    })

    if request.method == 'POST':
        if 'svg' not in request.FILES:
            c['alerts'].append({
                'text': 'Chybí SVG soubor',
                'type': 'danger',
            })
        if 'json' not in request.FILES:
            c['alerts'].append({
                'text': 'Chybí JSON soubor',
                'type': 'danger',
            })

        if len(c['alerts']) == 0:
            json_file = request.FILES['json']
            filepath = os.path.join(settings.DATA_DIR, json_file.name)
            handle_uploaded_file(json_file, filepath)

            management.call_command(
                'load_flashcards',
                filepath,
                ignored_flashcards='disable',
                skip_language_check=True,
                verbosity=0,
                interactive=False)

            svg_file = request.FILES['svg']
            filepath = os.path.join(settings.STATICFILES_DIRS[0], 'map', svg_file.name)
            handle_uploaded_file(svg_file, filepath)

            management.call_command(
                'collectstatic',
                verbosity=0,
                interactive=False)
            c['alerts'].append({
                'type': 'success',
                'text': 'Mapa byla úspěšně nahrána',
            })

    return render_to_response('load_data.html', c)

def handle_uploaded_file(f, filepath):
    destination = open(filepath, 'wb+')
    for chunk in f.chunks():
        destination.write(chunk)
    destination.close()

def save_screenshot(request):
    if request.body:
        data = json.loads(request.body.decode("utf-8"))
        image = data['image']
        filename = os.path.join(
            settings.MEDIA_ROOT, 'thumbs', data['name'] + '.png')
        save_base64_to_file(filename, image)
        if hasattr(request.user, "username"):
            filename = os.path.join(
                settings.MEDIA_ROOT,
                'userthumbs',
                request.user.username + '--' + data['name'] + '.png')
            save_base64_to_file(filename, image)

        response = """{
            "type": "success",
            "msg" : "Obrázek byl úspěšně nahrán"
        }"""
        return HttpResponse(response, content_type='application/javascript')

def save_base64_to_file(filename, image):
    directory = os.path.dirname(filename)
    if not os.path.exists(directory):
        os.makedirs(directory)
    head = 'data:image/png;base64,'
    if head in image:
        image = image[len(head):]
        fh = open(filename, "wb")
        fh.write(base64.b64decode(image))
        fh.close()
<|endoftext|>


<commit_before>
from dcore import *
from firedrake import IcosahedralSphereMesh, ExtrudedMesh, Expression, \
    VectorFunctionSpace
from firedrake import exp, acos, cos, sin
import numpy as np

nlayers = 10  # 10 horizontal layers
refinements = 3  # number of horizontal cells = 20*(4^refinements)

# build surface mesh
a_ref = 6.37122e6
X = 125.0  # Reduced-size Earth reduction factor
a = a_ref/X
g = 9.810616
N = 0.01  # Brunt-Vaisala frequency (1/s)
p_0 = 1000.0 * 100.0  # Reference pressure (Pa, not hPa)
c_p = 1004.5  # SHC of dry air at constant pressure (J/kg/K)
R_d = 287.0  # Gas constant for dry air (J/kg/K)
kappa = 2.0/7.0  # R_d/c_p
T_eq = 300.0  # Isothermal atmospheric temperature (K)
p_eq = 1000.0 * 100.0  # Reference surface pressure at the equator
u_0 = 20.0  # Maximum amplitude of the zonal wind (m/s)

d = 5000.0  # Width parameter for Theta'
lamda_c = 2.0*np.pi/3.0  # Longitudinal centerpoint of Theta'
phi_c = 0.0  # Latitudinal centerpoint of Theta' (equator)
deltaTheta = 1.0  # Maximum amplitude of Theta' (K)
L_z = 20000.0  # Vertical wave length of the Theta' perturbation

m = IcosahedralSphereMesh(radius=a,
                          refinement_level=refinements)

# build volume mesh
z_top = 1.0e4  # Height position of the model top
mesh = ExtrudedMesh(m, layers=nlayers, layer_height=z_top/nlayers,
                    extrusion_type="radial")

# Space for initialising velocity
W_VectorCG1 = VectorFunctionSpace(mesh, "CG", 1)
W_CG1 = FunctionSpace(mesh, "CG", 1)

Omega = Function(W_VectorCG1).assign(0.0)

# Create polar coordinates
z = Function(W_CG1).interpolate(Expression("sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]) - a",a=a))  # Since we use a CG1 field, this is constant on layers
lat = Function(W_CG1).interpolate(Expression("asin(x[2]/sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]))"))
lon = Function(W_CG1).interpolate(Expression("atan2(x[1], x[0])"))

k = Function(W_VectorCG1).interpolate(
    Expression(("x[0]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])",
               "x[1]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])",
                "x[2]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])")))

state = State(mesh, vertical_degree=0, horizontal_degree=0,
              family="BDM")

# Initial conditions
u0, theta0, rho0 = Function(state.V[0]), Function(state.V[2]), Function(state.V[1])

# Initial conditions without u0
# Surface temperature
G = g**2/(N**2*c_p)
Ts = Function(W_CG1).assign(T_eq)

# surface pressure
psexp = p_eq*((Ts/T_eq)**(1.0/kappa))

ps = Function(W_CG1).interpolate(psexp)

# Background pressure
pexp = ps*(1 + G/Ts*(exp(-N**2*z/g)-1))**(1.0/kappa)

p = Function(W_CG1).interpolate(pexp)

# Background temperature
# Tbexp = Ts*(p/ps)**kappa/(Ts/G*((p/ps)**kappa - 1) + 1)
Tbexp = G*(1 - exp(N**2*z/g)) + Ts*exp(N**2*z/g)

Tb = Function(W_CG1).interpolate(Tbexp)

# Background potential temperature
thetabexp = Tb*(p_0/p)**kappa

thetab = Function(W_CG1).interpolate(thetabexp)

print thetab.dat.data.min()

rhoexp = p/(R_d*Tb)

rhob = Function(W_CG1).interpolate(rhoexp)

print rhob.dat.data.min()

theta_b = Function(state.V[2]).interpolate(thetab)
rho_b = Function(state.V[1]).project(rhob)

sin_tmp = sin(lat) * sin(phi_c)
cos_tmp = cos(lat) * cos(phi_c)

r = a*acos(sin_tmp + cos_tmp*cos(lon-lamda_c))

s = (d**2)/(d**2 + r**2)

theta_pert = deltaTheta*s*sin(2*np.pi*z/L_z)

theta0.interpolate(theta_b + theta_pert)
rho0.assign(rho_b)

state.initialise(u0, rho0, theta0)
state.set_reference_profiles(rho_b, theta_b)

# Set up advection schemes
advection_list = []
velocity_advection = NoAdvection(state)
advection_list.append((velocity_advection, 0))
rho_advection = LinearAdvection_V3(state, state.V[1], rho_b)
advection_list.append((rho_advection, 1))
theta_advection = LinearAdvection_Vt(state, state.V[2], theta_b)
advection_list.append((theta_advection, 2))

# Set up linear solver
schur_params = {'pc_type': 'fieldsplit',
                'pc_fieldsplit_type': 'schur',
                'ksp_type': 'gmres',
                'ksp_monitor_true_residual': True,
                'ksp_max_it': 100,
                'ksp_gmres_restart': 50,
                'pc_fieldsplit_schur_fact_type': 'FULL',
                'pc_fieldsplit_schur_precondition': 'selfp',
                'fieldsplit_0_ksp_type': 'richardson',
                'fieldsplit_0_ksp_max_it': 5,
                'fieldsplit_0_pc_type': 'bjacobi',
                'fieldsplit_0_sub_pc_type': 'ilu',
                'fieldsplit_1_ksp_type': 'richardson',
                'fieldsplit_1_ksp_max_it': 5,
                "fieldsplit_1_ksp_monitor_true_residual": True,
                'fieldsplit_1_pc_type': 'gamg',
                'fieldsplit_1_pc_gamg_sym_graph': True,
                'fieldsplit_1_mg_levels_ksp_type': 'chebyshev',
                'fieldsplit_1_mg_levels_ksp_chebyshev_estimate_eigenvalues': True,
                'fieldsplit_1_mg_levels_ksp_chebyshev_estimate_eigenvalues_random': True,
                'fieldsplit_1_mg_levels_ksp_max_it': 5,
                'fieldsplit_1_mg_levels_pc_type': 'bjacobi',
                'fieldsplit_1_mg_levels_sub_pc_type': 'ilu'}

linear_solver = CompressibleSolver(state, alpha=0.5, params=schur_params)

# Set up forcing
compressible_forcing = CompressibleForcing(state)

# build time stepper
stepper = Timestepper(state, advection_list, linear_solver,
                      compressible_forcing)

stepper.run(t=0, tmax=3600.0)
<commit_msg>this example file now runs<commit_after>from dcore import *
from firedrake import IcosahedralSphereMesh, ExtrudedMesh, Expression, \
    VectorFunctionSpace
from firedrake import exp, acos, cos, sin
import numpy as np

nlayers = 10  # 10 horizontal layers
refinements = 3  # number of horizontal cells = 20*(4^refinements)

# build surface mesh
a_ref = 6.37122e6
X = 125.0  # Reduced-size Earth reduction factor
a = a_ref/X
g = 9.810616
N = 0.01  # Brunt-Vaisala frequency (1/s)
p_0 = 1000.0 * 100.0  # Reference pressure (Pa, not hPa)
c_p = 1004.5  # SHC of dry air at constant pressure (J/kg/K)
R_d = 287.0  # Gas constant for dry air (J/kg/K)
kappa = 2.0/7.0  # R_d/c_p
T_eq = 300.0  # Isothermal atmospheric temperature (K)
p_eq = 1000.0 * 100.0  # Reference surface pressure at the equator
u_0 = 20.0  # Maximum amplitude of the zonal wind (m/s)

d = 5000.0  # Width parameter for Theta'
lamda_c = 2.0*np.pi/3.0  # Longitudinal centerpoint of Theta'
phi_c = 0.0  # Latitudinal centerpoint of Theta' (equator)
deltaTheta = 1.0  # Maximum amplitude of Theta' (K)
L_z = 20000.0  # Vertical wave length of the Theta' perturbation

m = IcosahedralSphereMesh(radius=a,
                          refinement_level=refinements)

# build volume mesh
z_top = 1.0e4  # Height position of the model top
mesh = ExtrudedMesh(m, layers=nlayers, layer_height=z_top/nlayers,
                    extrusion_type="radial")

# Space for initialising velocity
W_VectorCG1 = VectorFunctionSpace(mesh, "CG", 1)
W_CG1 = FunctionSpace(mesh, "CG", 1)

Omega = Function(W_VectorCG1).assign(0.0)

# Create polar coordinates
z = Function(W_CG1).interpolate(Expression("sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]) - a",a=a))  # Since we use a CG1 field, this is constant on layers
lat = Function(W_CG1).interpolate(Expression("asin(x[2]/sqrt(x[0]*x[0] + x[1]*x[1] + x[2]*x[2]))"))
lon = Function(W_CG1).interpolate(Expression("atan2(x[1], x[0])"))

k = Function(W_VectorCG1).interpolate(
    Expression(("x[0]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])",
               "x[1]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])",
                "x[2]/sqrt(x[0]*x[0]+x[1]*x[1]+x[2]*x[2])")))

fieldlist = ['u','rho','theta']
timestepping = TimesteppingParameters(dt=10.0)
output = OutputParameters(Verbose=True, dumpfreq=1, dirname='dcmip_new')
parameters = CompressibleParameters()
state = CompressibleState(mesh, vertical_degree=0, horizontal_degree=0,
                          family="BDM", k=k, z=z,
                          timestepping=timestepping,
                          output=output,
                          parameters=parameters,
                          fieldlist=fieldlist)

# Initial conditions
u0, theta0, rho0 = Function(state.V[0]), Function(state.V[2]), Function(state.V[1])

# Initial conditions without u0
# Surface temperature
G = g**2/(N**2*c_p)
Ts = Function(W_CG1).assign(T_eq)

# surface pressure
psexp = p_eq*((Ts/T_eq)**(1.0/kappa))

ps = Function(W_CG1).interpolate(psexp)

# Background pressure
pexp = ps*(1 + G/Ts*(exp(-N**2*z/g)-1))**(1.0/kappa)

p = Function(W_CG1).interpolate(pexp)

# Background temperature
# Tbexp = Ts*(p/ps)**kappa/(Ts/G*((p/ps)**kappa - 1) + 1)
Tbexp = G*(1 - exp(N**2*z/g)) + Ts*exp(N**2*z/g)

Tb = Function(W_CG1).interpolate(Tbexp)

# Background potential temperature
thetabexp = Tb*(p_0/p)**kappa

thetab = Function(W_CG1).interpolate(thetabexp)

print thetab.dat.data.min()

rhoexp = p/(R_d*Tb)

rhob = Function(W_CG1).interpolate(rhoexp)

print rhob.dat.data.min()

theta_b = Function(state.V[2]).interpolate(thetab)
rho_b = Function(state.V[1]).project(rhob)

sin_tmp = sin(lat) * sin(phi_c)
cos_tmp = cos(lat) * cos(phi_c)

r = a*acos(sin_tmp + cos_tmp*cos(lon-lamda_c))

s = (d**2)/(d**2 + r**2)

theta_pert = deltaTheta*s*sin(2*np.pi*z/L_z)

theta0.interpolate(theta_b + theta_pert)
rho0.assign(rho_b)

state.initialise([u0, rho0, theta0])
state.set_reference_profiles(rho_b, theta_b)

# Set up advection schemes
advection_list = []
velocity_advection = NoAdvection(state)
advection_list.append((velocity_advection, 0))
rho_advection = LinearAdvection_V3(state, state.V[1], rho_b)
advection_list.append((rho_advection, 1))
theta_advection = LinearAdvection_Vt(state, state.V[2], theta_b)
advection_list.append((theta_advection, 2))

# Set up linear solver
schur_params = {'pc_type': 'fieldsplit',
                'pc_fieldsplit_type': 'schur',
                'ksp_type': 'gmres',
                'ksp_monitor_true_residual': True,
                'ksp_max_it': 100,
                'ksp_gmres_restart': 50,
                'pc_fieldsplit_schur_fact_type': 'FULL',
                'pc_fieldsplit_schur_precondition': 'selfp',
                'fieldsplit_0_ksp_type': 'richardson',
                'fieldsplit_0_ksp_max_it': 5,
                'fieldsplit_0_pc_type': 'bjacobi',
                'fieldsplit_0_sub_pc_type': 'ilu',
                'fieldsplit_1_ksp_type': 'richardson',
                'fieldsplit_1_ksp_max_it': 5,
                "fieldsplit_1_ksp_monitor_true_residual": True,
                'fieldsplit_1_pc_type': 'gamg',
                'fieldsplit_1_pc_gamg_sym_graph': True,
                'fieldsplit_1_mg_levels_ksp_type': 'chebyshev',
                'fieldsplit_1_mg_levels_ksp_chebyshev_estimate_eigenvalues': True,
                'fieldsplit_1_mg_levels_ksp_chebyshev_estimate_eigenvalues_random': True,
                'fieldsplit_1_mg_levels_ksp_max_it': 5,
                'fieldsplit_1_mg_levels_pc_type': 'bjacobi',
                'fieldsplit_1_mg_levels_sub_pc_type': 'ilu'}

linear_solver = CompressibleSolver(state, params=schur_params)

# Set up forcing
compressible_forcing = CompressibleForcing(state)

# build time stepper
stepper = Timestepper(state, advection_list, linear_solver,
                      compressible_forcing)

stepper.run(t=0, tmax=3600.0)
<|endoftext|>


<commit_before># pylint:disable=C0111

__title__ = 'geoip2'
__version__ = '0.5.0'
__author__ = 'Gregory Oschwald'
__license__ = 'Apache License, Version 2.0'
__copyright__ = 'Copyright 2014 Maxmind, Inc.'
<commit_msg>Update version number<commit_after># pylint:disable=C0111

__title__ = 'geoip2'
__version__ = '0.5.1'
__author__ = 'Gregory Oschwald'
__license__ = 'Apache License, Version 2.0'
__copyright__ = 'Copyright 2014 Maxmind, Inc.'
<|endoftext|>


<commit_before>__author__ = 'Ostico <ostico@gmail.com>'

import socket
import struct

from Constants.BinaryTypes import *
from Constants.ClientConstants import *
from pyorient.Commons.utils import *
from Constants.OrientPrimitives import *

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class OrientSocket(object):
    """docstring for OrientSocket"""

    MAX_READ_LENGTH = 512000

    def __init__(self, host, port):

        self._connected = False
        self.host = host
        self.port = port
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        """:type : socket.socket"""
        self.protocol = -1
        self.session_id = -1
        self.db_opened = None
        self.serialization_type = SERIALIZATION_DOCUMENT2CSV

    def get_connection(self):
        if not self._connected:
            self.connect()

        return self._socket

    def connect(self):
        dlog("Trying to connect...")
        try:
            self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._socket.connect( (self.host, self.port) )
            _value = self._socket.recv( FIELD_SHORT['bytes'] )
            self.protocol = struct.unpack('!h', _value)[0]
            if self.protocol > SUPPORTED_PROTOCOL:
                raise PyOrientWrongProtocolVersionException(
                    "Protocol version " + str(self.protocol) +
                    " is not supported yet by this client.", [])
            self._connected = True
        except socket.error, e:
            self._connected = False
            raise PyOrientConnectionException( "Socket Error: %s" % e, [] )

    def close(self):
        self.host = ''
        self.port = ''
        self.protocol = -1
        self.session_id = -1
        self._socket.close()
        self._connected = False

    def write(self, buff):
        return self._socket.send(buff)

    # The man page for recv says: The receive calls normally return
    #   any data available, up to the requested amount, rather than waiting
    #   for receipt of the full amount requested.
    #
    # If you need to read a given number of bytes, you need to call recv
    #   in a loop and concatenate the returned packets until
    #   you have read enough.
    def read(self, _len_to_read):

        buf = StringIO()
        try:
            while buf.tell() < _len_to_read:
                tmp = self._socket.recv( _len_to_read - buf.tell() )
                buf.write( tmp )

                if is_debug_verbose():
                    import pyorient.Commons.hexdump
                    print( "\n          -------------------\n" )
                    print( "    To read was: " + str( _len_to_read ) )
                    print( pyorient.Commons.hexdump.hexdump(tmp, 'return') )
                    print( "    left: " + str( _len_to_read - buf.tell() ) )

            buf.seek(0)
            return buf.read( _len_to_read )
        except Exception, e:
            self._connected = False
            raise e
        finally:
            buf.close()<commit_msg>Added Timeout in OrientSocket recv<commit_after>__author__ = 'Ostico <ostico@gmail.com>'

import socket
import struct

from Constants.BinaryTypes import *
from Constants.ClientConstants import *
from pyorient.Commons.utils import *
from Constants.OrientPrimitives import *

try:
    from cStringIO import StringIO
except ImportError:
    from StringIO import StringIO

class OrientSocket(object):
    """docstring for OrientSocket"""

    MAX_READ_LENGTH = 512000

    def __init__(self, host, port):

        self._connected = False
        self.host = host
        self.port = port
        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        """:type : socket.socket"""
        self.protocol = -1
        self.session_id = -1
        self.db_opened = None
        self.serialization_type = SERIALIZATION_DOCUMENT2CSV

    def get_connection(self):
        if not self._connected:
            self.connect()

        return self._socket

    def connect(self):
        dlog("Trying to connect...")
        try:
            self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._socket.connect( (self.host, self.port) )
            self._socket.settimeout(5.0)  # 15 secs of timeout
            _value = self._socket.recv( FIELD_SHORT['bytes'] )
            self.protocol = struct.unpack('!h', _value)[0]
            if self.protocol > SUPPORTED_PROTOCOL:
                raise PyOrientWrongProtocolVersionException(
                    "Protocol version " + str(self.protocol) +
                    " is not supported yet by this client.", [])
            self._connected = True
        except socket.error, e:
            self._connected = False
            raise PyOrientConnectionException( "Socket Error: %s" % e, [] )

    def close(self):
        self.host = ''
        self.port = ''
        self.protocol = -1
        self.session_id = -1
        self._socket.close()
        self._connected = False

    def write(self, buff):
        return self._socket.send(buff)

    # The man page for recv says: The receive calls normally return
    #   any data available, up to the requested amount, rather than waiting
    #   for receipt of the full amount requested.
    #
    # If you need to read a given number of bytes, you need to call recv
    #   in a loop and concatenate the returned packets until
    #   you have read enough.
    def read(self, _len_to_read):

        buf = StringIO()
        try:
            while buf.tell() < _len_to_read:
                tmp = self._socket.recv( _len_to_read - buf.tell() )
                buf.write( tmp )

                if is_debug_verbose():
                    import pyorient.Commons.hexdump
                    print( "\n          -------------------\n" )
                    print( "    To read was: " + str( _len_to_read ) )
                    print( pyorient.Commons.hexdump.hexdump(tmp, 'return') )
                    print( "    left: " + str( _len_to_read - buf.tell() ) )

            buf.seek(0)
            return buf.read( _len_to_read )
        except socket.timeout, e:
            raise PyOrientConnectionException(
                "Timeout reading from socket: %s" % e, [] )
        except Exception, e:
            self._connected = False
            raise e
        finally:
            buf.close()<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
""" Contains smart indent modes """
import re
from pyqode.core.api import TextHelper, get_block_symbol_data
from pyqode.core.qt.QtGui import QTextCursor
from pyqode.core.modes.autoindent import AutoIndentMode
from pyqode.core.modes.matcher import SymbolMatcherMode

class PyAutoIndentMode(AutoIndentMode):
    """
    Customised :class:`pyqode.core.modes.AutoIndentMode` for python
    that tries its best to follow the pep8 indentation guidelines.

    """
    def __init__(self):
        super(PyAutoIndentMode, self).__init__()

    def on_install(self, editor):
        super().on_install(editor)
        self._helper = TextHelper(editor)

    def has_two_empty_line_before(self, tc):
        ln = tc.blockNumber()
        limit = ln - 1
        while ln > limit:
            if self._helper.line_text(ln).strip() != "":
                return False
            ln -= 1
        return True

    def has_unclosed_paren(self, tc):
        ln = tc.blockNumber()
        while ln >= 0:
            line = self._helper.line_text(ln)
            if line.count("(") > line.count(")"):
                return True
            ln -= 1
        return False

    def is_in_string_def(self, full_line, column):
        count = 0
        char = "'"
        for i in range(len(full_line)):
            if full_line[i] == "'" or full_line[i] == '"':
                count += 1
            if full_line[i] == '"' and i < column:
                char = '"'
        count_after_col = 0
        for i in range(column, len(full_line)):
            if full_line[i] == "'" or full_line[i] == '"':
                count_after_col += 1
        return count % 2 == 0 and count_after_col % 2 == 1, char

    def is_paren_open(self, paren):
        return (paren.character == "(" or paren.character == "["
                or paren.character == '{')

    def is_paren_closed(self, paren):
        return (paren.character == ")" or paren.character == "]"
                or paren.character == '}')

    def get_full_line(self, tc):
        tc2 = QTextCursor(tc)
        tc2.select(QTextCursor.LineUnderCursor)
        full_line = tc2.selectedText()
        return full_line

    def parens_count_for_block(self, col, block):
        open_p = []
        closed_p = []
        lists = get_block_symbol_data(block)
        for symbols in lists:
            for paren in symbols:
                if paren.position >= col:
                    continue
                if self.is_paren_open(paren):
                    if not col:
                        return -1, -1, [], []
                    open_p.append(paren)
                if self.is_paren_closed(paren):
                    closed_p.append(paren)
        return len(open_p), len(closed_p), open_p, closed_p

    def between_paren(self, tc, col):
        block = tc.block()
        nb_open = nb_closed = 0
        while block.isValid():  # and nb_open >= nb_closed:
            o, c, _, _ = self.parens_count_for_block(col, block)
            nb_open += o
            nb_closed += c
            block = block.previous()
            col = len(block.text())
        return nb_open > nb_closed

    def get_last_word(self, tc):
        tc2 = QTextCursor(tc)
        tc2.movePosition(QTextCursor.Left, 1)
        tc2.movePosition(QTextCursor.WordLeft, tc.KeepAnchor)
        # tc2.movePosition(QTextCursor.Right, tc.KeepAnchor,
        #                  self.editor.cursorPosition[1])
        return tc2.selectedText().strip()

    def get_indent_of_opening_paren(self, tc):
        tc.movePosition(tc.Left, tc.KeepAnchor)
        char = tc.selectedText()
        tc.movePosition(tc.Right, tc.MoveAnchor)
        mapping = {')': ('(', 0), '}': ('{', 1), ']': ('[', 2)}
        try:
            character, char_type = mapping[char]
        except KeyError:
            return None
        else:
            ol, oc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
                tc, character, char_type)
            line = self._helper.line_text(ol)
            return len(line) - len(line.lstrip())

    def get_last_open_paren_pos(self, tc, column):
        pos = None
        char = None
        ln = tc.blockNumber() + 1
        tc_trav = QTextCursor(tc)
        while ln >= 1:
            tc_trav.movePosition(tc_trav.StartOfLine, tc_trav.MoveAnchor)
            lists = get_block_symbol_data(tc_trav.block())
            for symbols in lists:
                for paren in reversed(symbols):
                    if paren.position < column:
                        if self.is_paren_open(paren):
                            if paren.position > column:
                                continue
                            else:
                                pos = tc_trav.position() + paren.position
                                char = paren.character
                                # ensure it does not have a closing paren on
                                # the same line
                                tc3 = QTextCursor(tc)
                                tc3.setPosition(pos)
                                l, c = self.editor.modes.get(
                                    SymbolMatcherMode).symbol_pos(tc3, ')')
                                if l == ln and c < column:
                                    continue
                                return pos, char
            # check previous line
            tc_trav.movePosition(tc_trav.Up, tc_trav.MoveAnchor)
            ln = tc_trav.blockNumber() + 1
            column = len(self._helper.line_text(ln))
        return pos, char

    def get_paren_pos(self, tc, column):
        pos, char = self.get_last_open_paren_pos(tc, column)
        if char == '(':
            ptype = 0
            closingchar = ')'
        elif char == '[':
            ptype = 1
            closingchar = ']'
        elif char == '{':
            ptype = 2
            closingchar = '}'
        tc2 = QTextCursor(tc)
        tc2.setPosition(pos)
        ol, oc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
            tc2, char, ptype)
        cl, cc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
            tc2, closingchar, ptype)
        return (ol, oc), (cl, cc)

    def get_next_char(self, tc):
        tc2 = QTextCursor(tc)
        tc2.movePosition(QTextCursor.NextCharacter, QTextCursor.KeepAnchor)
        char = tc2.selectedText()
        return char

    def handle_indent_after_paren(self, column, line, fullline, tc):
        """
        Handle indent between symbols such as parenthesis, braces,...
        """
        # elements might be separated by ',' 'or' 'and'
        next_char = self.get_next_char(tc)
        nextcharisclosingsymbol = next_char in [']', ')', '}']
        (oL, oC), (cL, cC) = self.get_paren_pos(tc, column)
        closingline = self._helper.line_text(cL)
        openingline = self._helper.line_text(oL)
        openingline = re.sub(r'".*"', "", openingline)
        openingline = re.sub(r"'.*'", "", openingline)
        openingindent = len(openingline) - len(openingline.lstrip())
        tokens = [t.strip() for t in re.split(', |and |or ',
                                              line[oC:column]) if t]

        # align with first token pos
        if len(closingline) > cC and closingline[cC] == ":":
            post = openingindent * " " + 8 * " "
        else:
            # press enter before a '}', ']', ')'
            # which close an affectation (tuple, list , dict)
            if nextcharisclosingsymbol and re.match('.*=[\s][\W].*',
                                                    openingline):
                post = openingindent * " "
            else:
                # align elems in list, tuple, dict
                if re.match('.*=[\s][\W].*', openingline) and oL - cL == 1:
                    post = openingindent * " " + 4 * " "
                # align elems in fct declaration (we align with first
                # token)
                else:
                    if len(tokens):
                        otext = TextHelper(self.editor).line_text(oL)
                        if oL == TextHelper(self.editor).current_line_nbr() or otext[-1] not in ['(', '{', '[']:
                            # align with last opened paren
                            post = oC * " "
                        else:
                            # align with first open paren
                            indent = TextHelper(self.editor).line_indent()
                            post = indent * ' '
                    else:
                        # post = openingindent * " " + self.editor.tab_length * " "
                        indent = len(openingindent * " " + self.editor.tab_length * " ") // self.editor.tab_length * self.editor.tab_length
                        post = indent * ' '

        pre = ""
        in_string_def, char = self.is_in_string_def(fullline, column)
        if in_string_def:
            pre = char
            post += char
        return pre, post

    def at_block_start(self, tc, line):
        """
        Improve QTextCursor.atBlockStart to ignore spaces
        """
        if tc.atBlockStart():
            return True
        column = tc.columnNumber()
        indentation = len(line) - len(line.lstrip())
        return column <= indentation

    def at_block_end(self, tc, fullline):
        if tc.atBlockEnd():
            return True
        column = tc.columnNumber()
        return column >= len(fullline.rstrip()) - 1

    def _get_indent(self, cursor):
        pos = cursor.position()
        ln, column = self._helper.cursor_position()
        fullline = self.get_full_line(cursor)
        line = fullline[:column]
        # no indent
        if pos == 0 or column == 0:
            return "", ""
        pre, post = AutoIndentMode._get_indent(self, cursor)
        if self.at_block_start(cursor, line):
            return pre, post
        # return pressed in comments
        if (self._helper.is_comment_or_string(
                cursor, formats=['comment', 'docstring']) or
                fullline.endswith('"""')):
            if line.strip().startswith("#") and column != len(fullline):
                post += '# '
            return pre, post
        elif self.between_paren(cursor, column):
            try:
                pre, post = self.handle_indent_after_paren(
                    column, line, fullline, cursor)
            except TypeError:
                return pre, post
        else:
            lastword = self.get_last_word(cursor)
            inStringDef, char = self.is_in_string_def(fullline, column)
            if inStringDef:
                # the string might be between paren if multiline
                # check if there a at least a non closed paren on the previous
                # lines
                if self.has_unclosed_paren(cursor):
                    pre = char
                else:
                    pre = '" \\'
                    post += 4 * ' '
                if fullline.endswith(':'):
                    post += 4 * " "
                post += char
            elif fullline.rstrip().endswith(":") and \
                    lastword.rstrip().endswith(':') and \
                    self.at_block_end(cursor, fullline):
                try:
                    indent = (self.get_indent_of_opening_paren(cursor)
                              + 4)
                    post = indent * " "
                except TypeError:
                    # e.g indent is None (meaning the line does not ends with ):, ]: or }:
                    kw = ["if", "class", "def", "while", "for", "else", "elif",
                          "except", "finally", "try"]
                    l = fullline
                    ln = cursor.blockNumber()

                    def check_kw_in_line(kwds, lparam):
                        for kwd in kwds:
                            if kwd in lparam:
                                return True
                        return False

                    while not check_kw_in_line(kw, l) and ln:
                        ln -= 1
                        l = self._helper.line_text(ln)
                    indent = (len(l) - len(l.lstrip())) * " "
                    indent += 4 * " "
                    post = indent
            elif line.endswith("\\"):
                # increment indent
                post += 4 * " "
            elif (fullline.endswith((')', '}', ']')) and
                    lastword.endswith((')', '}', ']'))):
                # find line where the open braces can be found and align with
                # that line
                indent = self.get_indent_of_opening_paren(cursor)
                if indent is not None:
                    post = indent * " "
            elif ("\\" not in fullline and "#" not in fullline and
                  fullline.strip() and not fullline.endswith(')') and
                  not self.at_block_end(cursor, fullline)):
                if lastword and lastword[-1] != " ":
                    pre += " \\"
                else:
                    pre += '\\'
                post += 4 * " "
                if fullline.endswith(':'):
                    post += 4 * " "
            elif (lastword == "return" or lastword == "pass"):
                post = post[:-4]

        return pre, post
<commit_msg>AutoIndent: refactoring and cleanup<commit_after># -*- coding: utf-8 -*-
""" Contains smart indent modes """
import re
from pyqode.core.api import TextHelper, get_block_symbol_data
from pyqode.core.qt.QtGui import QTextCursor
from pyqode.core.modes.autoindent import AutoIndentMode
from pyqode.core.modes.matcher import SymbolMatcherMode

class PyAutoIndentMode(AutoIndentMode):
    """
    Customised :class:`pyqode.core.modes.AutoIndentMode` for python
    that tries its best to follow the pep8 indentation guidelines.

    """
    def __init__(self):
        super(PyAutoIndentMode, self).__init__()
        self._helper = None

    def on_install(self, editor):
        super().on_install(editor)
        self._helper = TextHelper(editor)

    def _get_indent(self, cursor):
        ln, column = self._helper.cursor_position()
        fullline = self._get_full_line(cursor)
        line = fullline[:column]
        pre, post = AutoIndentMode._get_indent(self, cursor)
        if self._at_block_start(cursor, line):
            return pre, post
        # return pressed in comments
        if (self._helper.is_comment_or_string(
                cursor, formats=['comment', 'docstring']) or
                fullline.endswith('"""')):
            if line.strip().startswith("#") and column != len(fullline):
                post += '# '
            return pre, post
        # between parens
        elif self._between_paren(cursor, column):
            try:
                pre, post = self._handle_indent_between_paren(
                    column, line, fullline, cursor)
            except TypeError:
                return pre, post
        else:
            lastword = self._get_last_word(cursor)
            in_string_def, char = self._is_in_string_def(fullline, column)
            if in_string_def:
                post, pre = self._handle_indent_inside_string(
                    char, cursor, fullline, post)
            elif (fullline.rstrip().endswith(":") and
                    lastword.rstrip().endswith(':') and
                    self._at_block_end(cursor, fullline)):
                post = self._handle_new_scope_indentation(
                    cursor, fullline)
            elif line.endswith("\\"):
                # if user typed \ and press enter -> indent is always
                # one level higher
                post += self.editor.tab_length * " "
            elif (fullline.endswith((')', '}', ']')) and
                    lastword.endswith((')', '}', ']'))):
                post = self._handle_indent_after_paren(cursor, post)
            elif ("\\" not in fullline and "#" not in fullline and
                  fullline.strip() and not fullline.endswith(')') and
                  not self._at_block_end(cursor, fullline)):
                # break in statement not enclose by parents
                post, pre = self._handle_indent_in_statement(
                    fullline, lastword, post, pre)
            elif lastword == "return" or lastword == "pass":
                post = post[:-self.editor.tab_length]

        return pre, post

    def _has_unclosed_paren(self, tc):
        ln = tc.blockNumber()
        while ln >= 0:
            line = self._helper.line_text(ln)
            if line.count("(") > line.count(")"):
                return True
            ln -= 1
        return False

    @staticmethod
    def _is_in_string_def(full_line, column):
        count = 0
        char = "'"
        for i in range(len(full_line)):
            if full_line[i] == "'" or full_line[i] == '"':
                count += 1
            if full_line[i] == '"' and i < column:
                char = '"'
        count_after_col = 0
        for i in range(column, len(full_line)):
            if full_line[i] == "'" or full_line[i] == '"':
                count_after_col += 1
        return count % 2 == 0 and count_after_col % 2 == 1, char

    @staticmethod
    def _is_paren_open(paren):
        return (paren.character == "(" or paren.character == "["
                or paren.character == '{')

    @staticmethod
    def _is_paren_closed(paren):
        return (paren.character == ")" or paren.character == "]"
                or paren.character == '}')

    @staticmethod
    def _get_full_line(tc):
        tc2 = QTextCursor(tc)
        tc2.select(QTextCursor.LineUnderCursor)
        full_line = tc2.selectedText()
        return full_line

    def _parens_count_for_block(self, col, block):
        open_p = []
        closed_p = []
        lists = get_block_symbol_data(block)
        for symbols in lists:
            for paren in symbols:
                if paren.position >= col:
                    continue
                if self._is_paren_open(paren):
                    if not col:
                        return -1, -1, [], []
                    open_p.append(paren)
                if self._is_paren_closed(paren):
                    closed_p.append(paren)
        return len(open_p), len(closed_p), open_p, closed_p

    def _between_paren(self, tc, col):
        block = tc.block()
        nb_open = nb_closed = 0
        while block.isValid():  # and nb_open >= nb_closed:
            o, c, _, _ = self._parens_count_for_block(col, block)
            nb_open += o
            nb_closed += c
            block = block.previous()
            col = len(block.text())
        return nb_open > nb_closed

    @staticmethod
    def _get_last_word(tc):
        tc2 = QTextCursor(tc)
        tc2.movePosition(QTextCursor.Left, 1)
        tc2.movePosition(QTextCursor.WordLeft, tc.KeepAnchor)
        return tc2.selectedText().strip()

    def _get_indent_of_opening_paren(self, tc):
        tc.movePosition(tc.Left, tc.KeepAnchor)
        char = tc.selectedText()
        tc.movePosition(tc.Right, tc.MoveAnchor)
        mapping = {')': ('(', 0), '}': ('{', 1), ']': ('[', 2)}
        try:
            character, char_type = mapping[char]
        except KeyError:
            return None
        else:
            ol, oc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
                tc, character, char_type)
            line = self._helper.line_text(ol)
            return len(line) - len(line.lstrip())

    def _get_last_open_paren_pos(self, tc, column):
        pos = None
        char = None
        ln = tc.blockNumber() + 1
        tc_trav = QTextCursor(tc)
        while ln >= 1:
            tc_trav.movePosition(tc_trav.StartOfLine, tc_trav.MoveAnchor)
            lists = get_block_symbol_data(tc_trav.block())
            for symbols in lists:
                for paren in reversed(symbols):
                    if paren.position < column:
                        if self._is_paren_open(paren):
                            if paren.position > column:
                                continue
                            else:
                                pos = tc_trav.position() + paren.position
                                char = paren.character
                                # ensure it does not have a closing paren on
                                # the same line
                                tc3 = QTextCursor(tc)
                                tc3.setPosition(pos)
                                l, c = self.editor.modes.get(
                                    SymbolMatcherMode).symbol_pos(tc3, ')')
                                if l == ln and c < column:
                                    continue
                                return pos, char
            # check previous line
            tc_trav.movePosition(tc_trav.Up, tc_trav.MoveAnchor)
            ln = tc_trav.blockNumber() + 1
            column = len(self._helper.line_text(ln))
        return pos, char

    def _get_paren_pos(self, tc, column):
        pos, char = self._get_last_open_paren_pos(tc, column)
        if char == '(':
            ptype = 0
            closingchar = ')'
        elif char == '[':
            ptype = 1
            closingchar = ']'
        else:
            ptype = 2
            closingchar = '}'
        tc2 = QTextCursor(tc)
        tc2.setPosition(pos)
        ol, oc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
            tc2, char, ptype)
        cl, cc = self.editor.modes.get(SymbolMatcherMode).symbol_pos(
            tc2, closingchar, ptype)
        return (ol, oc), (cl, cc)

    @staticmethod
    def _get_next_char(tc):
        tc2 = QTextCursor(tc)
        tc2.movePosition(QTextCursor.NextCharacter, QTextCursor.KeepAnchor)
        char = tc2.selectedText()
        return char

    def _handle_indent_between_paren(self, column, line, fullline, tc):
        """
        Handle indent between symbols such as parenthesis, braces,...
        """
        # elements might be separated by ',' 'or' 'and'
        next_char = self._get_next_char(tc)
        nextcharisclosingsymbol = next_char in [']', ')', '}']
        (ol, oc), (cl, cc) = self._get_paren_pos(tc, column)
        closingline = self._helper.line_text(cl)
        openingline = self._helper.line_text(ol)
        openingline = re.sub(r'".*"', "", openingline)
        openingline = re.sub(r"'.*'", "", openingline)
        openingindent = len(openingline) - len(openingline.lstrip())
        tokens = [t.strip() for t in re.split(', |and |or ',
                                              line[oc:column]) if t]
        # align with first token pos
        if len(closingline) > cc and closingline[cc] == ":":
            post = openingindent * " " + 8 * " "
        else:
            # press enter before a '}', ']', ')'
            # which close an affectation (tuple, list , dict)
            if nextcharisclosingsymbol and re.match('.*=[\s][\W].*',
                                                    openingline):
                post = openingindent * " "
            else:
                # align elems in list, tuple, dict
                if re.match('.*=[\s][\W].*', openingline) and ol - cl == 1:
                    post = openingindent * " " + self.editor.tab_length * " "
                # align elems in fct declaration (we align with first
                # token)
                else:
                    if len(tokens):
                        otext = TextHelper(self.editor).line_text(ol)
                        if ol == TextHelper(self.editor).current_line_nbr() or otext[-1] not in ['(', '{', '[']:
                            # align with last opened paren
                            post = oc * " "
                        else:
                            # align with first open paren
                            indent = TextHelper(self.editor).line_indent()
                            post = indent * ' '
                    else:
                        indent = (len(openingindent * " " +
                                      self.editor.tab_length * " ") //
                                  self.editor.tab_length * self.editor.tab_length)
                        post = indent * ' '
        pre = ""
        in_string_def, char = self._is_in_string_def(fullline, column)
        if in_string_def:
            pre = char
            post += char
        return pre, post

    @staticmethod
    def _at_block_start(tc, line):
        """
        Improve QTextCursor.atBlockStart to ignore spaces
        """
        if tc.atBlockStart():
            return True
        column = tc.columnNumber()
        indentation = len(line) - len(line.lstrip())
        return column <= indentation

    @staticmethod
    def _at_block_end(tc, fullline):
        if tc.atBlockEnd():
            return True
        column = tc.columnNumber()
        return column >= len(fullline.rstrip()) - 1

    def _handle_indent_inside_string(self, char, cursor, fullline, post):
        # the string might be between paren if multiline
        # check if there a at least a non closed paren on the previous
        # lines
        n = self.editor.tab_length
        if self._has_unclosed_paren(cursor):
            pre = char
        else:
            pre = '" \\'
            post += n * ' '
        if fullline.endswith(':'):
            post += n * " "
        post += char
        return post, pre

    def _handle_new_scope_indentation(self, cursor, fullline):
        try:
            indent = (self._get_indent_of_opening_paren(cursor) +
                      self.editor.tab_length)
            post = indent * " "
        except TypeError:
            # e.g indent is None (meaning the line does not ends with ):, ]: or }:
            kw = ["if", "class", "def", "while", "for", "else", "elif",
                  "except", "finally", "try"]
            l = fullline
            ln = cursor.blockNumber()

            def check_kw_in_line(kwds, lparam):
                for kwd in kwds:
                    if kwd in lparam:
                        return True
                return False

            while not check_kw_in_line(kw, l) and ln:
                ln -= 1
                l = self._helper.line_text(ln)
            indent = (len(l) - len(l.lstrip())) * " "
            indent += self.editor.tab_length * " "
            post = indent
        return post

    def _handle_indent_after_paren(self, cursor, post):
        indent = self._get_indent_of_opening_paren(cursor)
        if indent is not None:
            post = indent * " "
        return post

    def _handle_indent_in_statement(self, fullline, lastword, post, pre):
        if lastword and lastword[-1] != " ":
            pre += " \\"
        else:
            pre += '\\'
        post += self.editor.tab_length * " "
        if fullline.endswith(':'):
            post += self.editor.tab_length * " "
        return post, pre
<|endoftext|>