<commit_before>from rest_framework import serializers
from drf_extra_fields.geo_fields import PointField as PointFieldSerializer

from django.contrib.auth.models import User

from .models import Profile, Hospital, Ambulance, \
    AmbulancePermission, HospitalPermission, \
    AmbulanceUpdate

# Profile serializers

class AmbulancePermissionSerializer(serializers.ModelSerializer):

    ambulance_id = serializers.IntegerField(source='ambulance.id')
    ambulance_identifier = serializers.CharField(source='ambulance.identifier')

    class Meta:
        model = AmbulancePermission
        fields = ('ambulance_id', 'ambulance_identifier', 'can_read', 'can_write')
        read_only_fields = ('ambulance_id', 'ambulance_identifier', 'can_read', 'can_write')

class HospitalPermissionSerializer(serializers.ModelSerializer):

    hospital_id = serializers.IntegerField(source='hospital.id')
    hospital_name = serializers.CharField(source='hospital.name')

    class Meta:
        model = HospitalPermission
        fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')
        read_only_fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')

class ProfileSerializer(serializers.ModelSerializer):

    ambulances = AmbulancePermissionSerializer(read_only=True, many=True)
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances','hospitals')

# Ambulance serializers
class AmbulanceStatusSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = ('user', 'status', 'location', 'timestamp')

class AmbulanceSerializer(serializers.ModelSerializer):

    last_update = serializers.AmbulanceUpdateSerializer()

    class Meta:
        model = Ambulance
        fields = ('id', 'identifier', 'comment', 'capability',
                  'last_update')

<commit_msg>Streamlining ambulance updates<commit_after>from rest_framework import serializers
from drf_extra_fields.geo_fields import PointField as PointFieldSerializer

from django.contrib.auth.models import User

from .models import Profile, Hospital, Ambulance, \
    AmbulancePermission, HospitalPermission, \
    AmbulanceUpdate

# Profile serializers

class AmbulancePermissionSerializer(serializers.ModelSerializer):

    ambulance_id = serializers.IntegerField(source='ambulance.id')
    ambulance_identifier = serializers.CharField(source='ambulance.identifier')

    class Meta:
        model = AmbulancePermission
        fields = ('ambulance_id', 'ambulance_identifier', 'can_read', 'can_write')
        read_only_fields = ('ambulance_id', 'ambulance_identifier', 'can_read', 'can_write')

class HospitalPermissionSerializer(serializers.ModelSerializer):

    hospital_id = serializers.IntegerField(source='hospital.id')
    hospital_name = serializers.CharField(source='hospital.name')

    class Meta:
        model = HospitalPermission
        fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')
        read_only_fields = ('hospital_id', 'hospital_name', 'can_read', 'can_write')

class ProfileSerializer(serializers.ModelSerializer):

    ambulances = AmbulancePermissionSerializer(read_only=True, many=True)
    hospitals = HospitalPermissionSerializer(read_only=True, many=True)

    class Meta:
        model = Profile
        fields = ('ambulances','hospitals')

# Ambulance serializers
class AmbulanceUpdateSerializer(serializers.ModelSerializer):

    class Meta:
        model = Ambulance
        fields = ('user', 'status', 'location', 'timestamp')

class AmbulanceSerializer(serializers.ModelSerializer):

    last_update = serializers.AmbulanceUpdateSerializer()

    class Meta:
        model = Ambulance
        fields = ('id', 'identifier', 'comment', 'capability',
                  'last_update')

<|endoftext|>


<commit_before>import json

def bytes_to_lines(stream):
    """Reads single bytes from stream, emits lines.

       This hack makes me sick, but requests makes this impossible
       to do reliably, otherwise."""
    line = ""
    for byte in stream:
        line += byte
        if byte == "\n":
            yield line
            line = ""

class BaseResponse(object):
    def __init__(self, obj=None):
        self.obj = obj

    @classmethod
    def create(cls, res=None, trace_id=None, streaming=False):
        resp = None

        if streaming:
            stream = bytes_to_lines(res.iter_content(chunk_size=1))
            first_line = next(stream)
            data = json.loads(first_line)
        else:
            if res.text:
                data = json.loads(res.text)
            else:
                data = None
        if streaming:
            resp = StreamingJsonObjectResponse(obj=data['object'], stream=stream)
        elif data and 'object' in data:
            resp = ItemResponse(obj=data['object'])
        elif data and 'objects' in data:
            resp = ListResponse(obj=data['objects'])
        else:
            resp = NoItemResponse(obj=None)
        resp.trace_id = trace_id
        resp.res = res
        resp.data = data
        return resp

    def find_link(self, rel):
        for link in self.data.get('links', []):
            if link.get('rel') == rel:
                return link
        return None

class ListResponse(BaseResponse):
    @property
    def items(self):
        return self.obj

    @property
    def item(self):
        return self.obj[0]

class ItemResponse(BaseResponse):
    @property
    def items(self):
        return [self.obj]

    @property
    def item(self):
        return self.obj

class NoItemResponse(BaseResponse):
    @property
    def items(self):
        return None

    @property
    def item(self):
        return None

class StreamingJsonObjectResponse(BaseResponse):
    def __init__(self, obj, stream):
        BaseResponse.__init__(self, obj)
        self._stream = stream

    @property
    def items(self):
        def stream():
            for line in self._stream:
                yield json.loads(line)['object']
        return stream()

    @property
    def item(self):
        return self.obj
<commit_msg>Future-proof bytes_to_lines for pretty-printed JSON objects.<commit_after>import json

def bytes_to_lines(stream):
    """Reads single bytes from stream, emits lines.

       This hack makes me sick, but requests makes this impossible
       to do reliably, otherwise."""
    line = ""
    for byte in stream:
        line += byte
        if line.endswith("\r\n"):
            yield line
            line = ""

class BaseResponse(object):
    def __init__(self, obj=None):
        self.obj = obj

    @classmethod
    def create(cls, res=None, trace_id=None, streaming=False):
        resp = None

        if streaming:
            stream = bytes_to_lines(res.iter_content(chunk_size=1))
            first_line = next(stream)
            data = json.loads(first_line)
        else:
            if res.text:
                data = json.loads(res.text)
            else:
                data = None
        if streaming:
            resp = StreamingJsonObjectResponse(obj=data['object'], stream=stream)
        elif data and 'object' in data:
            resp = ItemResponse(obj=data['object'])
        elif data and 'objects' in data:
            resp = ListResponse(obj=data['objects'])
        else:
            resp = NoItemResponse(obj=None)
        resp.trace_id = trace_id
        resp.res = res
        resp.data = data
        return resp

    def find_link(self, rel):
        for link in self.data.get('links', []):
            if link.get('rel') == rel:
                return link
        return None

class ListResponse(BaseResponse):
    @property
    def items(self):
        return self.obj

    @property
    def item(self):
        return self.obj[0]

class ItemResponse(BaseResponse):
    @property
    def items(self):
        return [self.obj]

    @property
    def item(self):
        return self.obj

class NoItemResponse(BaseResponse):
    @property
    def items(self):
        return None

    @property
    def item(self):
        return None

class StreamingJsonObjectResponse(BaseResponse):
    def __init__(self, obj, stream):
        BaseResponse.__init__(self, obj)
        self._stream = stream

    @property
    def items(self):
        def stream():
            for line in self._stream:
                yield json.loads(line)['object']
        return stream()

    @property
    def item(self):
        return self.obj
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import json
import random
import hashlib

import requests
from heartbeat import Challenge, Heartbeat

from .utils import urlify
from .exc import DownstreamError

class DownstreamClient(object):
    def __init__(self, server_url):
        self.server = server_url.strip('/')
        self.challenges = []
        self.heartbeat = None

    def connect(self, url):
        raise NotImplementedError

    def store_path(self, path):
        raise NotImplementedError

    def get_chunk(self, hash):
        raise NotImplementedError

    def challenge(self, hash, challenge):
        raise NotImplementedError

    def answer(self, hash, hash_answer):
        raise NotImplementedError

    def _enc_fname(self, filename):
        return urlify(os.path.split(filename)[1])

    def get_challenges(self, filename):
        enc_fname = urlify(os.path.split(filename)[1])
        url = '%s/api/downstream/challenge/%s' % (self.server, enc_fname)
        resp = requests.get(url)
        try:
            resp.raise_for_status()
        except Exception as e:
            raise DownstreamError("Error connecting to downstream"
                                  "-node:", e.message)
        _json = resp.json()

        for challenge in _json['challenges']:
            chal = Challenge(challenge.get('block'), challenge.get('seed'))
            self.challenges.append(chal)

    def answer_challenge(self, filename):
        enc_fname = self._enc_fname(filename)
        raise NotImplementedError

    def random_challenge(self):
        random.choice(self.challenges)<commit_msg>Assert filename is a valid file<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import json
import random
import hashlib

import requests
from heartbeat import Challenge, Heartbeat

from .utils import urlify
from .exc import DownstreamError

class DownstreamClient(object):
    def __init__(self, server_url):
        self.server = server_url.strip('/')
        self.challenges = []
        self.heartbeat = None

    def connect(self, url):
        raise NotImplementedError

    def store_path(self, path):
        raise NotImplementedError

    def get_chunk(self, hash):
        raise NotImplementedError

    def challenge(self, hash, challenge):
        raise NotImplementedError

    def answer(self, hash, hash_answer):
        raise NotImplementedError

    def _enc_fname(self, filename):
        return urlify(os.path.split(filename)[1])

    def get_challenges(self, filename):
        enc_fname = urlify(os.path.split(filename)[1])
        url = '%s/api/downstream/challenge/%s' % (self.server, enc_fname)
        resp = requests.get(url)
        try:
            resp.raise_for_status()
        except Exception as e:
            raise DownstreamError("Error connecting to downstream"
                                  "-node:", e.message)
        _json = resp.json()

        for challenge in _json['challenges']:
            chal = Challenge(challenge.get('block'), challenge.get('seed'))
            self.challenges.append(chal)

    def answer_challenge(self, filename):
        try:
            assert os.path.isfile(filename)
        except AssertionError:
            raise DownstreamError('%s is not a valid file' % filename)

        enc_fname = self._enc_fname(filename)
        raise NotImplementedError

    def random_challenge(self):
        random.choice(self.challenges)<|endoftext|>


<commit_before>from __future__ import absolute_import

import gzip
import shutil
import StringIO
import tempfile

from django.core.cache import cache
from django.core.files import File
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.utils import unittest

from .models import Storage, temp_storage, temp_storage_location

class FileStorageTests(TestCase):
    def tearDown(self):
        shutil.rmtree(temp_storage_location)

    def test_files(self):
        temp_storage.save('tests/default.txt', ContentFile('default content'))
        # Attempting to access a FileField from the class raises a descriptive
        # error
        self.assertRaises(AttributeError, lambda: Storage.normal)

        # An object without a file has limited functionality.
        obj1 = Storage()
        self.assertEqual(obj1.normal.name, "")
        self.assertRaises(ValueError, lambda: obj1.normal.size)

        # Saving a file enables full functionality.
        obj1.normal.save("django_test.txt", ContentFile("content"))
        self.assertEqual(obj1.normal.name, "tests/django_test.txt")
        self.assertEqual(obj1.normal.size, 7)
        self.assertEqual(obj1.normal.read(), b"content")
        obj1.normal.close()

        # File objects can be assigned to FileField attributes, but shouldn't
        # get committed until the model it's attached to is saved.
        obj1.normal = SimpleUploadedFile("assignment.txt", b"content")
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(dirs, [])
        self.assertEqual(sorted(files), ["default.txt", "django_test.txt"])

        obj1.save()
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(
            sorted(files), ["assignment.txt", "default.txt", "django_test.txt"]
        )

        # Files can be read in a little at a time, if necessary.
        obj1.normal.open()
        self.assertEqual(obj1.normal.read(3), b"con")
        self.assertEqual(obj1.normal.read(), b"tent")
        self.assertEqual(list(obj1.normal.chunks(chunk_size=2)), [b"co", b"nt", b"en", b"t"])
        obj1.normal.close()

        # Save another file with the same name.
        obj2 = Storage()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        self.assertEqual(obj2.normal.name, "tests/django_test_1.txt")
        self.assertEqual(obj2.normal.size, 12)

        # Push the objects into the cache to make sure they pickle properly
        cache.set("obj1", obj1)
        cache.set("obj2", obj2)
        self.assertEqual(cache.get("obj2").normal.name, "tests/django_test_1.txt")

        # Deleting an object does not delete the file it uses.
        obj2.delete()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        self.assertEqual(obj2.normal.name, "tests/django_test_2.txt")

        # Multiple files with the same name get _N appended to them.
        objs = [Storage() for i in range(3)]
        for o in objs:
            o.normal.save("multiple_files.txt", ContentFile("Same Content"))
        self.assertEqual(
            [o.normal.name for o in objs],
            ["tests/multiple_files.txt", "tests/multiple_files_1.txt", "tests/multiple_files_2.txt"]
        )
        for o in objs:
            o.delete()

        # Default values allow an object to access a single file.
        obj3 = Storage.objects.create()
        self.assertEqual(obj3.default.name, "tests/default.txt")
        self.assertEqual(obj3.default.read(), b"default content")
        obj3.default.close()

        # But it shouldn't be deleted, even if there are no more objects using
        # it.
        obj3.delete()
        obj3 = Storage()
        self.assertEqual(obj3.default.read(), b"default content")
        obj3.default.close()

        # Verify the fix for #5655, making sure the directory is only
        # determined once.
        obj4 = Storage()
        obj4.random.save("random_file", ContentFile("random content"))
        self.assertTrue(obj4.random.name.endswith("/random_file"))

    def test_file_object(self):
        # Create sample file
        temp_storage.save('tests/example.txt', ContentFile('some content'))

        # Load it as python file object
        file_obj = open(temp_storage.path('tests/example.txt'))

        # Save it using storage and read its content
        temp_storage.save('tests/file_obj', file_obj)
        self.assertTrue(temp_storage.exists('tests/file_obj'))
        self.assertEqual(
            temp_storage.open('tests/file_obj').read(),
            'some content')

    def test_stringio(self):
        # Test passing StringIO instance as content argument to save
        output = StringIO.StringIO()
        output.write('content')
        output.seek(0)

        # Save it and read written file
        temp_storage.save('tests/stringio', output)
        self.assertTrue(temp_storage.exists('tests/stringio'))
        self.assertEqual(
            temp_storage.open('tests/stringio').read(),
            'content')

class FileTests(unittest.TestCase):
    def test_context_manager(self):
        orig_file = tempfile.TemporaryFile()
        base_file = File(orig_file)
        with base_file as f:
            self.assertIs(base_file, f)
            self.assertFalse(f.closed)
        self.assertTrue(f.closed)
        self.assertTrue(orig_file.closed)

    def test_file_mode(self):
        # Should not set mode to None if it is not present.
        # See #14681, stdlib gzip module crashes if mode is set to None
        file = SimpleUploadedFile("mode_test.txt", b"content")
        self.assertFalse(hasattr(file, 'mode'))
        g = gzip.GzipFile(fileobj=file)
<commit_msg>Fixed errors introduced in 664855b74e1f417384dc7aef35a3df5f86db52d6<commit_after>from __future__ import absolute_import

import gzip
import shutil
import tempfile

from django.core.cache import cache
from django.core.files import File
from django.core.files.base import ContentFile
from django.core.files.uploadedfile import SimpleUploadedFile
from django.test import TestCase
from django.utils import unittest
from django.utils.six import StringIO

from .models import Storage, temp_storage, temp_storage_location

class FileStorageTests(TestCase):
    def tearDown(self):
        shutil.rmtree(temp_storage_location)

    def test_files(self):
        temp_storage.save('tests/default.txt', ContentFile('default content'))
        # Attempting to access a FileField from the class raises a descriptive
        # error
        self.assertRaises(AttributeError, lambda: Storage.normal)

        # An object without a file has limited functionality.
        obj1 = Storage()
        self.assertEqual(obj1.normal.name, "")
        self.assertRaises(ValueError, lambda: obj1.normal.size)

        # Saving a file enables full functionality.
        obj1.normal.save("django_test.txt", ContentFile("content"))
        self.assertEqual(obj1.normal.name, "tests/django_test.txt")
        self.assertEqual(obj1.normal.size, 7)
        self.assertEqual(obj1.normal.read(), b"content")
        obj1.normal.close()

        # File objects can be assigned to FileField attributes, but shouldn't
        # get committed until the model it's attached to is saved.
        obj1.normal = SimpleUploadedFile("assignment.txt", b"content")
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(dirs, [])
        self.assertEqual(sorted(files), ["default.txt", "django_test.txt"])

        obj1.save()
        dirs, files = temp_storage.listdir("tests")
        self.assertEqual(
            sorted(files), ["assignment.txt", "default.txt", "django_test.txt"]
        )

        # Files can be read in a little at a time, if necessary.
        obj1.normal.open()
        self.assertEqual(obj1.normal.read(3), b"con")
        self.assertEqual(obj1.normal.read(), b"tent")
        self.assertEqual(list(obj1.normal.chunks(chunk_size=2)), [b"co", b"nt", b"en", b"t"])
        obj1.normal.close()

        # Save another file with the same name.
        obj2 = Storage()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        self.assertEqual(obj2.normal.name, "tests/django_test_1.txt")
        self.assertEqual(obj2.normal.size, 12)

        # Push the objects into the cache to make sure they pickle properly
        cache.set("obj1", obj1)
        cache.set("obj2", obj2)
        self.assertEqual(cache.get("obj2").normal.name, "tests/django_test_1.txt")

        # Deleting an object does not delete the file it uses.
        obj2.delete()
        obj2.normal.save("django_test.txt", ContentFile("more content"))
        self.assertEqual(obj2.normal.name, "tests/django_test_2.txt")

        # Multiple files with the same name get _N appended to them.
        objs = [Storage() for i in range(3)]
        for o in objs:
            o.normal.save("multiple_files.txt", ContentFile("Same Content"))
        self.assertEqual(
            [o.normal.name for o in objs],
            ["tests/multiple_files.txt", "tests/multiple_files_1.txt", "tests/multiple_files_2.txt"]
        )
        for o in objs:
            o.delete()

        # Default values allow an object to access a single file.
        obj3 = Storage.objects.create()
        self.assertEqual(obj3.default.name, "tests/default.txt")
        self.assertEqual(obj3.default.read(), b"default content")
        obj3.default.close()

        # But it shouldn't be deleted, even if there are no more objects using
        # it.
        obj3.delete()
        obj3 = Storage()
        self.assertEqual(obj3.default.read(), b"default content")
        obj3.default.close()

        # Verify the fix for #5655, making sure the directory is only
        # determined once.
        obj4 = Storage()
        obj4.random.save("random_file", ContentFile("random content"))
        self.assertTrue(obj4.random.name.endswith("/random_file"))

    def test_file_object(self):
        # Create sample file
        temp_storage.save('tests/example.txt', ContentFile('some content'))

        # Load it as python file object
        file_obj = open(temp_storage.path('tests/example.txt'))

        # Save it using storage and read its content
        temp_storage.save('tests/file_obj', file_obj)
        self.assertTrue(temp_storage.exists('tests/file_obj'))
        self.assertEqual(
            temp_storage.open('tests/file_obj').read(),
            b'some content')

    def test_stringio(self):
        # Test passing StringIO instance as content argument to save
        output = StringIO()
        output.write('content')
        output.seek(0)

        # Save it and read written file
        temp_storage.save('tests/stringio', output)
        self.assertTrue(temp_storage.exists('tests/stringio'))
        self.assertEqual(
            temp_storage.open('tests/stringio').read(),
            b'content')

class FileTests(unittest.TestCase):
    def test_context_manager(self):
        orig_file = tempfile.TemporaryFile()
        base_file = File(orig_file)
        with base_file as f:
            self.assertIs(base_file, f)
            self.assertFalse(f.closed)
        self.assertTrue(f.closed)
        self.assertTrue(orig_file.closed)

    def test_file_mode(self):
        # Should not set mode to None if it is not present.
        # See #14681, stdlib gzip module crashes if mode is set to None
        file = SimpleUploadedFile("mode_test.txt", b"content")
        self.assertFalse(hasattr(file, 'mode'))
        g = gzip.GzipFile(fileobj=file)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import, division, print_function, unicode_literals,
)

import logging
import os
import sys
from threading import Event, Thread

import jupyter_core.paths
from ipython_genutils.tempdir import TemporaryDirectory
from nose.plugins.attrib import attr as nose_attr
from nose.plugins.skip import SkipTest
from notebook.notebookapp import NotebookApp
from notebook.tests.launchnotebook import NotebookTestBase
from tornado.ioloop import IOLoop
from traitlets.config import Config
from traitlets.config.application import LevelFormatter

import themysto.install

try:
    from unittest.mock import patch  # py3
except ImportError:
    from mock import patch  # py2

no_selenium = True
try:
    from selenium import webdriver
except ImportError:
    pass
else:
    no_selenium = False
    from selenium.common.exceptions import TimeoutException
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support import expected_conditions as ec
    from selenium.webdriver.support.ui import WebDriverWait

def get_logger(name=__name__, log_level=logging.INFO):
    """
    Return a logger for use in install/uninstall functions.

    Adapted from
        tratilets.config.application.Application._log_default
    """
    log = logging.getLogger(name)
    log.setLevel(log_level)
    log.propagate = False
    _log = log  # copied from Logger.hasHandlers() (new in Python 3.2)
    while _log:
        if _log.handlers:
            return log
        if not _log.propagate:
            break
        else:
            _log = _log.parent
    if sys.executable.endswith('pythonw.exe'):
        # this should really go to a file, but file-logging is only
        # hooked up in parallel applications
        _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
    else:
        _log_handler = logging.StreamHandler()
    _log_formatter = LevelFormatter(
        fmt='%(message)s', datefmt="%Y-%m-%d %H:%M:%S")
    _log_handler.setFormatter(_log_formatter)
    log.addHandler(_log_handler)
    return log

class NbextensionTestBase(NotebookTestBase):
    """
    Base class for nbextensions test case classes.

    We override the setup_class method from NotebookTestBase in order to
    install themysto, and also to set log_level to debug.
    Also split some of the setup_class method into separate methods in order to
    simplify subclassing.
    """
    config = Config(NotebookApp={'log_level': logging.DEBUG})

    # this is added for notebook < 4.1, where it wasn't defined
    if not hasattr(NotebookTestBase, 'url_prefix'):
        url_prefix = '/a%40b/'

    @classmethod
    def pre_server_setup(cls):
        """Setup a temporary environment in which to run a notebook server."""
        cls.config_dir = TemporaryDirectory()
        cls.data_dir = TemporaryDirectory()
        cls.home_dir = TemporaryDirectory()
        cls.notebook_dir = TemporaryDirectory()
        cls.runtime_dir = TemporaryDirectory()

        cls.env_patch = patch.dict('os.environ', {
            'HOME': cls.home_dir.name,
            'IPYTHONDIR': os.path.join(cls.home_dir.name, '.ipython'),
            'JUPYTER_DATA_DIR': cls.data_dir.name
        })
        cls.env_patch.start()

        cls.path_patch = patch.object(
            jupyter_core.paths, 'SYSTEM_JUPYTER_PATH', [])
        cls.path_patch.start()

        # added to install themysto!
        logger = get_logger(
            name='themysto.install.install', log_level=logging.DEBUG)
        themysto.install.install(config_dir=cls.config_dir.name, logger=logger)

    @classmethod
    def start_server_thread(cls, started_event):
        """
        Start a notebook server in a separate thread.

        The start is signalled using the passed Event instance.
        """
        app = cls.notebook = NotebookApp(
            port=cls.port,
            port_retries=0,
            open_browser=False,
            config_dir=cls.config_dir.name,
            data_dir=cls.data_dir.name,
            runtime_dir=cls.runtime_dir.name,
            notebook_dir=cls.notebook_dir.name,
            base_url=cls.url_prefix,
            config=cls.config,
        )
        # don't register signal handler during tests
        app.init_signal = lambda: None
        # clear log handlers and propagate to root for nose to capture it.
        # Notebook version does this before and after initialize, which
        # resets logging handlers. However, logs about server extension
        # loading occur during the initialize call, and as such are missed
        # if we reset logging before calling initialize, for some reason I
        # don't fully understand. So, we just reset after initialize,
        # while adding a log note to explain why logs stop.
        # app.log.propagate = True
        # app.log.handlers = []
        app.initialize(argv=[])
        app.log.info(
            'Switching logging off, letting nose capture the rest.')
        app.log.propagate = True
        app.log.handlers = []
        loop = IOLoop.current()
        loop.add_callback(started_event.set)
        try:
            app.start()
        finally:
            # set the event, so failure to start doesn't cause a hang
            started_event.set()
            # app.session_manager.close call was added after notebook 4.0
            if hasattr(app.session_manager, 'close'):
                app.session_manager.close()

    @classmethod
    def setup_class(cls):
        """Install themysto, & setup a notebook server in a separate thread."""
        cls.pre_server_setup()
        started = Event()
        cls.notebook_thread = Thread(
            target=cls.start_server_thread, args=[started])
        cls.notebook_thread.start()
        started.wait()
        cls.wait_until_alive()

@nose_attr('uses_selenium')
class SeleniumNbextensionTestBase(NbextensionTestBase):

    @classmethod
    def setup_class(cls):
        if no_selenium:
            raise SkipTest('Selenium not installed.'
                           'Skipping selenium-based test.')
        super(SeleniumNbextensionTestBase, cls).setup_class()

        if os.environ.get('CI'):
            username = os.environ['SAUCE_USERNAME']
            access_key = os.environ['SAUCE_ACCESS_KEY']
            capabilities = {
                # 'platform': 'Mac OS X 10.9',
                'browserName': 'firefox',
                'version': 'latest',
                'tags': [os.environ['TOXENV'], 'CI'],
            }
            hub_url = 'http://{}:{}@ondemand.saucelabs.com:80/wd/hub'.format(
                username, access_key)
            if os.environ.get('TRAVIS'):
                # see https://docs.travis-ci.com/user/gui-and-headless-browsers
                # and https://docs.travis-ci.com/user/sauce-connect
                capabilities.update({
                    'tunnel-identifier': os.environ['TRAVIS_JOB_NUMBER'],
                    'build': os.environ['TRAVIS_BUILD_NUMBER'],
                })
            cls.driver = webdriver.Remote(
                desired_capabilities=capabilities, command_executor=hub_url)
        else:
            # local test
            cls.driver = webdriver.Firefox()

    @classmethod
    def teardown_class(cls):
        cls.driver.close()
        super(SeleniumNbextensionTestBase, cls).teardown_class()

    def wait_for_selector(self, css_selector, message='', timeout=5):
        """WebDriverWait for a selector to appear, fail test on timeout."""
        try:
            WebDriverWait(self.driver, 5).until(
                ec.presence_of_element_located((
                    By.CSS_SELECTOR, css_selector)))
        except TimeoutException:
            if message:
                message += '\n'
            self.fail(
                '{}No element matching selector {!r} found in {}s'.format(
                    message, css_selector, timeout))
<commit_msg>turns out you have to specify OS on sauce labs<commit_after># -*- coding: utf-8 -*-

from __future__ import (
    absolute_import, division, print_function, unicode_literals,
)

import logging
import os
import sys
from threading import Event, Thread

import jupyter_core.paths
from ipython_genutils.tempdir import TemporaryDirectory
from nose.plugins.attrib import attr as nose_attr
from nose.plugins.skip import SkipTest
from notebook.notebookapp import NotebookApp
from notebook.tests.launchnotebook import NotebookTestBase
from tornado.ioloop import IOLoop
from traitlets.config import Config
from traitlets.config.application import LevelFormatter

import themysto.install

try:
    from unittest.mock import patch  # py3
except ImportError:
    from mock import patch  # py2

no_selenium = True
try:
    from selenium import webdriver
except ImportError:
    pass
else:
    no_selenium = False
    from selenium.common.exceptions import TimeoutException
    from selenium.webdriver.common.by import By
    from selenium.webdriver.support import expected_conditions as ec
    from selenium.webdriver.support.ui import WebDriverWait

def get_logger(name=__name__, log_level=logging.INFO):
    """
    Return a logger for use in install/uninstall functions.

    Adapted from
        tratilets.config.application.Application._log_default
    """
    log = logging.getLogger(name)
    log.setLevel(log_level)
    log.propagate = False
    _log = log  # copied from Logger.hasHandlers() (new in Python 3.2)
    while _log:
        if _log.handlers:
            return log
        if not _log.propagate:
            break
        else:
            _log = _log.parent
    if sys.executable.endswith('pythonw.exe'):
        # this should really go to a file, but file-logging is only
        # hooked up in parallel applications
        _log_handler = logging.StreamHandler(open(os.devnull, 'w'))
    else:
        _log_handler = logging.StreamHandler()
    _log_formatter = LevelFormatter(
        fmt='%(message)s', datefmt="%Y-%m-%d %H:%M:%S")
    _log_handler.setFormatter(_log_formatter)
    log.addHandler(_log_handler)
    return log

class NbextensionTestBase(NotebookTestBase):
    """
    Base class for nbextensions test case classes.

    We override the setup_class method from NotebookTestBase in order to
    install themysto, and also to set log_level to debug.
    Also split some of the setup_class method into separate methods in order to
    simplify subclassing.
    """
    config = Config(NotebookApp={'log_level': logging.DEBUG})

    # this is added for notebook < 4.1, where it wasn't defined
    if not hasattr(NotebookTestBase, 'url_prefix'):
        url_prefix = '/a%40b/'

    @classmethod
    def pre_server_setup(cls):
        """Setup a temporary environment in which to run a notebook server."""
        cls.config_dir = TemporaryDirectory()
        cls.data_dir = TemporaryDirectory()
        cls.home_dir = TemporaryDirectory()
        cls.notebook_dir = TemporaryDirectory()
        cls.runtime_dir = TemporaryDirectory()

        cls.env_patch = patch.dict('os.environ', {
            'HOME': cls.home_dir.name,
            'IPYTHONDIR': os.path.join(cls.home_dir.name, '.ipython'),
            'JUPYTER_DATA_DIR': cls.data_dir.name
        })
        cls.env_patch.start()

        cls.path_patch = patch.object(
            jupyter_core.paths, 'SYSTEM_JUPYTER_PATH', [])
        cls.path_patch.start()

        # added to install themysto!
        logger = get_logger(
            name='themysto.install.install', log_level=logging.DEBUG)
        themysto.install.install(config_dir=cls.config_dir.name, logger=logger)

    @classmethod
    def start_server_thread(cls, started_event):
        """
        Start a notebook server in a separate thread.

        The start is signalled using the passed Event instance.
        """
        app = cls.notebook = NotebookApp(
            port=cls.port,
            port_retries=0,
            open_browser=False,
            config_dir=cls.config_dir.name,
            data_dir=cls.data_dir.name,
            runtime_dir=cls.runtime_dir.name,
            notebook_dir=cls.notebook_dir.name,
            base_url=cls.url_prefix,
            config=cls.config,
        )
        # don't register signal handler during tests
        app.init_signal = lambda: None
        # clear log handlers and propagate to root for nose to capture it.
        # Notebook version does this before and after initialize, which
        # resets logging handlers. However, logs about server extension
        # loading occur during the initialize call, and as such are missed
        # if we reset logging before calling initialize, for some reason I
        # don't fully understand. So, we just reset after initialize,
        # while adding a log note to explain why logs stop.
        # app.log.propagate = True
        # app.log.handlers = []
        app.initialize(argv=[])
        app.log.info(
            'Switching logging off, letting nose capture the rest.')
        app.log.propagate = True
        app.log.handlers = []
        loop = IOLoop.current()
        loop.add_callback(started_event.set)
        try:
            app.start()
        finally:
            # set the event, so failure to start doesn't cause a hang
            started_event.set()
            # app.session_manager.close call was added after notebook 4.0
            if hasattr(app.session_manager, 'close'):
                app.session_manager.close()

    @classmethod
    def setup_class(cls):
        """Install themysto, & setup a notebook server in a separate thread."""
        cls.pre_server_setup()
        started = Event()
        cls.notebook_thread = Thread(
            target=cls.start_server_thread, args=[started])
        cls.notebook_thread.start()
        started.wait()
        cls.wait_until_alive()

@nose_attr('uses_selenium')
class SeleniumNbextensionTestBase(NbextensionTestBase):

    @classmethod
    def setup_class(cls):
        if no_selenium:
            raise SkipTest('Selenium not installed.'
                           'Skipping selenium-based test.')
        super(SeleniumNbextensionTestBase, cls).setup_class()

        if os.environ.get('CI'):
            username = os.environ['SAUCE_USERNAME']
            access_key = os.environ['SAUCE_ACCESS_KEY']
            capabilities = {
                # 'platform': 'Mac OS X 10.9',
                'platform': 'Linux',
                'browserName': 'firefox',
                'version': 'latest',
                'tags': [os.environ['TOXENV'], 'CI'],
            }
            hub_url = 'http://{}:{}@ondemand.saucelabs.com:80/wd/hub'.format(
                username, access_key)
            if os.environ.get('TRAVIS'):
                # see https://docs.travis-ci.com/user/gui-and-headless-browsers
                # and https://docs.travis-ci.com/user/sauce-connect
                capabilities.update({
                    'tunnel-identifier': os.environ['TRAVIS_JOB_NUMBER'],
                    'build': os.environ['TRAVIS_BUILD_NUMBER'],
                })
            cls.driver = webdriver.Remote(
                desired_capabilities=capabilities, command_executor=hub_url)
        else:
            # local test
            cls.driver = webdriver.Firefox()

    @classmethod
    def teardown_class(cls):
        cls.driver.close()
        super(SeleniumNbextensionTestBase, cls).teardown_class()

    def wait_for_selector(self, css_selector, message='', timeout=5):
        """WebDriverWait for a selector to appear, fail test on timeout."""
        try:
            WebDriverWait(self.driver, 5).until(
                ec.presence_of_element_located((
                    By.CSS_SELECTOR, css_selector)))
        except TimeoutException:
            if message:
                message += '\n'
            self.fail(
                '{}No element matching selector {!r} found in {}s'.format(
                    message, css_selector, timeout))
<|endoftext|>


<commit_before>from __future__ import absolute_import
import itertools, sys

def minmax(*args):
  min = None
  max = None
  for x in args:
    if max < x:
      max = x
    if x > min:
      min = x
  return min, max

def sliceout(sequence, start, end=None):
  if end is None:
    end = start + 1
  return sequence[:start] + sequence[end:]

def issubset(iterable, set):
  return all(itertools.imap(set.__contains__, iterable))

def rdict(d):
  if isinstance(d, dict):
    d = d.iteritems()
  if __debug__:
    d = frozenset(d)
  rdict = {v: k for k, v in d}
  assert len(rdict) == len(d) # future keys should be unique
  return rdict

def min_index(*args, **kwargs):
  key = kwargs.get('key')
  kwargs['key'] = args.__getitem__ if key is None else lambda idx: key(args[idx])
  return min(*xrange(len(args)), **kwargs)

__openspecial_names = {'/dev/std' + s: getattr(sys, 'std' + s) for s in ('in', 'out', 'err')}

def openspecial(path, mode='r', *args):
  if path == '-':
    return sys.stdout if 'w' in mode else sys.stdin
  else:
    f = __openspecial_names.get(path)
    return open(path, mode, *args) if f is None else f
<commit_msg>Unicode utilities<commit_after>from __future__ import absolute_import
import itertools, sys, locale, unicodedata

infinity = float('inf')

def minmax(*args):
  min = None
  max = None
  for x in args:
    if max < x:
      max = x
    if x > min:
      min = x
  return min, max

def sliceout(sequence, start, end=None):
  if end is None:
    end = start + 1
  return sequence[:start] + sequence[end:]

def issubset(iterable, set):
  return all(itertools.imap(set.__contains__, iterable))

def rdict(d):
  if isinstance(d, dict):
    d = d.iteritems()
  if __debug__:
    d = frozenset(d)
  rdict = {v: k for k, v in d}
  assert len(rdict) == len(d) # future keys should be unique
  return rdict

def min_index(*args, **kwargs):
  key = kwargs.get('key')
  kwargs['key'] = args.__getitem__ if key is None else lambda idx: key(args[idx])
  return min(*xrange(len(args)), **kwargs)

__openspecial_names = {'/dev/std' + s: getattr(sys, 'std' + s) for s in ('in', 'out', 'err')}

def openspecial(path, mode='r', *args):
  if path == '-':
    return sys.stdout if 'w' in mode else sys.stdin
  else:
    f = __openspecial_names.get(path)
    return open(path, mode, *args) if f is None else f

def isprint(c):
  return unicodedata.category(c)[0] not in 'CZ'

def char_repr(c):
  if isprint(c):
    assert len(format(c, ''))
    return c
  else:
    code = ord(c)
    return \
      ('\\u{:04x}' if isinstance(c, unicode) and code >= 0x80 else '\\x{:02x}') \
        .format(code)

class DecodableUnicode(unicode):

  default_encoding = locale.getpreferredencoding()

  def __new__(cls, s, encoding=default_encoding, *args):
    return s if isinstance(s, unicode) else unicode.__new__(cls, s, encoding, *args)

  def decode(self, *args):
    return self
<|endoftext|>


<commit_before>
# coding: utf-8

# In[22]:

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import re
import time
import easygui as e
get_ipython().magic(u'matplotlib inline')

#Data import via pandas
try:
    toggle_preData = pd.read_csv("Toggl_projects_2016-01-01_to_2016-12-31.csv")
except IOError:
    e.msgbox("Could not read file 2016 Toggl CSV file. Please re-download that file and put it in the same directory as this Python script.", "Error")
    exit()

try:
    toggle_data = pd.read_csv("Toggl_projects_2017-01-01_to_2017-12-31.csv")
except IOError:
    e.msgbox("Could not read file 2016 Toggl CSV file. Please re-download that file and put it in the same directory as this Python script.", "Error")
    exit()

toggle_data.columns=['Project type', 'Project', 'Hours', 'Resources', 'Private/public']
toggle_preData.columns=['Project type', 'Project', 'Hours', 'Resources', 'Private/public']
#Define the finite amount of Project Types
projectStrings = ["ODD", "O&G", "Oil & Gas", "Reg A", "Essential", "Combined", "Interval Fund",
                  "BDC", "DST", "Conservation Easement","Fund", "REIT", "Energy"]
statusStrings = ["Private", "Public", "NA"]
resourceStrings = ["Internal", "External", "NA"]
##
#Create function to iterate through projectStrings (Project Types),
#and determine if it is a substring of the project title.
#Example: `DST` is a substring of `(final) GK Hy-Vee DST (private or exempt),`
#so then it returns the project type `DST.`
##
def propertyType(str, client):
    for x in range(0, len(projectStrings)):
        if projectStrings[x].lower() in str.lower():
            if projectStrings[x] == "Reg A":
                if "ASMX".lower() in client.lower() or "FundAmerica".lower() in client.lower():
                    return (projectStrings[x]+"+ ASMX/FA")
                else:
                    return (projectStrings[x]+"+ nonASMX/FA")
            if "nontraded" in str.lower() or "non-traded" in str.lower():
                return "Non-traded " + projectStrings[x]
            return projectStrings[x]
    return "NA"
##
#Create function to strip numbers for the HH:MM:SS format
#and convert this into decimal format by hours
##
def convertHours(str):
    hour = float()
    str_list = str.split(':')
    float_list = [float(x) for x in str_list]
    for x in range(0, len(float_list)):
        hour += float_list[x]/(60**float(x))
    return hour

##
#
#
##
def statusType(str):
    for x in range(0, len(statusStrings)):
        if statusStrings[x].lower() in str.lower():
            return statusStrings[x]
    return "NA"

##
#
#
##
def resourceType(str):
    if "ODD".lower() in str.lower() or "Combined".lower() in str.lower():
        return "NA"
    elif ("(IC)").lower() in str.lower():
        return "External"
    else:
        return "Internal"

##
#
#
##
def concatenateHours(str):
    for x in range(0, len(toggle_preData['Project'])):
        if str == toggle_preData['Project'][x]:
            return convertHours(toggle_preData['Hours'][x])
    return 0
##
#
#
##
for x in range(0, len(toggle_data['Project'])):
    toggle_data['Project type'][x] = propertyType(toggle_data['Project'][x],
                                                  toggle_data['Project type'][x])
    toggle_data['Hours'][x] = convertHours(toggle_data['Hours'][x]) + concatenateHours(toggle_data['Project'][x])
    toggle_data['Private/public'][x] = statusType(toggle_data['Project'][x])
    toggle_data['Resources'][x] = resourceType(toggle_data['Project'][x])

toggle_data
##
#Data export function, retains only `Project` and `Registered time` columns
#and overwrites previous file
##
toggle_base_url = "Toggl_projects_"
toggle_timestamp = str(time.strftime("%m-%d-%y"))
toggle_filetype = ".csv"
toggle_url = toggle_base_url + toggle_timestamp + toggle_filetype
toggle_data[['Project', 'Hours', 'Project type', 'Resources', 'Private/public']].to_csv(toggle_url,index=False)

<commit_msg>Second commit<commit_after>
# coding: utf-8

# In[14]:

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import re
import time
import easygui as e
import os
import sys
get_ipython().magic(u'matplotlib inline')

##
#Attempts to data import via pandas, if the file does not exist, then error prompt
#This operation occurs for both 2016 and 2017 data sets
##

def dataLoad():
    try:
        global toggle_preData
        toggle_preData = pd.read_csv("Toggl_projects_2016-01-01_to_2016-12-31.csv")
        global toggle_data
        toggle_data = pd.read_csv("Toggl_projects_2017-01-01_to_2017-12-31.csv")
        toggle_data.columns=['Project type', 'Project', 'Hours', 'Resources', 'Private/public']
        toggle_preData.columns=['Project type', 'Project', 'Hours', 'Resources', 'Private/public']
        #Define the finite amount of Project Types
        global projectStrings
        projectStrings = ["ODD", "O&G", "Oil & Gas", "Reg A", "Essential", "Combined", "Interval Fund",
                          "BDC", "DST", "Conservation Easement","Fund", "REIT", "Energy"]
        global statusStrings
        statusStrings = ["Private", "Public", "NA"]
        global resourceStrings
        resourceStrings = ["Internal", "External", "NA"]
    except IOError:
        e.msgbox("Could not read file Toggl CSV file. Please re-download that file and put it in the same directory as this Python script.", "Error")
        sys.exit()

##
#Iterates through the 2017 data, and populates data given from the functions above
##
def dataFormat():
    for x in range(0, len(toggle_data['Project'])):
        toggle_data['Project type'][x] = propertyType(toggle_data['Project'][x],
                                                      toggle_data['Project type'][x])
        toggle_data['Hours'][x] = convertHours(toggle_data['Hours'][x]) + concatenateHours(toggle_data['Project'][x])
        toggle_data['Private/public'][x] = statusType(toggle_data['Project'][x])
        toggle_data['Resources'][x] = resourceType(toggle_data['Project'][x])

##
#Create function to iterate through projectStrings (Project Types),
#and determine if it is a substring of the project title.
#Example: `DST` is a substring of `(final) GK Hy-Vee DST (private or exempt),`
#so then it returns the project type `DST.`
##
def propertyType(str, client):
    for x in range(0, len(projectStrings)):
        if projectStrings[x].lower() in str.lower():
            if projectStrings[x] == "Reg A":
                if "ASMX".lower() in client.lower() or "FundAmerica".lower() in client.lower():
                    return (projectStrings[x]+"+ ASMX/FA")
                else:
                    return (projectStrings[x]+"+ nonASMX/FA")
            if "nontraded" in str.lower() or "non-traded" in str.lower():
                return "Non-traded " + projectStrings[x]
            return projectStrings[x]
    return "NA"
##
#Create function to strip numbers for the HH:MM:SS format
#and convert this into decimal format by hours
##
def convertHours(str):
    hour = float()
    str_list = str.split(':')
    float_list = [float(x) for x in str_list]
    for x in range(0, len(float_list)):
        hour += float_list[x]/(60**float(x))
    return hour

##
#If private in str, then it is private; same for public
#otherwise "NA" is returned
##
def statusType(str):
    for x in range(0, len(statusStrings)):
        if statusStrings[x].lower() in str.lower():
            return statusStrings[x]
    return "NA"

##
#If a project is ODD/Combined, then it does not have a resourceType
#If a project contains (IC), then it is external, otherwise internal
##
def resourceType(str):
    if "ODD".lower() in str.lower() or "Combined".lower() in str.lower():
        return "NA"
    elif ("(IC)").lower() in str.lower():
        return "External"
    else:
        return "Internal"

##
#Given projects in 2017, concatenateHours(Str) looks them up in the 2016 projects
#if there is a match, then the hours are combined
##
def concatenateHours(str):
    for x in range(0, len(toggle_preData['Project'])):
        if str == toggle_preData['Project'][x]:
            return convertHours(toggle_preData['Hours'][x])
    return 0

##
#Tries data export function, retains only `Project` and `Registered time` columns
#and overwrites previous file. If not possible, then error prompt and system exit.
##
def dataExit():
    try:
        toggle_base_url = "Toggl_projects_"
        toggle_timestamp = str(time.strftime("%m-%d-%y"))
        toggle_filetype = ".csv"
        toggle_url = toggle_base_url + toggle_timestamp + toggle_filetype
        toggle_data[['Project', 'Hours', 'Project type', 'Resources', 'Private/public']].to_csv(toggle_url,index=False)
        os.remove("Toggl_projects_2017-01-01_to_2017-12-31.csv")
        sys.exit()
    except IOError:
        e.msgbox("Could not read file 2016 Toggl CSV file. Please re-download that file and put it in the same directory as this Python script.", "Error")
        sys.exit()

if __name__ == '__main__':
    dataLoad()
    dataFormat()
    dataExit()

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import codecs
import pywikibot
import re
import config
from os import environ
from klasa import *
from sjpClass import kategoriaSlowa, checkHistory

def checkIfExists(page):

        initial_length = len(page.listLangs)
        def determine(section):
                try: check = Haslo(section.titleHeader)
		except sectionsNotFound:
			pass
		except WrongHeader:
			pass
		else:
			if check.type == 3:
				for elem in check.listLangs:
					if elem.lang == u'polski':
                                                return 1
                return 0

        page.listLangs[:] = [x for x in page.listLangs if not determine(x)]

        if len(page.listLangs) != initial_length:
                return 1
	return 0

def createMapping(map):
	mapPage = pywikibot.Page(site, u'Wikipedysta:AlkamidBot/sjp/mapa')
	wordListFilename = u'%soutput/frequencyListPL.txt' % config.path['scripts']
	wordList = codecs.open(wordListFilename, 'r', 'utf-8')
	i=1
	table = u'{| class="wikitable"'
	for line in wordList:
		word = line.strip().split('=')[0]
		if i>500:
			break
		try: map[word]
		except KeyError:
			pass
		else:
			if u'\n|-\n|%s\n|[[' % word not in table:
				table += u'\n|-\n|%s\n|[[%s|%s]]' % (word, map[word], map[word].replace(u'Wikipedysta:AlkamidBot/sjp/', u''))
				i+=1
	table += u'\n|-\n|}'
	mapPage.put(table, comment=u'aktualizacja')

def checkImages(page, obrazki):
	changed = 0
	for sekcja in page.listLangs:
		for obr in obrazki:
			if obr == sekcja.title or u'%s się' % obr == sekcja.title:
				if checkHistory(u'%s%d' % (kat.pages, i+1)):
					sekcja.pola()
					for a in obrazki[obr]:
						if a not in sekcja.dodatki.text:
							sekcja.dodatki.text += u'\n%s' % (a)
							sekcja.saveChanges()
							changed = 1
	return changed

def updateTable(katPages, pageCount, tableText):
	page = pywikibot.Page(site, u'%s%d' % (katPages, pageCount))
	history = page.getVersionHistory()
	if history[0][2] == u'Olafbot' and history[0][3] == u'Zawartość została przeniesiona do artykułów.':
		tableText = tableText.replace(u'\n| [[%s%d|%d]]' % (katPages, pageCount, pageCount), u'')
	return tableText

def main():
	global site
	site = pywikibot.Site()
	wordsPerPage = 10

	kategorie = []
	kategorie.append(kategoriaSlowa(u'bezproblemu', wordsPerPage, u'łatwe/', u'\n|-\n| łatwe (jedno znaczenie, bez synonimów)', u'bezproblemu'))
	kategorie.append(kategoriaSlowa(u'zwrotne', wordsPerPage, u'zwrotne/', u'\n|-\n| czasowniki zwrotne', u'zwrotne'))
	##kategorie.append(kategoriaSlowa(u'ndm', wordsPerPage, u'ndm/', u'\n|-\n| nieodmienne', u'ndm'))
	kategorie.append(kategoriaSlowa(u'np', wordsPerPage, u'np/', u'\n|-\n| \"np.\" w znaczeniu', u'np'))
	kategorie.append(kategoriaSlowa(u'reszta', wordsPerPage, u'wszystkie/', u'\n|-\n| reszta', u'reszta'))
	kategorie.append(kategoriaSlowa(u'brak_znaczenia', wordsPerPage, u'brak_znaczen/', u'\n|-\n| bez znaczeń', u'brak_znaczenia'))
	kategorie.append(kategoriaSlowa(u'przymiotnik_od', wordsPerPage, u'przymiotnik_od/', u'\n|-\n| \"przymiotnik od\"', u'przymiotnik_od'))

	tabelkaPage = pywikibot.Page(site, u'Wikipedysta:AlkamidBot/sjp/tabelka')
	tabText = tabelkaPage.get()
	for kat in kategorie:
                s_limits = re.search(ur'%s([0-9]{1,2}?)\|[0-9]{1,2}]]\n\|(?=-|\})' % kat.pages, tabText)
		if s_limits:
			kat.addLimit(int(s_limits.group(1)))
	obrazki = obrazkiAndrzeja()

	mapping = {}

	for kat in kategorie:
                print kat.limit

		for i in range(kat.limit):
			try: haslo = Haslo(u'%s%d' % (kat.pages, i+1))
			except sectionsNotFound:
				tabText = updateTable(kat.pages, i+1, tabText)
			else:
				if haslo.type == 3:
					comment = u''
					if checkIfExists(haslo):
						comment += u'usunięcie istniejącego hasła'
					if checkImages(haslo, obrazki):
						comment += u', dodanie ilustracji z [[Wikipedysta:Andrzej 22/Ilustracje|listy Andrzeja]]'
					comment.strip(u', ')
					if comment != u'':
						haslo.push(False, comment)

					if kat.name != u'brak_znaczenia':
						for sekcja in haslo.listLangs:
							mapping[sekcja.titleHeader] = u'%s%d' % (kat.pages, i+1)

	tabelkaPage.put(tabText, u'aktualizacja')
	createMapping(mapping)

if __name__ == '__main__':
    try:
        main()
    finally:
        pywikibot.stopme()

<commit_msg>print was for debugging only<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import codecs
import pywikibot
import re
import config
from os import environ
from klasa import *
from sjpClass import kategoriaSlowa, checkHistory

def checkIfExists(page):

        initial_length = len(page.listLangs)
        def determine(section):
                try: check = Haslo(section.titleHeader)
		except sectionsNotFound:
			pass
		except WrongHeader:
			pass
		else:
			if check.type == 3:
				for elem in check.listLangs:
					if elem.lang == u'polski':
                                                return 1
                return 0

        page.listLangs[:] = [x for x in page.listLangs if not determine(x)]

        if len(page.listLangs) != initial_length:
                return 1
	return 0

def createMapping(map):
	mapPage = pywikibot.Page(site, u'Wikipedysta:AlkamidBot/sjp/mapa')
	wordListFilename = u'%soutput/frequencyListPL.txt' % config.path['scripts']
	wordList = codecs.open(wordListFilename, 'r', 'utf-8')
	i=1
	table = u'{| class="wikitable"'
	for line in wordList:
		word = line.strip().split('=')[0]
		if i>500:
			break
		try: map[word]
		except KeyError:
			pass
		else:
			if u'\n|-\n|%s\n|[[' % word not in table:
				table += u'\n|-\n|%s\n|[[%s|%s]]' % (word, map[word], map[word].replace(u'Wikipedysta:AlkamidBot/sjp/', u''))
				i+=1
	table += u'\n|-\n|}'
	mapPage.put(table, comment=u'aktualizacja')

def checkImages(page, obrazki):
	changed = 0
	for sekcja in page.listLangs:
		for obr in obrazki:
			if obr == sekcja.title or u'%s się' % obr == sekcja.title:
				if checkHistory(u'%s%d' % (kat.pages, i+1)):
					sekcja.pola()
					for a in obrazki[obr]:
						if a not in sekcja.dodatki.text:
							sekcja.dodatki.text += u'\n%s' % (a)
							sekcja.saveChanges()
							changed = 1
	return changed

def updateTable(katPages, pageCount, tableText):
	page = pywikibot.Page(site, u'%s%d' % (katPages, pageCount))
	history = page.getVersionHistory()
	if history[0][2] == u'Olafbot' and history[0][3] == u'Zawartość została przeniesiona do artykułów.':
		tableText = tableText.replace(u'\n| [[%s%d|%d]]' % (katPages, pageCount, pageCount), u'')
	return tableText

def main():
	global site
	site = pywikibot.Site()
	wordsPerPage = 10

	kategorie = []
	kategorie.append(kategoriaSlowa(u'bezproblemu', wordsPerPage, u'łatwe/', u'\n|-\n| łatwe (jedno znaczenie, bez synonimów)', u'bezproblemu'))
	kategorie.append(kategoriaSlowa(u'zwrotne', wordsPerPage, u'zwrotne/', u'\n|-\n| czasowniki zwrotne', u'zwrotne'))
	##kategorie.append(kategoriaSlowa(u'ndm', wordsPerPage, u'ndm/', u'\n|-\n| nieodmienne', u'ndm'))
	kategorie.append(kategoriaSlowa(u'np', wordsPerPage, u'np/', u'\n|-\n| \"np.\" w znaczeniu', u'np'))
	kategorie.append(kategoriaSlowa(u'reszta', wordsPerPage, u'wszystkie/', u'\n|-\n| reszta', u'reszta'))
	kategorie.append(kategoriaSlowa(u'brak_znaczenia', wordsPerPage, u'brak_znaczen/', u'\n|-\n| bez znaczeń', u'brak_znaczenia'))
	kategorie.append(kategoriaSlowa(u'przymiotnik_od', wordsPerPage, u'przymiotnik_od/', u'\n|-\n| \"przymiotnik od\"', u'przymiotnik_od'))

	tabelkaPage = pywikibot.Page(site, u'Wikipedysta:AlkamidBot/sjp/tabelka')
	tabText = tabelkaPage.get()
	for kat in kategorie:
                s_limits = re.search(ur'%s([0-9]{1,2}?)\|[0-9]{1,2}]]\n\|(?=-|\})' % kat.pages, tabText)
		if s_limits:
			kat.addLimit(int(s_limits.group(1)))
	obrazki = obrazkiAndrzeja()

	mapping = {}

	for kat in kategorie:
		for i in range(kat.limit):
			try: haslo = Haslo(u'%s%d' % (kat.pages, i+1))
			except sectionsNotFound:
				tabText = updateTable(kat.pages, i+1, tabText)
			else:
				if haslo.type == 3:
					comment = u''
					if checkIfExists(haslo):
						comment += u'usunięcie istniejącego hasła'
					if checkImages(haslo, obrazki):
						comment += u', dodanie ilustracji z [[Wikipedysta:Andrzej 22/Ilustracje|listy Andrzeja]]'
					comment.strip(u', ')
					if comment != u'':
						haslo.push(False, comment)

					if kat.name != u'brak_znaczenia':
						for sekcja in haslo.listLangs:
							mapping[sekcja.titleHeader] = u'%s%d' % (kat.pages, i+1)

	tabelkaPage.put(tabText, u'aktualizacja')
	createMapping(mapping)

if __name__ == '__main__':
    try:
        main()
    finally:
        pywikibot.stopme()

<|endoftext|>


<commit_before>from tornado import concurrent, gen, httputil
from h2.connection import H2Connection
from h2.events import (
    ResponseReceived, RequestReceived, DataReceived, StreamEnded,
    StreamReset, RemoteSettingsChanged
)

from microproxy.interceptor import signal_request, signal_response, signal_publish
from microproxy.utils import get_logger
from microproxy import http
logger = get_logger(__name__)

class Http2Layer(object):
    '''
    ForwardLayer: passing all the src data to destination. Will not intercept anything
    '''
    def __init__(self, context):
        super(Http2Layer, self).__init__()
        self.context = context
        self.src_conn = Connection(self, self.context.src_stream, client_side=False)
        self.dest_conn = Connection(self, self.context.dest_stream, client_side=True)
        self.streams = dict()
        self._future = concurrent.Future()

    @gen.coroutine
    def process_and_return_context(self):
        yield self._init_h2_connection()
        self.context.src_stream.read_until_close(streaming_callback=self.src_conn.on_data_received)
        self.context.src_stream.set_close_callback(self.on_src_close)
        self.context.dest_stream.read_until_close(streaming_callback=self.dest_conn.on_data_received)
        self.context.dest_stream.set_close_callback(self.on_dest_close)
        result = yield self._future
        raise gen.Return(result)

    @gen.coroutine
    def _init_h2_connection(self):
        self.dest_conn.initiate_connection()
        yield self.dest_conn.flush()
        self.src_conn.initiate_connection()
        yield self.src_conn.flush()

    def on_src_close(self):
        logger.debug("src stream closed")
        self.context.dest_stream.close()
        if self._future.running():
            self._future.set_result(self.context)

    def on_dest_close(self):
        logger.debug("dest stream closed")
        self.context.src_stream.close()
        if self._future.running():
            self._future.set_result(self.context)

    def get_target_conn(self, from_conn):
        return self.dest_conn if from_conn is self.src_conn else self.src_conn

    def on_request_header(self, src_stream_id, headers):
        stream = Stream(self.context)
        stream.write_request_headers(headers)
        self.streams[src_stream_id] = stream

    def on_request_body(self, src_stream_id, data):
        self.streams[src_stream_id].write_request_body(data)

    def get_request(self, src_stream_id):
        stream = self.streams[src_stream_id]
        stream.request_done()

        reqs = signal_request.send(self, request=stream.req)
        new_req = reqs[0][1] if len(reqs) else stream.req
        stream.req = new_req
        return (new_req.headers, new_req.body)

    def on_response_header(self, src_stream_id, headers):
        self.streams[src_stream_id].write_response_headers(headers)

    def on_response_body(self, src_stream_id, data):
        self.streams[src_stream_id].write_response_body(data)

    def get_response(self, src_stream_id):
        stream = self.streams[src_stream_id]
        stream.response_done()

        resps = signal_response.send(self, response=stream.resp)
        new_resp = resps[0][1] if len(resps) else stream.resp
        stream.resp = new_resp
        return (new_resp.headers, new_resp.body)

    def on_finish(self, src_stream_id):
        signal_publish.send(
            self,
            request=self.streams[src_stream_id].req,
            response=self.streams[src_stream_id].resp)

class Connection(H2Connection):
    def __init__(self, http2_layer, stream, *args, **kwargs):
        super(Connection, self).__init__(*args, **kwargs)
        self.http2_layer = http2_layer
        self.stream = stream
        self.to_stream_ids = dict([(0, 0)])
        self._type = "dest" if kwargs["client_side"] else "src"

    def flush(self):
        data = self.data_to_send()
        logger.debug("data send to {0} with length:  {1}".format(self._type, len(data)))
        return self.stream.write(data)

    def on_data_received(self, data):
        try:
            logger.debug("data received from {0} with length {1}".format(self._type, len(data)))
            events = self.receive_data(data)
            self.flush()
            self.handle_events(events)
        except Exception as e:
            logger.exception(e)
            self.stream.closed()

    def handle_events(self, events):
        for event in events:
            logger.debug("event received from {0}: {1}".format(self._type, event))
            if isinstance(event, ResponseReceived):
                self.handle_response(event.headers, event.stream_id)
            elif isinstance(event, RequestReceived):
                self.handle_request(event.headers, event.stream_id)
            elif isinstance(event, DataReceived):
                self.handle_data(event.data, event.stream_id, event.flow_controlled_length)
            elif isinstance(event, StreamEnded):
                self.handle_end_stream(event.stream_id)
            elif isinstance(event, StreamReset):
                self.handle_reset(event.stream_id, event.error_code)
            elif isinstance(event, RemoteSettingsChanged):
                self.handle_update_settings(event)
            else:
                logger.warn("not handled event: {0}".format(event))

    def handle_request(self, headers, stream_id):
        self.http2_layer.on_request_header(stream_id, headers)

    def handle_response(self, headers, stream_id):
        self.http2_layer.on_response_header(self.to_stream_ids[stream_id], headers)

    def handle_data(self, data, stream_id, flow_controlled_length):
        if self._is_source_conn():
            self.http2_layer.on_request_body(stream_id, data)
        else:
            self.http2_layer.on_response_body(self.to_stream_ids[stream_id], data)

        if flow_controlled_length > 0:
            self.increment_flow_control_window(flow_controlled_length)
            self.flush()

    def handle_update_settings(self, event):
        write_conn = self.http2_layer.get_target_conn(self)
        new_settings = dict([(id, cs.new_value) for (id, cs) in event.changed_settings.iteritems()])
        write_conn.update_settings(new_settings)
        write_conn.flush()

    def handle_end_stream(self, stream_id):
        write_conn = self.http2_layer.get_target_conn(self)
        if self._is_source_conn():
            dest_stream_id = write_conn.get_next_available_stream_id()
            self.to_stream_ids[stream_id] = dest_stream_id
            write_conn.to_stream_ids[dest_stream_id] = stream_id

            headers, body = self.http2_layer.get_request(stream_id)
            write_conn.write_headers(dest_stream_id, headers)
            write_conn.write(dest_stream_id, body)
            logger.debug("request {0}->{1}".format(stream_id, dest_stream_id))
        else:
            src_stream_id = self.to_stream_ids[stream_id]

            headers, body = self.http2_layer.get_response(src_stream_id)
            write_conn.write_headers(src_stream_id, headers)
            write_conn.write(src_stream_id, body)

            self.http2_layer.on_finish(src_stream_id)
            logger.debug("response {0}->{1}".format(stream_id, src_stream_id))

    def write_headers(self, stream_id, headers):
        sorted_headers = sorted(headers.get_all(), key=lambda h: h[0])
        self.send_headers(stream_id, sorted_headers)
        self.flush()

    def write(self, stream_id, body):
        chunks = [body] if isinstance(body, str) else body
        position = 0
        for chunk in chunks:
            while position < len(chunk):
                max_outbound_frame_size = self.max_outbound_frame_size
                frame_chunk = chunk[position:position + max_outbound_frame_size]
                self.send_data(stream_id, frame_chunk)
                self.flush()
                position += max_outbound_frame_size
        self.end_stream(stream_id)
        self.flush()

    def handle_reset(self, stream_id, error_code):
        if error_code == 0x8:
            write_conn = self.http2_layer.get_target_conn(self)
            write_conn.reset_stream(self.to_stream_ids[stream_id], error_code)
            logger.debug("reset {0}({1})->{2}({3}) with {4}".format(
                self._type, stream_id, write_conn._type, self.to_stream_ids[stream_id], error_code))

    def _is_source_conn(self):
        return self._type == "src"

class Stream(object):
    def __init__(self, context):
        super(Stream, self).__init__()
        self.context = context
        self.req_headers = None
        self.req_chunks = []
        self.resp_headers = None
        self.resp_chunks = []

    def write_request_headers(self, headers):
        self.req_headers = headers

    def write_request_body(self, data):
        self.req_chunks.append(data)

    def request_done(self):
        headers_dict = dict(self.req_headers)
        self.req = http.HttpRequest(
            version="HTTP/2",
            method=headers_dict[":method"],
            path=headers_dict[":path"],
            headers=httputil.HTTPHeaders(self.req_headers),
            body=b"".join(self.req_chunks))

    def write_response_headers(self, headers):
        self.resp_headers = headers

    def write_response_body(self, data):
        self.resp_chunks.append(data)

    def response_done(self):
        headers_dict = dict(self.resp_headers)
        self.resp = http.HttpResponse(
            version="HTTP/2",
            code=headers_dict[":status"],
            headers=httputil.HTTPHeaders(self.resp_headers),
            body=b"".join(self.resp_chunks))
<commit_msg>Refactor code and add note for unclearly design about http2 header<commit_after>from tornado import concurrent, gen, httputil
from h2.connection import H2Connection
from h2.events import (
    ResponseReceived, RequestReceived, DataReceived, StreamEnded,
    StreamReset, RemoteSettingsChanged
)

from microproxy.interceptor import signal_request, signal_response, signal_publish
from microproxy.utils import get_logger
from microproxy import http
logger = get_logger(__name__)

class Http2Layer(object):
    '''
    ForwardLayer: passing all the src data to destination. Will not intercept anything
    '''
    def __init__(self, context):
        super(Http2Layer, self).__init__()
        self.context = context
        self.src_conn = Connection(self, self.context.src_stream, client_side=False)
        self.dest_conn = Connection(self, self.context.dest_stream, client_side=True)
        self.streams = dict()
        self._future = concurrent.Future()

    @gen.coroutine
    def process_and_return_context(self):
        yield self._init_h2_connection()
        self.context.src_stream.read_until_close(streaming_callback=self.src_conn.on_data_received)
        self.context.src_stream.set_close_callback(self.on_src_close)
        self.context.dest_stream.read_until_close(streaming_callback=self.dest_conn.on_data_received)
        self.context.dest_stream.set_close_callback(self.on_dest_close)
        result = yield self._future
        raise gen.Return(result)

    @gen.coroutine
    def _init_h2_connection(self):
        self.dest_conn.initiate_connection()
        yield self.dest_conn.flush()
        self.src_conn.initiate_connection()
        yield self.src_conn.flush()

    def on_src_close(self):
        logger.debug("src stream closed")
        self.context.dest_stream.close()
        if self._future.running():
            self._future.set_result(self.context)

    def on_dest_close(self):
        logger.debug("dest stream closed")
        self.context.src_stream.close()
        if self._future.running():
            self._future.set_result(self.context)

    def get_target_conn(self, from_conn):
        return self.dest_conn if from_conn is self.src_conn else self.src_conn

    def on_request_header(self, src_stream_id, headers):
        stream = Stream(self.context)
        stream.write_request_headers(headers)
        self.streams[src_stream_id] = stream

    def on_request_body(self, src_stream_id, data):
        self.streams[src_stream_id].write_request_body(data)

    def get_request(self, src_stream_id):
        stream = self.streams[src_stream_id]
        stream.request_done()

        reqs = signal_request.send(self, request=stream.req)
        new_req = reqs[0][1] if len(reqs) else stream.req
        stream.req = new_req
        return (new_req.headers, new_req.body)

    def on_response_header(self, src_stream_id, headers):
        self.streams[src_stream_id].write_response_headers(headers)

    def on_response_body(self, src_stream_id, data):
        self.streams[src_stream_id].write_response_body(data)

    def get_response(self, src_stream_id):
        stream = self.streams[src_stream_id]
        stream.response_done()

        resps = signal_response.send(self, response=stream.resp)
        new_resp = resps[0][1] if len(resps) else stream.resp
        stream.resp = new_resp
        return (new_resp.headers, new_resp.body)

    def on_finish(self, src_stream_id):
        signal_publish.send(
            self,
            request=self.streams[src_stream_id].req,
            response=self.streams[src_stream_id].resp)

class Connection(H2Connection):
    def __init__(self, http2_layer, stream, *args, **kwargs):
        super(Connection, self).__init__(*args, **kwargs)
        self.http2_layer = http2_layer
        self.stream = stream
        self.to_stream_ids = dict([(0, 0)])
        self._type = "dest" if kwargs["client_side"] else "src"

    def flush(self):
        data = self.data_to_send()
        logger.debug("data send to {0} with length:  {1}".format(self._type, len(data)))
        return self.stream.write(data)

    def on_data_received(self, data):
        try:
            logger.debug("data received from {0} with length {1}".format(self._type, len(data)))
            events = self.receive_data(data)
            self.flush()
            self.handle_events(events)
        except Exception as e:
            logger.exception(e)
            self.stream.closed()

    def handle_events(self, events):
        for event in events:
            logger.debug("event received from {0}: {1}".format(self._type, event))
            if isinstance(event, ResponseReceived):
                self.handle_response(event.headers, event.stream_id)
            elif isinstance(event, RequestReceived):
                self.handle_request(event.headers, event.stream_id)
            elif isinstance(event, DataReceived):
                self.handle_data(event.data, event.stream_id, event.flow_controlled_length)
            elif isinstance(event, StreamEnded):
                self.handle_end_stream(event.stream_id)
            elif isinstance(event, StreamReset):
                self.handle_reset(event.stream_id, event.error_code)
            elif isinstance(event, RemoteSettingsChanged):
                self.handle_update_settings(event)
            else:
                logger.warn("not handled event: {0}".format(event))

    def handle_request(self, headers, stream_id):
        self.http2_layer.on_request_header(stream_id, headers)

    def handle_response(self, headers, stream_id):
        self.http2_layer.on_response_header(self.to_stream_ids[stream_id], headers)

    def handle_data(self, data, stream_id, flow_controlled_length):
        if self._is_source_conn():
            self.http2_layer.on_request_body(stream_id, data)
        else:
            self.http2_layer.on_response_body(self.to_stream_ids[stream_id], data)

        if flow_controlled_length > 0:
            self.increment_flow_control_window(flow_controlled_length)
            self.flush()

    def handle_update_settings(self, event):
        write_conn = self.http2_layer.get_target_conn(self)
        new_settings = {id: cs.new_value for (id, cs) in event.changed_settings.iteritems()}
        write_conn.update_settings(new_settings)
        write_conn.flush()

    def handle_end_stream(self, stream_id):
        write_conn = self.http2_layer.get_target_conn(self)
        if self._is_source_conn():
            dest_stream_id = write_conn.get_next_available_stream_id()
            self.to_stream_ids[stream_id] = dest_stream_id
            write_conn.to_stream_ids[dest_stream_id] = stream_id

            headers, body = self.http2_layer.get_request(stream_id)
            write_conn.write_headers(dest_stream_id, headers)
            write_conn.write(dest_stream_id, body)
            logger.debug("request {0}->{1}".format(stream_id, dest_stream_id))
        else:
            src_stream_id = self.to_stream_ids[stream_id]

            headers, body = self.http2_layer.get_response(src_stream_id)
            write_conn.write_headers(src_stream_id, headers)
            write_conn.write(src_stream_id, body)

            self.http2_layer.on_finish(src_stream_id)
            logger.debug("response {0}->{1}".format(stream_id, src_stream_id))

    def write_headers(self, stream_id, headers):
        # Note: headers with key had prefix ":" that must before any other headers
        # So used sorted function to let header could had the correct order
        sorted_headers = sorted(headers.get_all(), key=lambda h: h[0])
        self.send_headers(stream_id, sorted_headers)
        self.flush()

    def write(self, stream_id, body):
        chunks = [body] if isinstance(body, str) else body
        position = 0
        for chunk in chunks:
            while position < len(chunk):
                max_outbound_frame_size = self.max_outbound_frame_size
                frame_chunk = chunk[position:position + max_outbound_frame_size]
                self.send_data(stream_id, frame_chunk)
                self.flush()
                position += max_outbound_frame_size
        self.end_stream(stream_id)
        self.flush()

    def handle_reset(self, stream_id, error_code):
        if error_code == 0x8:
            write_conn = self.http2_layer.get_target_conn(self)
            write_conn.reset_stream(self.to_stream_ids[stream_id], error_code)
            logger.debug("reset {0}({1})->{2}({3}) with {4}".format(
                self._type, stream_id, write_conn._type, self.to_stream_ids[stream_id], error_code))

    def _is_source_conn(self):
        return self._type == "src"

class Stream(object):
    def __init__(self, context):
        super(Stream, self).__init__()
        self.context = context
        self.req_headers = None
        self.req_chunks = []
        self.resp_headers = None
        self.resp_chunks = []

    def write_request_headers(self, headers):
        self.req_headers = headers

    def write_request_body(self, data):
        self.req_chunks.append(data)

    def request_done(self):
        headers_dict = dict(self.req_headers)
        self.req = http.HttpRequest(
            version="HTTP/2",
            method=headers_dict[":method"],
            path=headers_dict[":path"],
            headers=httputil.HTTPHeaders(self.req_headers),
            body=b"".join(self.req_chunks))

    def write_response_headers(self, headers):
        self.resp_headers = headers

    def write_response_body(self, data):
        self.resp_chunks.append(data)

    def response_done(self):
        headers_dict = dict(self.resp_headers)
        self.resp = http.HttpResponse(
            version="HTTP/2",
            code=headers_dict[":status"],
            headers=httputil.HTTPHeaders(self.resp_headers),
            body=b"".join(self.resp_chunks))
<|endoftext|>


<commit_before># :coding: utf-8

import os

from .class_parser import get_class_environment
from .function_parser import get_function_environment
from .data_parser import get_data_environment

def get_file_environment(file_path, file_id, module_id, environment=None):
    """Return file environment from *file_path*.

    *file_id* represent the ID of the file.

    *module_id* represent the ID of the module.

    Update the *environment* is available and return it as-is if the file
    is not readable.

    Return dictionary in the form of::

        {
            "class": {...},
            "method": {...},
            "attribute": {...},
            "function": {...},
            "data": {...},
            "file": {
                "path/to/file.js": {
                    "id": "path/to/file.js",
                    "name": "script.js",
                    "path": "/path/to/script.js",
                    "content": "'use strict'\\n\\n..."
                },
                ...
            }
        }

    """
    if environment is None:
        environment = {}

    file_environment = {
        "id": file_id,
        "module_id": module_id,
        "name": os.path.basename(file_path),
        "path": file_path,
        "content": None,
        "class": {},
        "data": {},
        "function": {},
    }
    try:
        with open(file_path, "r") as f:
            content = f.read()
    except (IOError, OSError):
        return environment

    file_environment["class"] = get_class_environment(content, module_id)
    file_environment["function"] = get_function_environment(content, module_id)
    file_environment["data"] = get_data_environment(content, module_id)
    file_environment["content"] = content

    method_environment = {}
    attribute_environment = {}

    # Extract methods and attributes from class environment to set it in the
    # top level environment.
    for _class in file_environment["class"].values():
        method_environment.update(_class["method"].copy())
        attribute_environment.update(_class["attribute"].copy())

    if "class" not in environment.keys():
        environment["class"] = {}
    environment["class"].update(file_environment["class"])

    if "method" not in environment.keys():
        environment["method"] = {}
    environment["method"].update(method_environment)

    if "attribute" not in environment.keys():
        environment["attribute"] = {}
    environment["attribute"].update(attribute_environment)

    if "function" not in environment.keys():
        environment["function"] = {}
    environment["function"].update(file_environment["function"])

    if "data" not in environment.keys():
        environment["data"] = {}
    environment["data"].update(file_environment["data"])

    if "file" not in environment.keys():
        environment["file"] = {}
    environment["file"][file_id] = file_environment

    return environment
<commit_msg>Fix typo<commit_after># :coding: utf-8

import os

from .class_parser import get_class_environment
from .function_parser import get_function_environment
from .data_parser import get_data_environment

def get_file_environment(file_path, file_id, module_id, environment=None):
    """Return file environment from *file_path*.

    *file_id* represent the ID of the file.

    *module_id* represent the ID of the module.

    Update the *environment* if available and return it as-is if the file
    is not readable.

    Return dictionary in the form of::

        {
            "class": {...},
            "method": {...},
            "attribute": {...},
            "function": {...},
            "data": {...},
            "file": {
                "path/to/file.js": {
                    "id": "path/to/file.js",
                    "name": "script.js",
                    "path": "/path/to/script.js",
                    "content": "'use strict'\\n\\n..."
                },
                ...
            }
        }

    """
    if environment is None:
        environment = {}

    file_environment = {
        "id": file_id,
        "module_id": module_id,
        "name": os.path.basename(file_path),
        "path": file_path,
        "content": None,
        "class": {},
        "data": {},
        "function": {},
    }
    try:
        with open(file_path, "r") as f:
            content = f.read()
    except (IOError, OSError):
        return environment

    file_environment["class"] = get_class_environment(content, module_id)
    file_environment["function"] = get_function_environment(content, module_id)
    file_environment["data"] = get_data_environment(content, module_id)
    file_environment["content"] = content

    method_environment = {}
    attribute_environment = {}

    # Extract methods and attributes from class environment to set it in the
    # top level environment.
    for _class in file_environment["class"].values():
        method_environment.update(_class["method"].copy())
        attribute_environment.update(_class["attribute"].copy())

    if "class" not in environment.keys():
        environment["class"] = {}
    environment["class"].update(file_environment["class"])

    if "method" not in environment.keys():
        environment["method"] = {}
    environment["method"].update(method_environment)

    if "attribute" not in environment.keys():
        environment["attribute"] = {}
    environment["attribute"].update(attribute_environment)

    if "function" not in environment.keys():
        environment["function"] = {}
    environment["function"].update(file_environment["function"])

    if "data" not in environment.keys():
        environment["data"] = {}
    environment["data"].update(file_environment["data"])

    if "file" not in environment.keys():
        environment["file"] = {}
    environment["file"][file_id] = file_environment

    return environment
<|endoftext|>