<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`urldiscoverer` --- URL unshortener plugin
===============================================
"""

__docformat__ = "restructuredtext en"

import httplib
import urlparse
import urllib2

from Skype4Py.enums import cmsReceived

from plugin import Plugin

def truncate_url(url, max_path_length=20):
    """
    Truncates URL path component to `max_path_length` characters.

    :param url: URL
    :type url: `unicode`

    :param max_path_length: maximum URL path length
    :type max_path_length: `int`

    :return: truncated URL
    :rtype: `unicode`

    :raise: ValueError if URL isn't a valid one.

    .. note::
        There's a limited Cyrillic URLs support (currently limited to URL
        path only). Skype won't recognize Cyrillic domain name parts anyway.

    ::

        >>> truncate_url("http://test.me/XXXXX", 3)
        'test.me/XXX...'
        >>> truncate_url("http://test.me/XXXXX")
        'test.me/XXXXX'
        >>> truncate_url("http://test.me/ссылка", 12)
        'test.me/%D1%81%D1%81...'
    """

    url = urlparse.urlparse(url)
    if not url.netloc:
        raise ValueError, "Invalid URL: {0}".format(url)
    quoted_path = urllib2.quote(url.path)
    if len(quoted_path) < max_path_length:
        return url.netloc + quoted_path
    return url.netloc + quoted_path[:max_path_length + 1] + "..."

class URLDiscoverer(Plugin):
    """
    This plugin monitors incoming chat messages and recursively "unshortens"
    common URL shortener services. Long source URLs are being truncated to
    suppress potentially excessive output.
    """

    def __init__(self, parent):
        super(URLDiscoverer, self).__init__(parent)
        self._shorteners = [
            "tinyurl.com",
            "bit.ly",
            "t.co",
            "ls.gd",
            "goo.gl",
            "bitly.com",
            "ow.ly",
            "fb.me",
            "is.gd",
            "tr.im",
            "cli.gs",
            "tiny.cc",
            "short.to",
            "trib.al",
            "nblo.gs",
            "go.ign.com"
            ]

    def on_message_status(self, message, status):
        if status != cmsReceived:
            return

        if not any(s in message.Body for s in self._shorteners):
            return

        chat = message.Chat

        output = []

        destinations = []
        for chunk in message.Body.split():
            if any(s in chunk.lower() for s in self._shorteners):
                destinations.append(chunk)

        for destination in destinations:
            valid = True
            source = None

            for r in ("http://", "https://"):
                destination.replace(r, "")

            destination = "http://" + destination
            while any("http://" + s in destination for s in self._shorteners):
                if not source:
                    source = destination
                url = urlparse.urlparse(destination)
                connection = httplib.HTTPConnection(url.netloc, timeout=5)
                connection.request("GET", urllib2.quote(url.path))
                response = connection.getresponse()
                destination = response.getheader("Location")
                if destination is None:
                    valid = False
                    msg = u"{0} -> unable to resolve".format(
                        truncate_url(source)
                    )
                    output.append(msg)
                    log_msg = u"Unable to resolve {0}".format(source)
                    self._logger.error(log_msg)
                    break

            if source and valid:
                output.append(u"{0} -> {1}".format(
                    truncate_url(source), destination
                ))
                self._logger.info("Resolving URL {0} for {1} ({2})".format(
                    truncate_url(source), message.FromDisplayName,
                    message.FromHandle
                ))

        if output:
            chat.SendMessage(", ".join(output))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Quick fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
:mod:`urldiscoverer` --- URL unshortener plugin
===============================================
"""

__docformat__ = "restructuredtext en"

import httplib
import urlparse
import urllib2

from Skype4Py.enums import cmsReceived

from plugin import Plugin

def truncate_url(url, max_path_length=20):
    """
    Truncates URL path component to `max_path_length` characters.

    :param url: URL
    :type url: `unicode`

    :param max_path_length: maximum URL path length
    :type max_path_length: `int`

    :return: truncated URL
    :rtype: `unicode`

    :raise: ValueError if URL isn't a valid one.

    .. note::
        There's a limited Cyrillic URLs support (currently limited to URL
        path only). Skype won't recognize Cyrillic domain name parts anyway.

    ::

        >>> truncate_url("http://test.me/XXXXX", 3)
        'test.me/XXX...'
        >>> truncate_url("http://test.me/XXXXX")
        'test.me/XXXXX'
        >>> truncate_url("http://test.me/ссылка", 12)
        'test.me/%D1%81%D1%81...'
    """

    url = urlparse.urlparse(url)
    if not url.netloc:
        raise ValueError, "Invalid URL: {0}".format(url)
    quoted_path = urllib2.quote(url.path)
    if len(quoted_path) < max_path_length:
        return url.netloc + quoted_path
    return url.netloc + quoted_path[:max_path_length + 1] + "..."

class URLDiscoverer(Plugin):
    """
    This plugin monitors incoming chat messages and recursively "unshortens"
    common URL shortener services. Long source URLs are being truncated to
    suppress potentially excessive output.
    """

    def __init__(self, parent):
        super(URLDiscoverer, self).__init__(parent)
        self._shorteners = [
            "tinyurl.com",
            "bit.ly",
            "t.co",
            "ls.gd",
            "goo.gl",
            "bitly.com",
            "ow.ly",
            "fb.me",
            "is.gd",
            "tr.im",
            "cli.gs",
            "tiny.cc",
            "short.to",
            "trib.al",
            "nblo.gs",
            "go.ign.com"
            ]

    def on_message_status(self, message, status):
        if status != cmsReceived:
            return

        if not any(s in message.Body for s in self._shorteners):
            return

        chat = message.Chat

        output = []

        destinations = []
        for chunk in message.Body.split():
            if any(s in chunk.lower() for s in self._shorteners):
                destinations.append(chunk)

        for destination in destinations:
            valid = True
            source = None

            for r in ("http://", "https://"):
                destination = destination.replace(r, "")

            destination = "http://" + destination
            while any("http://" + s in destination for s in self._shorteners):
                if not source:
                    source = destination
                url = urlparse.urlparse(destination)
                connection = httplib.HTTPConnection(url.netloc, timeout=5)

                # FIXME: Cyrillic URLs. (KeyError in quote)
                connection.request("GET", urllib2.quote(url.path))

                response = connection.getresponse()
                destination = response.getheader("Location")
                if destination is None:
                    valid = False
                    msg = u"{0} -> unable to resolve".format(
                        truncate_url(source)
                    )
                    output.append(msg)
                    log_msg = u"Unable to resolve {0}".format(source)
                    self._logger.error(log_msg)
                    break

            if source and valid:
                output.append(u"{0} -> {1}".format(
                    truncate_url(source), destination
                ))
                self._logger.info("Resolving URL {0} for {1} ({2})".format(
                    truncate_url(source), message.FromDisplayName,
                    message.FromHandle
                ))

        if output:
            chat.SendMessage(", ".join(output))

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<|endoftext|>


<commit_before>"""
Tests for the views
"""
from __future__ import absolute_import

from datetime import datetime

import ddt
import six
from django.urls import reverse
from edx_oauth2_provider.tests.factories import AccessTokenFactory, ClientFactory
from pytz import UTC

from capa.tests.response_xml_factory import MultipleChoiceResponseXMLFactory
from lms.djangoapps.courseware.tests.factories import GlobalStaffFactory, StaffFactory
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory

@ddt.ddt
class GradingPolicyTestMixin(object):
    """
    Mixin class for Grading Policy tests
    """
    view_name = None

    def setUp(self):
        super(GradingPolicyTestMixin, self).setUp()
        self.create_user_and_access_token()

    def create_user_and_access_token(self):
        self.user = GlobalStaffFactory.create()
        self.oauth_client = ClientFactory.create()
        self.access_token = AccessTokenFactory.create(user=self.user, client=self.oauth_client).token

    @classmethod
    def create_course_data(cls):
        cls.invalid_course_id = 'foo/bar/baz'
        cls.course = CourseFactory.create(display_name='An Introduction to API Testing', raw_grader=cls.raw_grader)
        cls.course_id = six.text_type(cls.course.id)
        with cls.store.bulk_operations(cls.course.id, emit_signals=False):
            cls.sequential = ItemFactory.create(
                category="sequential",
                parent_location=cls.course.location,
                display_name="Lesson 1",
                format="Homework",
                graded=True
            )

            factory = MultipleChoiceResponseXMLFactory()
            args = {'choices': [False, True, False]}
            problem_xml = factory.build_xml(**args)
            cls.problem = ItemFactory.create(
                category="problem",
                parent_location=cls.sequential.location,
                display_name="Problem 1",
                format="Homework",
                data=problem_xml,
            )

            cls.video = ItemFactory.create(
                category="video",
                parent_location=cls.sequential.location,
                display_name="Video 1",
            )

            cls.html = ItemFactory.create(
                category="html",
                parent_location=cls.sequential.location,
                display_name="HTML 1",
            )

    def http_get(self, uri, **headers):
        """
        Submit an HTTP GET request
        """

        default_headers = {
            'HTTP_AUTHORIZATION': 'Bearer ' + self.access_token
        }
        default_headers.update(headers)

        response = self.client.get(uri, follow=True, **default_headers)
        return response

    def assert_get_for_course(self, course_id=None, expected_status_code=200, **headers):
        """
        Submit an HTTP GET request to the view for the given course.
        Validates the status_code of the response is as expected.
        """

        response = self.http_get(
            reverse(self.view_name, kwargs={'course_id': course_id or self.course_id}),
            **headers
        )
        self.assertEqual(response.status_code, expected_status_code)
        return response

    def get_auth_header(self, user):
        """
        Returns Bearer auth header with a generated access token
        for the given user.
        """
        access_token = AccessTokenFactory.create(user=user, client=self.oauth_client).token
        return 'Bearer ' + access_token

    def test_get_invalid_course(self):
        """
        The view should return a 404 for an invalid course ID.
        """
        self.assert_get_for_course(course_id=self.invalid_course_id, expected_status_code=404)

    def test_get(self):
        """
        The view should return a 200 for a valid course ID.
        """
        return self.assert_get_for_course()

    def test_not_authenticated(self):
        """
        The view should return HTTP status 401 if user is unauthenticated.
        """
        self.assert_get_for_course(expected_status_code=401, HTTP_AUTHORIZATION=None)

    def test_staff_authorized(self):
        """
        The view should return a 200 when provided an access token
        for course staff.
        """
        user = StaffFactory(course_key=self.course.id)
        auth_header = self.get_auth_header(user)
        self.assert_get_for_course(HTTP_AUTHORIZATION=auth_header)

    def test_not_authorized(self):
        """
        The view should return HTTP status 404 when provided an
        access token for an unauthorized user.
        """
        user = UserFactory()
        auth_header = self.get_auth_header(user)
        self.assert_get_for_course(expected_status_code=403, HTTP_AUTHORIZATION=auth_header)

    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
    def test_course_keys(self, modulestore_type):
        """
        The view should be addressable by course-keys from both module stores.
        """
        course = CourseFactory.create(
            start=datetime(2014, 6, 16, 14, 30, tzinfo=UTC),
            end=datetime(2015, 1, 16, tzinfo=UTC),
            org="MTD",
            default_store=modulestore_type,
        )
        self.assert_get_for_course(course_id=six.text_type(course.id))

class CourseGradingPolicyTests(GradingPolicyTestMixin, SharedModuleStoreTestCase):
    """
    Tests for CourseGradingPolicy view.
    """
    view_name = 'grades_api:v1:course_grading_policy'

    raw_grader = [
        {
            "min_count": 24,
            "weight": 0.2,
            "type": "Homework",
            "drop_count": 0,
            "short_label": "HW"
        },
        {
            "min_count": 4,
            "weight": 0.8,
            "type": "Exam",
            "drop_count": 0,
            "short_label": "Exam"
        }
    ]

    @classmethod
    def setUpClass(cls):
        super(CourseGradingPolicyTests, cls).setUpClass()
        cls.create_course_data()

    def test_get(self):
        """
        The view should return grading policy for a course.
        """
        response = super(CourseGradingPolicyTests, self).test_get()

        expected = [
            {
                "count": 24,
                "weight": 0.2,
                "assignment_type": "Homework",
                "dropped": 0
            },
            {
                "count": 4,
                "weight": 0.8,
                "assignment_type": "Exam",
                "dropped": 0
            }
        ]
        self.assertListEqual(response.data, expected)

class CourseGradingPolicyMissingFieldsTests(GradingPolicyTestMixin, SharedModuleStoreTestCase):
    """
    Tests for CourseGradingPolicy view when fields are missing.
    """
    view_name = 'grades_api:v1:course_grading_policy'

    # Raw grader with missing keys
    raw_grader = [
        {
            "min_count": 24,
            "weight": 0.2,
            "type": "Homework",
            "drop_count": 0,
            "short_label": "HW"
        },
        {
            # Deleted "min_count" key
            "weight": 0.8,
            "type": "Exam",
            "drop_count": 0,
            "short_label": "Exam"
        }
    ]

    @classmethod
    def setUpClass(cls):
        super(CourseGradingPolicyMissingFieldsTests, cls).setUpClass()
        cls.create_course_data()

    def test_get(self):
        """
        The view should return grading policy for a course.
        """
        response = super(CourseGradingPolicyMissingFieldsTests, self).test_get()

        expected = [
            {
                "count": 24,
                "weight": 0.2,
                "assignment_type": "Homework",
                "dropped": 0
            },
            {
                "count": None,
                "weight": 0.8,
                "assignment_type": "Exam",
                "dropped": 0
            }
        ]
        self.assertListEqual(response.data, expected)
<commit_msg>Is this right? It fixes two tests<commit_after>"""
Tests for the views
"""
from __future__ import absolute_import

from datetime import datetime

import ddt
import six
from django.urls import reverse
from edx_oauth2_provider.tests.factories import AccessTokenFactory, ClientFactory
from pytz import UTC

from capa.tests.response_xml_factory import MultipleChoiceResponseXMLFactory
from lms.djangoapps.courseware.tests.factories import GlobalStaffFactory, StaffFactory
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory

@ddt.ddt
class GradingPolicyTestMixin(object):
    """
    Mixin class for Grading Policy tests
    """
    view_name = None

    def setUp(self):
        super(GradingPolicyTestMixin, self).setUp()
        self.create_user_and_access_token()

    def create_user_and_access_token(self):
        self.user = GlobalStaffFactory.create()
        self.oauth_client = ClientFactory.create()
        self.access_token = AccessTokenFactory.create(user=self.user, client=self.oauth_client).token

    @classmethod
    def create_course_data(cls):
        cls.invalid_course_id = 'foo/bar/baz'
        cls.course = CourseFactory.create(display_name='An Introduction to API Testing', raw_grader=cls.raw_grader)
        cls.course_id = six.text_type(cls.course.id)
        with cls.store.bulk_operations(cls.course.id, emit_signals=False):
            cls.sequential = ItemFactory.create(
                category="sequential",
                parent_location=cls.course.location,
                display_name="Lesson 1",
                format="Homework",
                graded=True
            )

            factory = MultipleChoiceResponseXMLFactory()
            args = {'choices': [False, True, False]}
            problem_xml = factory.build_xml(**args)
            cls.problem = ItemFactory.create(
                category="problem",
                parent_location=cls.sequential.location,
                display_name="Problem 1",
                format="Homework",
                data=problem_xml,
            )

            cls.video = ItemFactory.create(
                category="video",
                parent_location=cls.sequential.location,
                display_name="Video 1",
            )

            cls.html = ItemFactory.create(
                category="html",
                parent_location=cls.sequential.location,
                display_name="HTML 1",
            )

    def http_get(self, uri, **headers):
        """
        Submit an HTTP GET request
        """

        default_headers = {
            'HTTP_AUTHORIZATION': 'Bearer ' + self.access_token
        }
        default_headers.update(headers)

        response = self.client.get(uri, follow=True, **default_headers)
        return response

    def assert_get_for_course(self, course_id=None, expected_status_code=200, **headers):
        """
        Submit an HTTP GET request to the view for the given course.
        Validates the status_code of the response is as expected.
        """

        response = self.http_get(
            reverse(self.view_name, kwargs={'course_id': course_id or self.course_id}),
            **headers
        )
        self.assertEqual(response.status_code, expected_status_code)
        return response

    def get_auth_header(self, user):
        """
        Returns Bearer auth header with a generated access token
        for the given user.
        """
        access_token = AccessTokenFactory.create(user=user, client=self.oauth_client).token
        return 'Bearer ' + access_token

    def test_get_invalid_course(self):
        """
        The view should return a 404 for an invalid course ID.
        """
        self.assert_get_for_course(course_id=self.invalid_course_id, expected_status_code=404)

    def test_get(self):
        """
        The view should return a 200 for a valid course ID.
        """
        return self.assert_get_for_course()

    def test_not_authenticated(self):
        """
        The view should return HTTP status 401 if user is unauthenticated.
        """
        self.assert_get_for_course(expected_status_code=401, HTTP_AUTHORIZATION="")

    def test_staff_authorized(self):
        """
        The view should return a 200 when provided an access token
        for course staff.
        """
        user = StaffFactory(course_key=self.course.id)
        auth_header = self.get_auth_header(user)
        self.assert_get_for_course(HTTP_AUTHORIZATION=auth_header)

    def test_not_authorized(self):
        """
        The view should return HTTP status 404 when provided an
        access token for an unauthorized user.
        """
        user = UserFactory()
        auth_header = self.get_auth_header(user)
        self.assert_get_for_course(expected_status_code=403, HTTP_AUTHORIZATION=auth_header)

    @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
    def test_course_keys(self, modulestore_type):
        """
        The view should be addressable by course-keys from both module stores.
        """
        course = CourseFactory.create(
            start=datetime(2014, 6, 16, 14, 30, tzinfo=UTC),
            end=datetime(2015, 1, 16, tzinfo=UTC),
            org="MTD",
            default_store=modulestore_type,
        )
        self.assert_get_for_course(course_id=six.text_type(course.id))

class CourseGradingPolicyTests(GradingPolicyTestMixin, SharedModuleStoreTestCase):
    """
    Tests for CourseGradingPolicy view.
    """
    view_name = 'grades_api:v1:course_grading_policy'

    raw_grader = [
        {
            "min_count": 24,
            "weight": 0.2,
            "type": "Homework",
            "drop_count": 0,
            "short_label": "HW"
        },
        {
            "min_count": 4,
            "weight": 0.8,
            "type": "Exam",
            "drop_count": 0,
            "short_label": "Exam"
        }
    ]

    @classmethod
    def setUpClass(cls):
        super(CourseGradingPolicyTests, cls).setUpClass()
        cls.create_course_data()

    def test_get(self):
        """
        The view should return grading policy for a course.
        """
        response = super(CourseGradingPolicyTests, self).test_get()

        expected = [
            {
                "count": 24,
                "weight": 0.2,
                "assignment_type": "Homework",
                "dropped": 0
            },
            {
                "count": 4,
                "weight": 0.8,
                "assignment_type": "Exam",
                "dropped": 0
            }
        ]
        self.assertListEqual(response.data, expected)

class CourseGradingPolicyMissingFieldsTests(GradingPolicyTestMixin, SharedModuleStoreTestCase):
    """
    Tests for CourseGradingPolicy view when fields are missing.
    """
    view_name = 'grades_api:v1:course_grading_policy'

    # Raw grader with missing keys
    raw_grader = [
        {
            "min_count": 24,
            "weight": 0.2,
            "type": "Homework",
            "drop_count": 0,
            "short_label": "HW"
        },
        {
            # Deleted "min_count" key
            "weight": 0.8,
            "type": "Exam",
            "drop_count": 0,
            "short_label": "Exam"
        }
    ]

    @classmethod
    def setUpClass(cls):
        super(CourseGradingPolicyMissingFieldsTests, cls).setUpClass()
        cls.create_course_data()

    def test_get(self):
        """
        The view should return grading policy for a course.
        """
        response = super(CourseGradingPolicyMissingFieldsTests, self).test_get()

        expected = [
            {
                "count": 24,
                "weight": 0.2,
                "assignment_type": "Homework",
                "dropped": 0
            },
            {
                "count": None,
                "weight": 0.8,
                "assignment_type": "Exam",
                "dropped": 0
            }
        ]
        self.assertListEqual(response.data, expected)
<|endoftext|>


<commit_before>"""llvm

Tool-specific initialization for LLVM

"""

#
# Copyright (c) 2009 VMware, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

import os
import os.path
import subprocess

import SCons.Action
import SCons.Errors
import SCons.Util

def generate(env):
    try:
        llvm_dir = os.environ['LLVM']
    except KeyError:
        # Do nothing -- use the system headers/libs
        pass
    else:
        if not os.path.isdir(llvm_dir):
            raise SCons.Errors.InternalError, "Specified LLVM directory not found"

        if env['debug']:
            llvm_subdir = 'Debug'
        else:
            llvm_subdir = 'Release'

        llvm_bin_dir = os.path.join(llvm_dir, llvm_subdir, 'bin')
        if not os.path.isdir(llvm_bin_dir):
            raise SCons.Errors.InternalError, "LLVM build directory not found"

        env.PrependENVPath('PATH', llvm_bin_dir)

    if env.Detect('llvm-config'):
        pipe = SCons.Action._subproc(env,
                                     ['llvm-config', '--version'],
                                     stdin = 'devnull',
                                     stderr = 'devnull',
                                     stdout = subprocess.PIPE)
        if pipe.wait() != 0:
            return
        line = pipe.stdout.read().strip()
        if not line:
            return
        env['LLVM_VERSION'] = line

        env.ParseConfig('llvm-config --cppflags')
        env.ParseConfig('llvm-config --libs jit interpreter nativecodegen bitwriter')
        env.ParseConfig('llvm-config --ldflags')
        env['LINK'] = env['CXX']

def exists(env):
    return True

# vim:set ts=4 sw=4 et:
<commit_msg>scons: Don't use scons internal functions.<commit_after>"""llvm

Tool-specific initialization for LLVM

"""

#
# Copyright (c) 2009 VMware, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

import os
import os.path

import SCons.Errors
import SCons.Util

def generate(env):
    try:
        llvm_dir = os.environ['LLVM']
    except KeyError:
        # Do nothing -- use the system headers/libs
        pass
    else:
        if not os.path.isdir(llvm_dir):
            raise SCons.Errors.InternalError, "Specified LLVM directory not found"

        if env['debug']:
            llvm_subdir = 'Debug'
        else:
            llvm_subdir = 'Release'

        llvm_bin_dir = os.path.join(llvm_dir, llvm_subdir, 'bin')
        if not os.path.isdir(llvm_bin_dir):
            raise SCons.Errors.InternalError, "LLVM build directory not found"

        env.PrependENVPath('PATH', llvm_bin_dir)

    if env.Detect('llvm-config'):
        try:
            env['LLVM_VERSION'] = env.backtick('llvm-config --version')
        except NameError:
            env['LLVM_VERSION'] = 'X.X'

        env.ParseConfig('llvm-config --cppflags')
        env.ParseConfig('llvm-config --libs jit interpreter nativecodegen bitwriter')
        env.ParseConfig('llvm-config --ldflags')
        env['LINK'] = env['CXX']

def exists(env):
    return True

# vim:set ts=4 sw=4 et:
<|endoftext|>


<commit_before>import sys
import subprocess
sys.path.append('/Users/dmlockhart/vc/git-brg/parc/pymtl')

from pisa.pisa_inst_addiu_test import gen_basic_test
from pisa.pisa_inst_addiu_test import gen_dest_byp_test
from pisa.pisa_inst_addiu_test import gen_src_byp_test
from pisa.pisa_inst_addiu_test import gen_srcs_dest_test
from pisa.pisa_inst_addiu_test import gen_value_test
from pisa.pisa_inst_addiu_test import gen_random_test

cmd = './interp_asm_jit-c'
#cmd = 'python interp_asm_jit.py'
asm = 'test.s'

#-----------------------------------------------------------------------
# collect tests
#-----------------------------------------------------------------------
tests = []
test_names = []
for name, obj in vars().items():
  if not name.startswith('gen_'):
    continue
  test = obj()
  if isinstance( test, str ):
    test_names.append( name )
    tests.append( test )
  else:
    names = [ '{}_{}'.format( name, x ) for x, _ in enumerate(test) ]
    test_names.extend( names )
    tests.extend( test )

#-----------------------------------------------------------------------
# test_asm
#-----------------------------------------------------------------------
import pytest
@pytest.mark.parametrize( 'test_str', tests, ids=test_names )
def test_asm( test_str ):

  with open(asm, 'w') as asm_file:
    lines = test_str.split('\n')
    for line in lines:
      asm_file.write( line.lstrip()+'\n' )

  try:
    subprocess.check_call(
        (cmd+' '+asm).split(),
        env={'PYTHONPATH': '/Users/dmlockhart/vc/hg-opensource/pypy'}
    )
  except subprocess.CalledProcessError as e:
    print test_str
    raise e

  print "DONE"

<commit_msg>[rpython] autodetect imported tests<commit_after>import sys
import subprocess
sys.path.append('/Users/dmlockhart/vc/git-brg/parc/pymtl')

from inspect import getmembers, ismodule, isfunction

import pisa.pisa_inst_addiu_test
#import pisa

cmd = './interp_asm_jit-c'
#cmd = 'python interp_asm_jit.py'
asm = 'test.s'

#-----------------------------------------------------------------------
# collect tests
#-----------------------------------------------------------------------
tests = []
test_names = []

for mname, module in getmembers( pisa, ismodule ):
  if not (mname.startswith('pisa_inst') and mname.endswith('test')):
    continue
  for func_name, func in getmembers( module, isfunction ):
    if not (func_name.startswith('gen') and func.__module__.endswith( mname )):
      continue
    test = func()
    if isinstance( test, str ):
      test_names.append( func_name )
      tests.append( test )
    else:
      names = [ '{}_{}'.format( func_name, x ) for x, _ in enumerate(test) ]
      test_names.extend( names )
      tests.extend( test )

#-----------------------------------------------------------------------
# test_asm
#-----------------------------------------------------------------------
import pytest
@pytest.mark.parametrize( 'test_str', tests, ids=test_names )
def test_asm( test_str ):

  with open(asm, 'w') as asm_file:
    lines = test_str.split('\n')
    for line in lines:
      asm_file.write( line.lstrip()+'\n' )

  try:
    subprocess.check_call(
        (cmd+' '+asm).split(),
        env={'PYTHONPATH': '/Users/dmlockhart/vc/hg-opensource/pypy'}
    )
  except subprocess.CalledProcessError as e:
    print test_str
    raise e

  print "DONE"

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python
##-------------------------------------------------------------------
## @copyright 2017 DennyZhang.com
## Licensed under MIT
##   https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
##
## File : check_elasticsearch_replica.py
## Author : Denny <denny@dennyzhang.com>
## Description :
##    Check all ES indices have more than $min_replica_count replicas
## --
## Created : <2017-02-24>
## Updated: Time-stamp: <2017-03-13 17:42:54>
##-------------------------------------------------------------------
import argparse
import requests
import sys
import socket
import re

NAGIOS_OK_ERROR=0
NAGIOS_EXIT_ERROR=2

def get_es_index_list(es_host, es_port):
    index_list = []
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    r = requests.get(url)
    '''
Sample output:
root@test:/# curl 172.17.0.8:9200/_cat/indices?v
health status index                                          pri rep docs.count docs.deleted store.size pri.store.size
green  open   master-index-098f6bcd4621d373cade4e832627b4f6    1   0          1            0      8.1kb          8.1kb
green  open   master-index-13a1f8adbec032ed68f3d035449ef48d    1   0          1            0     10.6kb         10.6kb
...
...
'''
    # TODO: error handling, if curl requests fails
    for line in r.content.split("\n"):
        # remove the header
        if " index " in line or line == '':
            continue
        else:
            # get the column of index name
            line = ' '.join(line.split())
            l = line.split(" ")
            index_list.append(l[2])
    return index_list

def get_es_replica_count(es_host, es_port, index_name):
    number_of_replicas = -1
    url = "http://%s:%s/%s/_settings?pretty" % (es_host, es_port, index_name)
    r = requests.get(url)
    '''
Sample output:
        ...
        ...
        },
        "number_of_replicas" : "0",
        "uuid" : "31CQbyYKTT6UGtI3lbPfAg",
        "version" : {
          "created" : "2030399"
        ...
        ...
    '''
    # TODO: error handling, if curl requests fails
    for line in r.content.split("\n"):
        if "\"number_of_replicas\" : " in line:
            value = line.split(":")[1]
            value = value.replace(" ", "").replace("\"", "").replace(",", "")
            number_of_replicas = int(value)
    if number_of_replicas == -1:
        raise Exception("Error: fail to get index replica for %s." % (index_name))
    return number_of_replicas

def confirm_es_replica_count(es_host, es_port, es_index_list, \
                             min_replica_count, es_pattern_regexp):
    # Check all ES indices have more than $min_replica_count replicas
    failed_index_list = []
    for index_name in es_index_list:
        if es_pattern_regexp != "":
            m = re.search(es_pattern_regexp, index_name)
            # Skip ES index which doesn't match the pattern
            if m is None:
                continue
        number_of_replicas = get_es_replica_count(es_host, es_port, index_name)
        if number_of_replicas < min_replica_count:
            print "ERROR: index(%s) only has %d replicas, less than %d." \
                % (index_name, number_of_replicas, min_replica_count)
            failed_index_list.append(index_name)
    return failed_index_list

if __name__ == '__main__':
    # get parameters from users
    parser = argparse.ArgumentParser()
    parser.add_argument('--es_host', required=False, \
                        help="server ip or hostname for elasticsearch instance. Default value is ip of eth0", type=str)
    parser.add_argument('--es_port', default='9200', required=False, \
                        help="server port for elasticsearch instance", type=str)
    parser.add_argument('--es_pattern_regexp', required=False, default='', \
                        help="ES index name pattern. Only ES indices with matched pattern will be examined", type=str)
    parser.add_argument('--min_replica_count', default=1, required=False, \
                        help="minimal replica each elasticsearch index should have", type=str)
    l = parser.parse_args()

    es_port = l.es_port
    min_replica_count = int(l.min_replica_count)
    es_pattern_regexp = l.es_pattern_regexp
    es_host = l.es_host

    if min_replica_count == 0:
        print "OK: skip the check, since the given min_replica_count is 0"
        sys.exit(NAGIOS_OK_ERROR)

    # get ip of eth0, if es_host is not given
    if es_host is None:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        es_host = s.getsockname()[0]

    es_index_list = get_es_index_list(es_host, es_port)

    failed_index_list = confirm_es_replica_count(es_host, es_port, es_index_list, \
                                                 min_replica_count, es_pattern_regexp)

    if len(failed_index_list) != 0:
        print "ERROR: Below indices don't have enough replica:\n%s" % \
            (",".join(failed_index_list))
        sys.exit(NAGIOS_EXIT_ERROR)
    else:
        print "OK: all ES indices have no less than %d replicas" % (min_replica_count)
        sys.exit(NAGIOS_OK_ERROR)
## File : check_elasticsearch_replica.py ends
<commit_msg>update shard<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python
##-------------------------------------------------------------------
## @copyright 2017 DennyZhang.com
## Licensed under MIT
##   https://raw.githubusercontent.com/DennyZhang/devops_public/master/LICENSE
##
## File : check_elasticsearch_replica.py
## Author : Denny <denny@dennyzhang.com>
## Description :
##    Check all ES indices have more than $min_replica_count replicas
## --
## Created : <2017-02-24>
## Updated: Time-stamp: <2017-03-27 19:26:29>
##-------------------------------------------------------------------
import argparse
import requests
import sys
import socket
import re

NAGIOS_OK_ERROR=0
NAGIOS_EXIT_ERROR=2

def get_es_index_list(es_host, es_port):
    index_list = []
    url = "http://%s:%s/_cat/indices?v" % (es_host, es_port)
    r = requests.get(url)
    '''
Sample output:
root@test:/# curl 172.17.0.8:9200/_cat/indices?v
health status index                                          pri rep docs.count docs.deleted store.size pri.store.size
green  open   master-index-098f6bcd4621d373cade4e832627b4f6    1   0          1            0      8.1kb          8.1kb
green  open   master-index-13a1f8adbec032ed68f3d035449ef48d    1   0          1            0     10.6kb         10.6kb
...
...
'''
    # TODO: error handling, if curl requests fails
    for line in r.content.split("\n"):
        # remove the header, and skip closed ES indices
        if line == '' or " index " in line  or " close " in line:
        else:
            # get the column of index name
            line = ' '.join(line.split())
            l = line.split(" ")
            index_list.append(l[2])
    return index_list

def get_es_replica_count(es_host, es_port, index_name):
    number_of_replicas = -1
    url = "http://%s:%s/%s/_settings?pretty" % (es_host, es_port, index_name)
    r = requests.get(url)
    '''
Sample output:
        ...
        ...
        },
        "number_of_replicas" : "0",
        "uuid" : "31CQbyYKTT6UGtI3lbPfAg",
        "version" : {
          "created" : "2030399"
        ...
        ...
    '''
    # TODO: error handling, if curl requests fails
    for line in r.content.split("\n"):
        if "\"number_of_replicas\" : " in line:
            value = line.split(":")[1]
            value = value.replace(" ", "").replace("\"", "").replace(",", "")
            number_of_replicas = int(value)
    if number_of_replicas == -1:
        raise Exception("Error: fail to get index replica for %s." % (index_name))
    return number_of_replicas

def confirm_es_replica_count(es_host, es_port, es_index_list, \
                             min_replica_count, es_pattern_regexp):
    # Check all ES indices have more than $min_replica_count replicas
    failed_index_list = []
    for index_name in es_index_list:
        if es_pattern_regexp != "":
            m = re.search(es_pattern_regexp, index_name)
            # Skip ES index which doesn't match the pattern
            if m is None:
                continue
        number_of_replicas = get_es_replica_count(es_host, es_port, index_name)
        if number_of_replicas < min_replica_count:
            print "ERROR: index(%s) only has %d replicas, less than %d." \
                % (index_name, number_of_replicas, min_replica_count)
            failed_index_list.append(index_name)
    return failed_index_list

if __name__ == '__main__':
    # get parameters from users
    parser = argparse.ArgumentParser()
    parser.add_argument('--es_host', required=False, \
                        help="server ip or hostname for elasticsearch instance. Default value is ip of eth0", type=str)
    parser.add_argument('--es_port', default='9200', required=False, \
                        help="server port for elasticsearch instance", type=str)
    parser.add_argument('--es_pattern_regexp', required=False, default='', \
                        help="ES index name pattern. Only ES indices with matched pattern will be examined", type=str)
    parser.add_argument('--min_replica_count', default=1, required=False, \
                        help="minimal replica each elasticsearch index should have", type=str)
    l = parser.parse_args()

    es_port = l.es_port
    min_replica_count = int(l.min_replica_count)
    es_pattern_regexp = l.es_pattern_regexp
    es_host = l.es_host

    if min_replica_count == 0:
        print "OK: skip the check, since the given min_replica_count is 0"
        sys.exit(NAGIOS_OK_ERROR)

    # get ip of eth0, if es_host is not given
    if es_host is None:
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(("8.8.8.8", 80))
        es_host = s.getsockname()[0]

    es_index_list = get_es_index_list(es_host, es_port)

    failed_index_list = confirm_es_replica_count(es_host, es_port, es_index_list, \
                                                 min_replica_count, es_pattern_regexp)

    if len(failed_index_list) != 0:
        print "ERROR: Below indices don't have enough replica:\n%s" % \
            (",".join(failed_index_list))
        sys.exit(NAGIOS_EXIT_ERROR)
    else:
        print "OK: all ES indices have no less than %d replicas" % (min_replica_count)
        sys.exit(NAGIOS_OK_ERROR)
## File : check_elasticsearch_replica.py ends
<|endoftext|>


<commit_before>sizes = [1024] #[10240]

DACIT_URL = "dacit.cs.uni-dortmund.de/datasets"

download_and_extract("large", sizes, [
    DACIT_URL + "/commoncrawl_10240.ascii",

    # http://acube.di.unipi.it/rlzap-dataset/
    DACIT_URL + "/rlzap_dlcp_female",

    # http://www.internationalgenome.org/data-portal/sample/NA19000
    DACIT_URL + "/1000Genome_JPT_NA19000_SRR099528_1.fastq.filtered4_no_N.dna",

    # https://panthema.net/2012/1119-eSAIS-Inducing-Suffix-and-LCP-Arrays-in-External-Memory/
    # Gutenberg Concatenation from September 2012
    DACIT_URL + "/gutenberg-201209.24090588160.xz",
])
<commit_msg>actually generate 10Gb texts<commit_after>sizes = [1, 10240]

DACIT_URL = "dacit.cs.uni-dortmund.de/datasets"

download_and_extract("large", sizes, [
    DACIT_URL + "/commoncrawl_10240.ascii",

    # http://acube.di.unipi.it/rlzap-dataset/
    # (array of 32bit values)
    DACIT_URL + "/rlzap_dlcp_female",

    # http://www.internationalgenome.org/data-portal/sample/NA19000
    DACIT_URL + "/1000Genome_JPT_NA19000_SRR099528_1.fastq.filtered4_no_N.dna",

    # https://panthema.net/2012/1119-eSAIS-Inducing-Suffix-and-LCP-Arrays-in-External-Memory/
    # Gutenberg Concatenation from September 2012
    DACIT_URL + "/gutenberg-201209.24090588160.xz",
])
<|endoftext|>


<commit_before># This file is part of morss
#
# Copyright (C) 2013-2020 pictuga <contact@pictuga.com>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.

import cgitb
import mimetypes
import os.path
import re
import sys
import wsgiref.handlers
import wsgiref.simple_server
import wsgiref.util

import lxml.etree

try:
    # python 2
    from urllib import unquote
except ImportError:
    # python 3
    from urllib.parse import unquote

from . import caching, crawler, readabilite
from .morss import (DELAY, TIMEOUT, FeedFetch, FeedFormat, FeedGather,
                    MorssException, Options, log)
from .util import data_path

PORT = int(os.getenv('PORT', 8000))

def parse_options(options):
    """ Turns ['md=True'] into {'md':True} """
    out = {}

    for option in options:
        split = option.split('=', 1)

        if len(split) > 1:
            out[split[0]] = unquote(split[1]).replace('|', '/') # | -> / for backward compatibility (and Apache)

        else:
            out[split[0]] = True

    return out

def request_uri(environ):
    if 'REQUEST_URI' in environ:
        # when running on Apache/uwsgi
        url = environ['REQUEST_URI']

    elif 'RAW_URI' in environ:
        # gunicorn
        url = environ['RAW_URI']

    else:
        # when using other servers
        url = environ['PATH_INFO']

        if environ['QUERY_STRING']:
            url += '?' + environ['QUERY_STRING']

    return url

def cgi_parse_environ(environ):
    # get options

    url = request_uri(environ)[1:]
    url = re.sub(r'^(cgi/)?(morss.py|main.py)/', '', url)

    if url.startswith(':'):
        parts = url.split('/', 1)
        raw_options = parts[0].split(':')[1:]
        url = parts[1] if len(parts) > 1 else ''

    else:
        raw_options = []

    # init
    options = Options(parse_options(raw_options))

    return (url, options)

def cgi_app(environ, start_response):
    url, options = cgi_parse_environ(environ)

    headers = {}

    # headers
    headers['status'] = '200 OK'
    headers['cache-control'] = 'max-age=%s' % DELAY
    headers['x-content-type-options'] = 'nosniff' # safari work around

    if options.cors:
        headers['access-control-allow-origin'] = '*'

    if options.format == 'html':
        headers['content-type'] = 'text/html'
    elif options.txt or options.silent:
        headers['content-type'] = 'text/plain'
    elif options.format == 'json':
        headers['content-type'] = 'application/json'
    elif options.callback:
        headers['content-type'] = 'application/javascript'
    elif options.format == 'csv':
        headers['content-type'] = 'text/csv'
        headers['content-disposition'] = 'attachment; filename="feed.csv"'
    else:
        headers['content-type'] = 'text/xml'

    headers['content-type'] += '; charset=utf-8'

    # get the work done
    url, rss = FeedFetch(url, options)

    start_response(headers['status'], list(headers.items()))

    rss = FeedGather(rss, url, options)
    out = FeedFormat(rss, options)

    if options.silent:
        return ['']

    else:
        return [out]

def middleware(func):
    " Decorator to turn a function into a wsgi middleware "
    # This is called when parsing the "@middleware" code

    def app_builder(app):
        # This is called when doing app = cgi_wrapper(app)

        def app_wrap(environ, start_response):
            # This is called when a http request is being processed

            return func(environ, start_response, app)

        return app_wrap

    return app_builder

@middleware
def cgi_file_handler(environ, start_response, app):
    " Simple HTTP server to serve static files (.html, .css, etc.) "

    url = request_uri(environ)[1:]

    if url == '':
        url = 'index.html'

    if re.match(r'^/?([a-zA-Z0-9_-][a-zA-Z0-9\._-]+/?)*$', url):
        # if it is a legitimate url (no funny relative paths)
        try:
            f = open(data_path(url), 'rb')

        except IOError:
            # problem with file (cannot open or not found)
            continue

        else:
            # file successfully open
            headers = {}
            headers['status'] = '200 OK'
            headers['content-type'] = mimetypes.guess_type(path)[0] or 'application/octet-stream'
            start_response(headers['status'], list(headers.items()))
            return wsgiref.util.FileWrapper(f)

    # regex didn't validate or no file found
    return app(environ, start_response)

def cgi_get(environ, start_response):
    url, options = cgi_parse_environ(environ)

    # get page
    req = crawler.adv_get(url=url, timeout=TIMEOUT)

    if req['contenttype'] in ['text/html', 'application/xhtml+xml', 'application/xml']:
        if options['get'] == 'page':
            html = readabilite.parse(req['data'], encoding=req['encoding'])
            html.make_links_absolute(req['url'])

            kill_tags = ['script', 'iframe', 'noscript']

            for tag in kill_tags:
                for elem in html.xpath('//'+tag):
                    elem.getparent().remove(elem)

            output = lxml.etree.tostring(html.getroottree(), encoding='utf-8', method='html')

        elif options['get'] == 'article':
            output = readabilite.get_article(req['data'], url=req['url'], encoding_in=req['encoding'], encoding_out='utf-8', debug=options.debug)

        else:
            raise MorssException('no :get option passed')

    else:
        output = req['data']

    # return html page
    headers = {'status': '200 OK', 'content-type': 'text/html; charset=utf-8', 'X-Frame-Options': 'SAMEORIGIN'} # SAMEORIGIN to avoid potential abuse
    start_response(headers['status'], list(headers.items()))
    return [output]

dispatch_table = {
    'get': cgi_get,
    }

@middleware
def cgi_dispatcher(environ, start_response, app):
    url, options = cgi_parse_environ(environ)

    for key in dispatch_table.keys():
        if key in options:
            return dispatch_table[key](environ, start_response)

    return app(environ, start_response)

@middleware
def cgi_error_handler(environ, start_response, app):
    try:
        return app(environ, start_response)

    except (KeyboardInterrupt, SystemExit):
        raise

    except Exception as e:
        headers = {'status': '404 Not Found', 'content-type': 'text/html', 'x-morss-error': repr(e)}
        start_response(headers['status'], list(headers.items()), sys.exc_info())
        log('ERROR: %s' % repr(e))
        return [cgitb.html(sys.exc_info())]

@middleware
def cgi_encode(environ, start_response, app):
    out = app(environ, start_response)
    return [x if isinstance(x, bytes) else str(x).encode('utf-8') for x in out]

application = cgi_app
application = cgi_file_handler(application)
application = cgi_dispatcher(application)
application = cgi_error_handler(application)
application = cgi_encode(application)

def cgi_handle_request():
    app = cgi_app
    app = cgi_dispatcher(app)
    app = cgi_error_handler(app)
    app = cgi_encode(app)

    wsgiref.handlers.CGIHandler().run(app)

class WSGIRequestHandlerRequestUri(wsgiref.simple_server.WSGIRequestHandler):
    def get_environ(self):
        env = super().get_environ()
        env['REQUEST_URI'] = self.path
        return env

def cgi_start_server():
    caching.default_cache.autotrim()

    print('Serving http://localhost:%s/' % PORT)
    httpd = wsgiref.simple_server.make_server('', PORT, application, handler_class=WSGIRequestHandlerRequestUri)
    httpd.serve_forever()

if 'gunicorn' in os.getenv('SERVER_SOFTWARE', ''):
    caching.default_cache.autotrim()
<commit_msg>wsgi: fix bug following the removal of the loop<commit_after># This file is part of morss
#
# Copyright (C) 2013-2020 pictuga <contact@pictuga.com>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License along
# with this program. If not, see <https://www.gnu.org/licenses/>.

import cgitb
import mimetypes
import os.path
import re
import sys
import wsgiref.handlers
import wsgiref.simple_server
import wsgiref.util

import lxml.etree

try:
    # python 2
    from urllib import unquote
except ImportError:
    # python 3
    from urllib.parse import unquote

from . import caching, crawler, readabilite
from .morss import (DELAY, TIMEOUT, FeedFetch, FeedFormat, FeedGather,
                    MorssException, Options, log)
from .util import data_path

PORT = int(os.getenv('PORT', 8000))

def parse_options(options):
    """ Turns ['md=True'] into {'md':True} """
    out = {}

    for option in options:
        split = option.split('=', 1)

        if len(split) > 1:
            out[split[0]] = unquote(split[1]).replace('|', '/') # | -> / for backward compatibility (and Apache)

        else:
            out[split[0]] = True

    return out

def request_uri(environ):
    if 'REQUEST_URI' in environ:
        # when running on Apache/uwsgi
        url = environ['REQUEST_URI']

    elif 'RAW_URI' in environ:
        # gunicorn
        url = environ['RAW_URI']

    else:
        # when using other servers
        url = environ['PATH_INFO']

        if environ['QUERY_STRING']:
            url += '?' + environ['QUERY_STRING']

    return url

def cgi_parse_environ(environ):
    # get options

    url = request_uri(environ)[1:]
    url = re.sub(r'^(cgi/)?(morss.py|main.py)/', '', url)

    if url.startswith(':'):
        parts = url.split('/', 1)
        raw_options = parts[0].split(':')[1:]
        url = parts[1] if len(parts) > 1 else ''

    else:
        raw_options = []

    # init
    options = Options(parse_options(raw_options))

    return (url, options)

def cgi_app(environ, start_response):
    url, options = cgi_parse_environ(environ)

    headers = {}

    # headers
    headers['status'] = '200 OK'
    headers['cache-control'] = 'max-age=%s' % DELAY
    headers['x-content-type-options'] = 'nosniff' # safari work around

    if options.cors:
        headers['access-control-allow-origin'] = '*'

    if options.format == 'html':
        headers['content-type'] = 'text/html'
    elif options.txt or options.silent:
        headers['content-type'] = 'text/plain'
    elif options.format == 'json':
        headers['content-type'] = 'application/json'
    elif options.callback:
        headers['content-type'] = 'application/javascript'
    elif options.format == 'csv':
        headers['content-type'] = 'text/csv'
        headers['content-disposition'] = 'attachment; filename="feed.csv"'
    else:
        headers['content-type'] = 'text/xml'

    headers['content-type'] += '; charset=utf-8'

    # get the work done
    url, rss = FeedFetch(url, options)

    start_response(headers['status'], list(headers.items()))

    rss = FeedGather(rss, url, options)
    out = FeedFormat(rss, options)

    if options.silent:
        return ['']

    else:
        return [out]

def middleware(func):
    " Decorator to turn a function into a wsgi middleware "
    # This is called when parsing the "@middleware" code

    def app_builder(app):
        # This is called when doing app = cgi_wrapper(app)

        def app_wrap(environ, start_response):
            # This is called when a http request is being processed

            return func(environ, start_response, app)

        return app_wrap

    return app_builder

@middleware
def cgi_file_handler(environ, start_response, app):
    " Simple HTTP server to serve static files (.html, .css, etc.) "

    url = request_uri(environ)[1:]

    if url == '':
        url = 'index.html'

    if re.match(r'^/?([a-zA-Z0-9_-][a-zA-Z0-9\._-]+/?)*$', url):
        # if it is a legitimate url (no funny relative paths)
        try:
            f = open(data_path(url), 'rb')

        except IOError:
            # problem with file (cannot open or not found)
            pass

        else:
            # file successfully open
            headers = {}
            headers['status'] = '200 OK'
            headers['content-type'] = mimetypes.guess_type(path)[0] or 'application/octet-stream'
            start_response(headers['status'], list(headers.items()))
            return wsgiref.util.FileWrapper(f)

    # regex didn't validate or no file found
    return app(environ, start_response)

def cgi_get(environ, start_response):
    url, options = cgi_parse_environ(environ)

    # get page
    req = crawler.adv_get(url=url, timeout=TIMEOUT)

    if req['contenttype'] in ['text/html', 'application/xhtml+xml', 'application/xml']:
        if options['get'] == 'page':
            html = readabilite.parse(req['data'], encoding=req['encoding'])
            html.make_links_absolute(req['url'])

            kill_tags = ['script', 'iframe', 'noscript']

            for tag in kill_tags:
                for elem in html.xpath('//'+tag):
                    elem.getparent().remove(elem)

            output = lxml.etree.tostring(html.getroottree(), encoding='utf-8', method='html')

        elif options['get'] == 'article':
            output = readabilite.get_article(req['data'], url=req['url'], encoding_in=req['encoding'], encoding_out='utf-8', debug=options.debug)

        else:
            raise MorssException('no :get option passed')

    else:
        output = req['data']

    # return html page
    headers = {'status': '200 OK', 'content-type': 'text/html; charset=utf-8', 'X-Frame-Options': 'SAMEORIGIN'} # SAMEORIGIN to avoid potential abuse
    start_response(headers['status'], list(headers.items()))
    return [output]

dispatch_table = {
    'get': cgi_get,
    }

@middleware
def cgi_dispatcher(environ, start_response, app):
    url, options = cgi_parse_environ(environ)

    for key in dispatch_table.keys():
        if key in options:
            return dispatch_table[key](environ, start_response)

    return app(environ, start_response)

@middleware
def cgi_error_handler(environ, start_response, app):
    try:
        return app(environ, start_response)

    except (KeyboardInterrupt, SystemExit):
        raise

    except Exception as e:
        headers = {'status': '404 Not Found', 'content-type': 'text/html', 'x-morss-error': repr(e)}
        start_response(headers['status'], list(headers.items()), sys.exc_info())
        log('ERROR: %s' % repr(e))
        return [cgitb.html(sys.exc_info())]

@middleware
def cgi_encode(environ, start_response, app):
    out = app(environ, start_response)
    return [x if isinstance(x, bytes) else str(x).encode('utf-8') for x in out]

application = cgi_app
application = cgi_file_handler(application)
application = cgi_dispatcher(application)
application = cgi_error_handler(application)
application = cgi_encode(application)

def cgi_handle_request():
    app = cgi_app
    app = cgi_dispatcher(app)
    app = cgi_error_handler(app)
    app = cgi_encode(app)

    wsgiref.handlers.CGIHandler().run(app)

class WSGIRequestHandlerRequestUri(wsgiref.simple_server.WSGIRequestHandler):
    def get_environ(self):
        env = super().get_environ()
        env['REQUEST_URI'] = self.path
        return env

def cgi_start_server():
    caching.default_cache.autotrim()

    print('Serving http://localhost:%s/' % PORT)
    httpd = wsgiref.simple_server.make_server('', PORT, application, handler_class=WSGIRequestHandlerRequestUri)
    httpd.serve_forever()

if 'gunicorn' in os.getenv('SERVER_SOFTWARE', ''):
    caching.default_cache.autotrim()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

""" Models registries.

"""
from contextlib import contextmanager
import logging
import threading

import openerp.sql_db
import openerp.osv.orm
import openerp.tools
import openerp.modules.db
import openerp.tools.config
from openerp.tools import assertion_report

_logger = logging.getLogger(__name__)

class Registry(object):
    """ Model registry for a particular database.

    The registry is essentially a mapping between model names and model
    instances. There is one registry instance per database.

    """

    def __init__(self, db_name):
        self.models = {}    # model name/model instance mapping
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self._assertion_report = assertion_report.assertion_report()
        self.fields_by_model = None

        # modules fully loaded (maintained during init phase by `loading` module)
        self._init_modules = set()

        self.db_name = db_name
        self.db = openerp.sql_db.db_connect(db_name)

        # Indicates that the registry is
        self.ready = False

        # Inter-process signaling (used only when openerp.multi_process is True):
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        self.base_registry_signaling_sequence = 1
        self.base_cache_signaling_sequence = 1

        # Flag indicating if at least one model cache has been cleared.
        # Useful only in a multi-process context.
        self._any_cache_cleared = False

        cr = self.db.cursor()
        has_unaccent = openerp.modules.db.has_unaccent(cr)
        if openerp.tools.config['unaccent'] and not has_unaccent:
            _logger.warning("The option --unaccent was given but no unaccent() function was found in database.")
        self.has_unaccent = openerp.tools.config['unaccent'] and has_unaccent
        cr.close()

    def do_parent_store(self, cr):
        for o in self._init_parent:
            self.get(o)._parent_store_compute(cr)
        self._init = False

    def obj_list(self):
        """ Return the list of model names in this registry."""
        return self.models.keys()

    def add(self, model_name, model):
        """ Add or replace a model in the registry."""
        self.models[model_name] = model

    def get(self, model_name):
        """ Return a model for a given name or None if it doesn't exist."""
        return self.models.get(model_name)

    def __getitem__(self, model_name):
        """ Return a model for a given name or raise KeyError if it doesn't exist."""
        return self.models[model_name]

    def load(self, cr, module):
        """ Load a given module in the registry.

        At the Python level, the modules are already loaded, but not yet on a
        per-registry level. This method populates a registry with the given
        modules, i.e. it instanciates all the classes of a the given module
        and registers them in the registry.

        """
        models_to_load = [] # need to preserve loading order
        # Instantiate registered classes (via the MetaModel automatic discovery
        # or via explicit constructor call), and add them to the pool.
        for cls in openerp.osv.orm.MetaModel.module_to_models.get(module.name, []):
            # models register themselves in self.models
            model = cls.create_instance(self, cr)
            if model._name not in models_to_load:
                # avoid double-loading models whose declaration is split
                models_to_load.append(model._name)
        return [self.models[m] for m in models_to_load]

    def clear_caches(self):
        """ Clear the caches
        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models.
        """
        for model in self.models.itervalues():
            model.clear_caches()
        # Special case for ir_ui_menu which does not use openerp.tools.ormcache.
        ir_ui_menu = self.models.get('ir.ui.menu')
        if ir_ui_menu:
            ir_ui_menu.clear_cache()

    # Useful only in a multi-process context.
    def reset_any_cache_cleared(self):
        self._any_cache_cleared = False

    # Useful only in a multi-process context.
    def any_cache_cleared(self):
        return self._any_cache_cleared

    @classmethod
    def setup_multi_process_signaling(cls, cr):
        if not openerp.multi_process:
            return

        # Inter-process signaling:
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        cr.execute("""SELECT sequence_name FROM information_schema.sequences WHERE sequence_name='base_registry_signaling'""")
        if not cr.fetchall():
            cr.execute("""CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1""")
            cr.execute("""SELECT nextval('base_registry_signaling')""")
            cr.execute("""CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1""")
            cr.execute("""SELECT nextval('base_cache_signaling')""")

    @contextmanager
    def cursor(self, auto_commit=True):
        cr = self.db.cursor()
        try:
            yield cr
            if auto_commit:
                cr.commit()
        finally:
            cr.close()

class RegistryManager(object):
    """ Model registries manager.

        The manager is responsible for creation and deletion of model
        registries (essentially database connection/model registry pairs).

    """
    # Mapping between db name and model registry.
    # Accessed through the methods below.
    registries = {}
    registries_lock = threading.RLock()

    @classmethod
    def get(cls, db_name, force_demo=False, status=None, update_module=False):
        """ Return a registry for a given database name."""
        try:
            return cls.registries[db_name]
        except KeyError:
            return cls.new(db_name, force_demo, status,
                           update_module)

    @classmethod
    def new(cls, db_name, force_demo=False, status=None,
            update_module=False):
        """ Create and return a new registry for a given database name.

        The (possibly) previous registry for that database name is discarded.

        """
        import openerp.modules
        with cls.registries_lock:
            registry = Registry(db_name)

            # Initializing a registry will call general code which will in turn
            # call registries.get (this object) to obtain the registry being
            # initialized. Make it available in the registries dictionary then
            # remove it if an exception is raised.
            cls.delete(db_name)
            cls.registries[db_name] = registry
            try:
                # This should be a method on Registry
                openerp.modules.load_modules(registry.db, force_demo, status, update_module)
            except Exception:
                del cls.registries[db_name]
                raise

            # load_modules() above can replace the registry by calling
            # indirectly new() again (when modules have to be uninstalled).
            # Yeah, crazy.
            registry = cls.registries[db_name]

            cr = registry.db.cursor()
            try:
                Registry.setup_multi_process_signaling(cr)
                registry.do_parent_store(cr)
                cr.commit()
            finally:
                cr.close()

        registry.ready = True

        return registry

    @classmethod
    def delete(cls, db_name):
        """Delete the registry linked to a given database.  """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()
                del cls.registries[db_name]

    @classmethod
    def delete_all(cls):
        """Delete all the registries. """
        with cls.registries_lock:
            for db_name in cls.registries.keys():
                cls.delete(db_name)

    @classmethod
    def clear_caches(cls, db_name):
        """Clear caches

        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models
        of the given database name.

        This method is given to spare you a ``RegistryManager.get(db_name)``
        that would loads the given database if it was not already loaded.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()

    @classmethod
    def check_registry_signaling(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name)
            cr = registry.db.cursor()
            try:
                cr.execute("""
                    SELECT base_registry_signaling.last_value,
                           base_cache_signaling.last_value
                    FROM base_registry_signaling, base_cache_signaling""")
                r, c = cr.fetchone()
                # Check if the model registry must be reloaded (e.g. after the
                # database has been updated by another process).
                if registry.base_registry_signaling_sequence != r:
                    _logger.info("Reloading the model registry after database signaling.")
                    registry = cls.new(db_name)
                    registry.base_registry_signaling_sequence = r
                # Check if the model caches must be invalidated (e.g. after a write
                # occured on another process). Don't clear right after a registry
                # has been reload.
                elif registry.base_cache_signaling_sequence != c:
                    _logger.info("Invalidating all model caches after database signaling.")
                    registry.base_cache_signaling_sequence = c
                    registry.clear_caches()
                    registry.reset_any_cache_cleared()
                    # One possible reason caches have been invalidated is the
                    # use of decimal_precision.write(), in which case we need
                    # to refresh fields.float columns.
                    for model in registry.models.values():
                        for column in model._columns.values():
                            if hasattr(column, 'digits_change'):
                                column.digits_change(cr)
            finally:
                cr.close()

    @classmethod
    def signal_caches_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check the registries if any cache has been cleared and signal it
            # through the database to other processes.
            registry = cls.get(db_name)
            if registry.any_cache_cleared():
                _logger.info("At least one model cache has been cleared, signaling through the database.")
                cr = registry.db.cursor()
                r = 1
                try:
                    cr.execute("select nextval('base_cache_signaling')")
                    r = cr.fetchone()[0]
                finally:
                    cr.close()
                registry.base_cache_signaling_sequence = r
                registry.reset_any_cache_cleared()

    @classmethod
    def signal_registry_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name)
            cr = registry.db.cursor()
            r = 1
            try:
                cr.execute("select nextval('base_registry_signaling')")
                r = cr.fetchone()[0]
            finally:
                cr.close()
            registry.base_registry_signaling_sequence = r

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>[IMP] registry: add convenience operators ('in' and 'iter') on registry objects<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    OpenERP, Open Source Management Solution
#    Copyright (C) 2004-2009 Tiny SPRL (<http://tiny.be>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

""" Models registries.

"""
from contextlib import contextmanager
import logging
import threading

import openerp.sql_db
import openerp.osv.orm
import openerp.tools
import openerp.modules.db
import openerp.tools.config
from openerp.tools import assertion_report

_logger = logging.getLogger(__name__)

class Registry(object):
    """ Model registry for a particular database.

    The registry is essentially a mapping between model names and model
    instances. There is one registry instance per database.

    """

    def __init__(self, db_name):
        self.models = {}    # model name/model instance mapping
        self._sql_error = {}
        self._store_function = {}
        self._init = True
        self._init_parent = {}
        self._assertion_report = assertion_report.assertion_report()
        self.fields_by_model = None

        # modules fully loaded (maintained during init phase by `loading` module)
        self._init_modules = set()

        self.db_name = db_name
        self.db = openerp.sql_db.db_connect(db_name)

        # Indicates that the registry is
        self.ready = False

        # Inter-process signaling (used only when openerp.multi_process is True):
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        self.base_registry_signaling_sequence = 1
        self.base_cache_signaling_sequence = 1

        # Flag indicating if at least one model cache has been cleared.
        # Useful only in a multi-process context.
        self._any_cache_cleared = False

        cr = self.db.cursor()
        has_unaccent = openerp.modules.db.has_unaccent(cr)
        if openerp.tools.config['unaccent'] and not has_unaccent:
            _logger.warning("The option --unaccent was given but no unaccent() function was found in database.")
        self.has_unaccent = openerp.tools.config['unaccent'] and has_unaccent
        cr.close()

    def do_parent_store(self, cr):
        for o in self._init_parent:
            self.get(o)._parent_store_compute(cr)
        self._init = False

    def obj_list(self):
        """ Return the list of model names in this registry."""
        return self.models.keys()

    def add(self, model_name, model):
        """ Add or replace a model in the registry."""
        self.models[model_name] = model

    def get(self, model_name):
        """ Return the model with the given name or None if it doesn't exist."""
        return self.models.get(model_name)

    def __getitem__(self, model_name):
        """ Return the model with the given name or raise KeyError if it doesn't exist."""
        return self.models[model_name]

    def __contains__(self, model_name):
        """ Test whether the model with the given name exists. """
        return model_name in self.models

    def __iter__(self):
        """ Return an iterator over all model names. """
        return iter(self.models)

    def load(self, cr, module):
        """ Load a given module in the registry.

        At the Python level, the modules are already loaded, but not yet on a
        per-registry level. This method populates a registry with the given
        modules, i.e. it instanciates all the classes of a the given module
        and registers them in the registry.

        """
        models_to_load = [] # need to preserve loading order
        # Instantiate registered classes (via the MetaModel automatic discovery
        # or via explicit constructor call), and add them to the pool.
        for cls in openerp.osv.orm.MetaModel.module_to_models.get(module.name, []):
            # models register themselves in self.models
            model = cls.create_instance(self, cr)
            if model._name not in models_to_load:
                # avoid double-loading models whose declaration is split
                models_to_load.append(model._name)
        return [self.models[m] for m in models_to_load]

    def clear_caches(self):
        """ Clear the caches
        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models.
        """
        for model in self.models.itervalues():
            model.clear_caches()
        # Special case for ir_ui_menu which does not use openerp.tools.ormcache.
        ir_ui_menu = self.models.get('ir.ui.menu')
        if ir_ui_menu:
            ir_ui_menu.clear_cache()

    # Useful only in a multi-process context.
    def reset_any_cache_cleared(self):
        self._any_cache_cleared = False

    # Useful only in a multi-process context.
    def any_cache_cleared(self):
        return self._any_cache_cleared

    @classmethod
    def setup_multi_process_signaling(cls, cr):
        if not openerp.multi_process:
            return

        # Inter-process signaling:
        # The `base_registry_signaling` sequence indicates the whole registry
        # must be reloaded.
        # The `base_cache_signaling sequence` indicates all caches must be
        # invalidated (i.e. cleared).
        cr.execute("""SELECT sequence_name FROM information_schema.sequences WHERE sequence_name='base_registry_signaling'""")
        if not cr.fetchall():
            cr.execute("""CREATE SEQUENCE base_registry_signaling INCREMENT BY 1 START WITH 1""")
            cr.execute("""SELECT nextval('base_registry_signaling')""")
            cr.execute("""CREATE SEQUENCE base_cache_signaling INCREMENT BY 1 START WITH 1""")
            cr.execute("""SELECT nextval('base_cache_signaling')""")

    @contextmanager
    def cursor(self, auto_commit=True):
        cr = self.db.cursor()
        try:
            yield cr
            if auto_commit:
                cr.commit()
        finally:
            cr.close()

class RegistryManager(object):
    """ Model registries manager.

        The manager is responsible for creation and deletion of model
        registries (essentially database connection/model registry pairs).

    """
    # Mapping between db name and model registry.
    # Accessed through the methods below.
    registries = {}
    registries_lock = threading.RLock()

    @classmethod
    def get(cls, db_name, force_demo=False, status=None, update_module=False):
        """ Return a registry for a given database name."""
        try:
            return cls.registries[db_name]
        except KeyError:
            return cls.new(db_name, force_demo, status,
                           update_module)

    @classmethod
    def new(cls, db_name, force_demo=False, status=None,
            update_module=False):
        """ Create and return a new registry for a given database name.

        The (possibly) previous registry for that database name is discarded.

        """
        import openerp.modules
        with cls.registries_lock:
            registry = Registry(db_name)

            # Initializing a registry will call general code which will in turn
            # call registries.get (this object) to obtain the registry being
            # initialized. Make it available in the registries dictionary then
            # remove it if an exception is raised.
            cls.delete(db_name)
            cls.registries[db_name] = registry
            try:
                # This should be a method on Registry
                openerp.modules.load_modules(registry.db, force_demo, status, update_module)
            except Exception:
                del cls.registries[db_name]
                raise

            # load_modules() above can replace the registry by calling
            # indirectly new() again (when modules have to be uninstalled).
            # Yeah, crazy.
            registry = cls.registries[db_name]

            cr = registry.db.cursor()
            try:
                Registry.setup_multi_process_signaling(cr)
                registry.do_parent_store(cr)
                cr.commit()
            finally:
                cr.close()

        registry.ready = True

        return registry

    @classmethod
    def delete(cls, db_name):
        """Delete the registry linked to a given database.  """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()
                del cls.registries[db_name]

    @classmethod
    def delete_all(cls):
        """Delete all the registries. """
        with cls.registries_lock:
            for db_name in cls.registries.keys():
                cls.delete(db_name)

    @classmethod
    def clear_caches(cls, db_name):
        """Clear caches

        This clears the caches associated to methods decorated with
        ``tools.ormcache`` or ``tools.ormcache_multi`` for all the models
        of the given database name.

        This method is given to spare you a ``RegistryManager.get(db_name)``
        that would loads the given database if it was not already loaded.
        """
        with cls.registries_lock:
            if db_name in cls.registries:
                cls.registries[db_name].clear_caches()

    @classmethod
    def check_registry_signaling(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name)
            cr = registry.db.cursor()
            try:
                cr.execute("""
                    SELECT base_registry_signaling.last_value,
                           base_cache_signaling.last_value
                    FROM base_registry_signaling, base_cache_signaling""")
                r, c = cr.fetchone()
                # Check if the model registry must be reloaded (e.g. after the
                # database has been updated by another process).
                if registry.base_registry_signaling_sequence != r:
                    _logger.info("Reloading the model registry after database signaling.")
                    registry = cls.new(db_name)
                    registry.base_registry_signaling_sequence = r
                # Check if the model caches must be invalidated (e.g. after a write
                # occured on another process). Don't clear right after a registry
                # has been reload.
                elif registry.base_cache_signaling_sequence != c:
                    _logger.info("Invalidating all model caches after database signaling.")
                    registry.base_cache_signaling_sequence = c
                    registry.clear_caches()
                    registry.reset_any_cache_cleared()
                    # One possible reason caches have been invalidated is the
                    # use of decimal_precision.write(), in which case we need
                    # to refresh fields.float columns.
                    for model in registry.models.values():
                        for column in model._columns.values():
                            if hasattr(column, 'digits_change'):
                                column.digits_change(cr)
            finally:
                cr.close()

    @classmethod
    def signal_caches_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            # Check the registries if any cache has been cleared and signal it
            # through the database to other processes.
            registry = cls.get(db_name)
            if registry.any_cache_cleared():
                _logger.info("At least one model cache has been cleared, signaling through the database.")
                cr = registry.db.cursor()
                r = 1
                try:
                    cr.execute("select nextval('base_cache_signaling')")
                    r = cr.fetchone()[0]
                finally:
                    cr.close()
                registry.base_cache_signaling_sequence = r
                registry.reset_any_cache_cleared()

    @classmethod
    def signal_registry_change(cls, db_name):
        if openerp.multi_process and db_name in cls.registries:
            registry = cls.get(db_name)
            cr = registry.db.cursor()
            r = 1
            try:
                cr.execute("select nextval('base_registry_signaling')")
                r = cr.fetchone()[0]
            finally:
                cr.close()
            registry.base_registry_signaling_sequence = r

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''An alternative of built-in `json`.

It is compatible with :py:mod:`mosql.result` and built-in `datetime`.

.. versionadded :: 0.2
    It also supports new model.

.. versionadded :: 0.1.1
'''

__all__ = ['dump', 'dumps', 'load', 'loads', 'ModelJSONEncoder']

import imp

try:
    # it imports module from built-in first, so it skipped this json.py
    json = imp.load_module('json', *imp.find_module('json'))
except ImportError:
    import simplejson as json

from datetime import datetime, date
from functools import partial

from .result import Model

class ModelJSONEncoder(json.JSONEncoder):
    '''It is compatible with :py:mod:`mosql.result` and built-in `datetime`.'''

    def default(self, obj):

        try:
            return json.JSONEncoder.default(self, obj)
        except TypeError, e:
            if isinstance(obj, (datetime, date)):
                return obj.isoformat()
            elif isinstance(obj, Model):
                return dict(obj)
            else:
                raise e

dump = partial(json.dump, cls=ModelJSONEncoder)
'''It uses the :py:class:`ModelJSONEncoder`.'''

dumps = partial(json.dumps, cls=ModelJSONEncoder)
'''It uses the :py:class:`ModelJSONEncoder`.'''

load = json.load
'''It is same as `json.load`.'''

loads = json.loads
'''It is same as `json.loads`.'''
<commit_msg>rewording<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''An alternative of built-in `json`.

It is compatible with :py:mod:`mosql.result` and built-in `datetime`.

.. versionadded :: 0.2
    It supports the new :mod:`mosql.result`.

.. versionadded :: 0.1.1
'''

__all__ = ['dump', 'dumps', 'load', 'loads', 'ModelJSONEncoder']

import imp

try:
    # it imports module from built-in first, so it skipped this json.py
    json = imp.load_module('json', *imp.find_module('json'))
except ImportError:
    import simplejson as json

from datetime import datetime, date
from functools import partial

from .result import Model

class ModelJSONEncoder(json.JSONEncoder):
    '''It is compatible with :py:mod:`mosql.result` and built-in `datetime`.'''

    def default(self, obj):

        try:
            return json.JSONEncoder.default(self, obj)
        except TypeError, e:
            if isinstance(obj, (datetime, date)):
                return obj.isoformat()
            elif isinstance(obj, Model):
                return dict(obj)
            else:
                raise e

dump = partial(json.dump, cls=ModelJSONEncoder)
'''It uses the :py:class:`ModelJSONEncoder`.'''

dumps = partial(json.dumps, cls=ModelJSONEncoder)
'''It uses the :py:class:`ModelJSONEncoder`.'''

load = json.load
'''It is same as `json.load`.'''

loads = json.loads
'''It is same as `json.loads`.'''
<|endoftext|>


<commit_before>__author__ = 'georg'

from de.hochschuletrier.jpy.Constants import Constants, Fonts
from de.hochschuletrier.jpy.overlays.Overlay import Overlay
from Tkinter import Label, StringVar

class DoubleOverlay(Overlay):
    def __init__(self, *args, **kwargs):
        Overlay.__init__(self, *args, **kwargs)
        self.label = ""
        self.money = ""
        self.worth = StringVar()
        self.config(
            padx=0,
            pady=30,
            bd=4,
            background="gold"
        )
        self.place(x=0, y=0, relwidth=1, relheight=1, width=-2000, height=-1100)

        self.renderLabel()

    def renderLabel(self):
        self.label = Label(self)
        self.label.config(
            text="Double Jeopardy",
            font=Fonts.MONEY_MEDIUM,
            background="gold"
        )
        self.label.pack()

        self.money = Label(self)
        self.money.config(
            textvar=self.worth,
            font=Fonts.MONEY_BIG,
            background="gold",
            wraplength=Constants.SCREENW
        )
        self.money.pack()<commit_msg>fix double jeopardy size<commit_after>__author__ = 'georg'

from de.hochschuletrier.jpy.Constants import Constants, Fonts
from de.hochschuletrier.jpy.overlays.Overlay import Overlay
from Tkinter import Label, StringVar

class DoubleOverlay(Overlay):
    def __init__(self, *args, **kwargs):
        Overlay.__init__(self, *args, **kwargs)
        self.label = ""
        self.money = ""
        self.worth = StringVar()
        self.config(
            padx=0,
            pady=30,
            bd=4,
            background="gold"
        )
        self.place(x=0, y=0, relwidth=1, relheight=1, width=-1200, height=-750)

        self.renderLabel()

    def renderLabel(self):
        self.label = Label(self)
        self.label.config(
            text="Double Jeopardy",
            font=Fonts.MONEY_MEDIUM,
            background="gold"
        )
        self.label.pack()

        self.money = Label(self)
        self.money.config(
            textvar=self.worth,
            font=Fonts.MONEY_BIG,
            background="gold",
            wraplength=Constants.SCREENW
        )
        self.money.pack()<|endoftext|>