<commit_before>
import datetime
from unittest import skipIf

from django.contrib.auth.models import User
from django.conf import settings
from django.test import skipIfDBFeature
from django.utils import timezone

# import tempfile
# import os
# import shutil

from fleet.workers import Manager
from kive.tests import BaseTestCases
import kive.testing_utils as tools
from metadata.models import everyone_group
from fleet.slurmlib import SlurmScheduler, DummySlurmScheduler
from sandbox.tests_rm import BadRunTestsBase

def execute_simple_run(environment, slurm_sched_class):
    """
    A helper function that creates a simple pipeline and executes a run.

    This also populates the object -- e.g. a TestCase or a FixtureBuilder --
    with some variables.

    Returns the Manager object that executed the run.
    """
    tools.create_eric_martin_test_environment(environment)
    tools.create_sandbox_testing_tools_environment(environment)

    user = User.objects.get(username='john')
    # Everything in this pipeline will be a no-op, so all can be linked together
    # without remorse.
    p_basic = tools.make_first_pipeline("P_basic", "Innermost pipeline", user)
    tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out")
    p_basic.family.grant_everyone_access()
    p_basic.grant_everyone_access()
    p_basic.create_outputs()
    p_basic.save()

    # Set up a dataset with words in it called environment.dataset_words.
    tools.make_words_dataset(environment)

    return Manager.execute_pipeline(
        environment.user_bob,
        p_basic,
        [environment.dataset_words],
        groups_allowed=[everyone_group()],
        slurm_sched_class=slurm_sched_class
    )

def execute_nested_run(environment, slurm_sched_class=DummySlurmScheduler):
    """
    A helper function that creates a nested pipeline and executes a run.

    This also populates the object -- e.g. a TestCase or a FixtureBuilder --
    with some variables.

    Returns the Manager object that executed the run.
    """
    tools.create_eric_martin_test_environment(environment)
    tools.create_sandbox_testing_tools_environment(environment)
    user = User.objects.get(username='john')

    # Everything in this pipeline will be a no-op, so all can be linked together
    # without remorse.
    p_basic = tools.make_first_pipeline("p_basic", "innermost pipeline", user)
    tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out")
    p_basic.family.grant_everyone_access()
    p_basic.grant_everyone_access()
    p_basic.create_outputs()
    p_basic.save()

    p_sub = tools.make_first_pipeline("p_sub", "second-level pipeline", user)
    tools.create_linear_pipeline(p_sub, [p_basic, p_basic], "sub_in", "sub_out")
    p_sub.family.grant_everyone_access()
    p_sub.grant_everyone_access()
    p_sub.create_outputs()
    p_sub.save()

    p_top = tools.make_first_pipeline("p_top", "top-level pipeline", user)
    tools.create_linear_pipeline(p_top, [p_sub, p_sub, p_sub], "top_in", "top_out")
    p_top.family.grant_everyone_access()
    p_top.grant_everyone_access()
    p_top.create_outputs()
    p_top.save()

    # Set up a dataset with words in it called environment.dataset_words.
    tools.make_words_dataset(environment)

    return Manager.execute_pipeline(
        environment.user_bob,
        p_top,
        [environment.dataset_words],
        groups_allowed=[everyone_group()],
        slurm_sched_class=slurm_sched_class
    )

@skipIfDBFeature('is_mocked')
class SlurmExecutionTests(BaseTestCases.SlurmExecutionTestCase):

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_simple_run(self):
        """
        Execute a simple run.
        """
        mgr = execute_simple_run(self, slurm_sched_class=SlurmScheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_nested_run(self):
        """
        Execute a nested run.
        """
        mgr = execute_nested_run(self, slurm_sched_class=SlurmScheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

# @skipIfDBFeature('is_mocked')
# class SlurmExecutionPathWithSpacesTests(SlurmExecutionTests):
#     """
#     Repeat the same tests as SlurmExecutionTests, but with spaces in the Sandbox path.
#     """
#     def setUp(self):
#         self.media_root_original = settings.MEDIA_ROOT
#         # Create this directory/probe that it exists.
#         try:
#             os.mkdir(settings.MEDIA_ROOT)
#         except OSError:
#             # It already exists.
#             pass
#
#         # Make a temporary directory whose name has spaces in it.
#         self.base_with_spaces = tempfile.mkdtemp(
#             suffix="Extra Folder With Spaces",
#             dir=self.media_root_original
#         )
#         # Just to be safe, we end MEDIA_ROOT with a directory named "Testing" as
#         # this is consistent with the way we handle other tests that install fixture files.
#         self.media_root_with_spaces = os.path.join(self.base_with_spaces, "Testing")
#         settings.MEDIA_ROOT = self.media_root_with_spaces
#         SlurmExecutionTests.setUp(self)
#
#     def tearDown(self):
#         SlurmExecutionTests.tearDown(self)
#         settings.MEDIA_ROOT = self.media_root_original
#         shutil.rmtree(self.base_with_spaces)

@skipIfDBFeature('is_mocked')
@skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
class SlurmBadRunTests(BaseTestCases.SlurmExecutionTestCase, BadRunTestsBase):
    """
    Tests a bad run using SlurmScheduler instead of DummySlurmScheduler.

    This inherits from the original test, so that we can test it both ways
    (once in the original way, and once here).
    """
    def setUp(self):
        BaseTestCases.SlurmExecutionTestCase.setUp(self)
        BadRunTestsBase.setUp(self)

    def tearDown(self):
        BaseTestCases.SlurmExecutionTestCase.tearDown(self)
        BadRunTestsBase.tearDown(self)

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_method_fails(self):
        BadRunTestsBase.test_method_fails(self, slurm_sched_class=SlurmScheduler)

class MockSlurmScheduler(DummySlurmScheduler):
    """ A mocked -up slurm scheduler which will create NODE_FAIL events
    with a job end time set to now() + my_time_delta.
    The NODE_FAIL events are injected every second time get_accounting_info() is called,
    in other times, the status information is passed through unchanged.
    """
    count = 0
    my_time_delta = datetime.timedelta(seconds=-2*settings.NODE_FAIL_TIME_OUT_SECS)
    name_tag = "PAST--"

    @classmethod
    def slurm_ident(cls):
        return "{}--{}--{}".format(cls.name_tag,
                                   cls.my_time_delta,
                                   super(MockSlurmScheduler, cls).slurm_ident())

    @classmethod
    def _mod_to_node_fail(cls, stat_dct):
        """ Modify the status dict to a node_fail state
        with an end time of now() + cls.my_time_delta
        """
        now_time = datetime.datetime.now(timezone.get_current_timezone())
        end_time = now_time + cls.my_time_delta
        for dct in stat_dct.values():
            dct[SlurmScheduler.ACC_STATE] = SlurmScheduler.NODE_FAIL
            dct[SlurmScheduler.ACC_END_TIME] = end_time

    @classmethod
    def get_accounting_info(cls, job_handle_iter=None):
        # print("mock accounting {}".format(cls.count))
        stat_dct = super(MockSlurmScheduler, cls).get_accounting_info(job_handle_iter=job_handle_iter)
        if cls.count % 2 == 0:
            # print("overriding to NODE_FAIL: {}".format(cls.my_time_delta))
            cls._mod_to_node_fail(stat_dct)
        else:
            pass
            # print("returning unchanged accounting info")
        # print("RETURNING {}".format(stat_dct))
        cls.count += 1
        return stat_dct

class Recent_NF_Scheduler(MockSlurmScheduler):
    count = 0
    my_time_delta = datetime.timedelta(seconds=0)
    name_tag = "RECENT--"

class NodeFailExecutionTests(BaseTestCases.SlurmExecutionTestCase):

    def _sched_run_simple(self, slurm_sched_class):
        # print("Running simple run with slurm : '%s'" % slurm_sched_class.slurm_ident())
        mgr = execute_simple_run(self, slurm_sched_class=slurm_sched_class)
        return mgr

    def test_NF_future_run(self):
        """
        Execute a simple run.
        NODE_FAIL is set with a recent job end_time  --> the job should complete
        as normal.
        """
        mgr = self._sched_run_simple(Recent_NF_Scheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

    def test_NF_fail_run(self):
        """
        Execute a simple run.
        NODE_FAIL is set with a job end_time in the distant past -->
        the job should fail.
        """
        mgr = self._sched_run_simple(MockSlurmScheduler)
        run = mgr.get_last_run()
        self.assertTrue(run.is_failed())
<commit_msg>The new NODE_FAIL tests must not run when DB is mocked.<commit_after>
import datetime
from unittest import skipIf

from django.contrib.auth.models import User
from django.conf import settings
from django.test import skipIfDBFeature
from django.utils import timezone

# import tempfile
# import os
# import shutil

from fleet.workers import Manager
from kive.tests import BaseTestCases
import kive.testing_utils as tools
from metadata.models import everyone_group
from fleet.slurmlib import SlurmScheduler, DummySlurmScheduler
from sandbox.tests_rm import BadRunTestsBase

def execute_simple_run(environment, slurm_sched_class):
    """
    A helper function that creates a simple pipeline and executes a run.

    This also populates the object -- e.g. a TestCase or a FixtureBuilder --
    with some variables.

    Returns the Manager object that executed the run.
    """
    tools.create_eric_martin_test_environment(environment)
    tools.create_sandbox_testing_tools_environment(environment)

    user = User.objects.get(username='john')
    # Everything in this pipeline will be a no-op, so all can be linked together
    # without remorse.
    p_basic = tools.make_first_pipeline("P_basic", "Innermost pipeline", user)
    tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out")
    p_basic.family.grant_everyone_access()
    p_basic.grant_everyone_access()
    p_basic.create_outputs()
    p_basic.save()

    # Set up a dataset with words in it called environment.dataset_words.
    tools.make_words_dataset(environment)

    return Manager.execute_pipeline(
        environment.user_bob,
        p_basic,
        [environment.dataset_words],
        groups_allowed=[everyone_group()],
        slurm_sched_class=slurm_sched_class
    )

def execute_nested_run(environment, slurm_sched_class=DummySlurmScheduler):
    """
    A helper function that creates a nested pipeline and executes a run.

    This also populates the object -- e.g. a TestCase or a FixtureBuilder --
    with some variables.

    Returns the Manager object that executed the run.
    """
    tools.create_eric_martin_test_environment(environment)
    tools.create_sandbox_testing_tools_environment(environment)
    user = User.objects.get(username='john')

    # Everything in this pipeline will be a no-op, so all can be linked together
    # without remorse.
    p_basic = tools.make_first_pipeline("p_basic", "innermost pipeline", user)
    tools.create_linear_pipeline(p_basic, [environment.method_noop, environment.method_noop], "basic_in", "basic_out")
    p_basic.family.grant_everyone_access()
    p_basic.grant_everyone_access()
    p_basic.create_outputs()
    p_basic.save()

    p_sub = tools.make_first_pipeline("p_sub", "second-level pipeline", user)
    tools.create_linear_pipeline(p_sub, [p_basic, p_basic], "sub_in", "sub_out")
    p_sub.family.grant_everyone_access()
    p_sub.grant_everyone_access()
    p_sub.create_outputs()
    p_sub.save()

    p_top = tools.make_first_pipeline("p_top", "top-level pipeline", user)
    tools.create_linear_pipeline(p_top, [p_sub, p_sub, p_sub], "top_in", "top_out")
    p_top.family.grant_everyone_access()
    p_top.grant_everyone_access()
    p_top.create_outputs()
    p_top.save()

    # Set up a dataset with words in it called environment.dataset_words.
    tools.make_words_dataset(environment)

    return Manager.execute_pipeline(
        environment.user_bob,
        p_top,
        [environment.dataset_words],
        groups_allowed=[everyone_group()],
        slurm_sched_class=slurm_sched_class
    )

@skipIfDBFeature('is_mocked')
class SlurmExecutionTests(BaseTestCases.SlurmExecutionTestCase):

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_simple_run(self):
        """
        Execute a simple run.
        """
        mgr = execute_simple_run(self, slurm_sched_class=SlurmScheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_nested_run(self):
        """
        Execute a nested run.
        """
        mgr = execute_nested_run(self, slurm_sched_class=SlurmScheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

# @skipIfDBFeature('is_mocked')
# class SlurmExecutionPathWithSpacesTests(SlurmExecutionTests):
#     """
#     Repeat the same tests as SlurmExecutionTests, but with spaces in the Sandbox path.
#     """
#     def setUp(self):
#         self.media_root_original = settings.MEDIA_ROOT
#         # Create this directory/probe that it exists.
#         try:
#             os.mkdir(settings.MEDIA_ROOT)
#         except OSError:
#             # It already exists.
#             pass
#
#         # Make a temporary directory whose name has spaces in it.
#         self.base_with_spaces = tempfile.mkdtemp(
#             suffix="Extra Folder With Spaces",
#             dir=self.media_root_original
#         )
#         # Just to be safe, we end MEDIA_ROOT with a directory named "Testing" as
#         # this is consistent with the way we handle other tests that install fixture files.
#         self.media_root_with_spaces = os.path.join(self.base_with_spaces, "Testing")
#         settings.MEDIA_ROOT = self.media_root_with_spaces
#         SlurmExecutionTests.setUp(self)
#
#     def tearDown(self):
#         SlurmExecutionTests.tearDown(self)
#         settings.MEDIA_ROOT = self.media_root_original
#         shutil.rmtree(self.base_with_spaces)

@skipIfDBFeature('is_mocked')
@skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
class SlurmBadRunTests(BaseTestCases.SlurmExecutionTestCase, BadRunTestsBase):
    """
    Tests a bad run using SlurmScheduler instead of DummySlurmScheduler.

    This inherits from the original test, so that we can test it both ways
    (once in the original way, and once here).
    """
    def setUp(self):
        BaseTestCases.SlurmExecutionTestCase.setUp(self)
        BadRunTestsBase.setUp(self)

    def tearDown(self):
        BaseTestCases.SlurmExecutionTestCase.tearDown(self)
        BadRunTestsBase.tearDown(self)

    @skipIf(not settings.RUN_SLURM_TESTS, "Slurm tests are disabled")
    def test_method_fails(self):
        BadRunTestsBase.test_method_fails(self, slurm_sched_class=SlurmScheduler)

class MockSlurmScheduler(DummySlurmScheduler):
    """ A mocked -up slurm scheduler which will create NODE_FAIL events
    with a job end time set to now() + my_time_delta.
    The NODE_FAIL events are injected every second time get_accounting_info() is called,
    in other times, the status information is passed through unchanged.
    """
    count = 0
    my_time_delta = datetime.timedelta(seconds=-2*settings.NODE_FAIL_TIME_OUT_SECS)
    name_tag = "PAST--"

    @classmethod
    def slurm_ident(cls):
        return "{}--{}--{}".format(cls.name_tag,
                                   cls.my_time_delta,
                                   super(MockSlurmScheduler, cls).slurm_ident())

    @classmethod
    def _mod_to_node_fail(cls, stat_dct):
        """ Modify the status dict to a node_fail state
        with an end time of now() + cls.my_time_delta
        """
        now_time = datetime.datetime.now(timezone.get_current_timezone())
        end_time = now_time + cls.my_time_delta
        for dct in stat_dct.values():
            dct[SlurmScheduler.ACC_STATE] = SlurmScheduler.NODE_FAIL
            dct[SlurmScheduler.ACC_END_TIME] = end_time

    @classmethod
    def get_accounting_info(cls, job_handle_iter=None):
        # print("mock accounting {}".format(cls.count))
        stat_dct = super(MockSlurmScheduler, cls).get_accounting_info(job_handle_iter=job_handle_iter)
        if cls.count % 2 == 0:
            # print("overriding to NODE_FAIL: {}".format(cls.my_time_delta))
            cls._mod_to_node_fail(stat_dct)
        else:
            pass
            # print("returning unchanged accounting info")
        # print("RETURNING {}".format(stat_dct))
        cls.count += 1
        return stat_dct

class Recent_NF_Scheduler(MockSlurmScheduler):
    count = 0
    my_time_delta = datetime.timedelta(seconds=0)
    name_tag = "RECENT--"

@skipIfDBFeature('is_mocked')
class NodeFailExecutionTests(BaseTestCases.SlurmExecutionTestCase):

    def _sched_run_simple(self, slurm_sched_class):
        # print("Running simple run with slurm : '%s'" % slurm_sched_class.slurm_ident())
        mgr = execute_simple_run(self, slurm_sched_class=slurm_sched_class)
        return mgr

    def test_NF_future_run(self):
        """
        Execute a simple run.
        NODE_FAIL is set with a recent job end_time  --> the job should complete
        as normal.
        """
        mgr = self._sched_run_simple(Recent_NF_Scheduler)
        run = mgr.get_last_run()

        self.check_run_OK(run)

        self.assertTrue(run.is_complete())
        self.assertTrue(run.is_successful())

        self.assertIsNone(run.clean())
        self.assertIsNone(run.complete_clean())

    def test_NF_fail_run(self):
        """
        Execute a simple run.
        NODE_FAIL is set with a job end_time in the distant past -->
        the job should fail.
        """
        mgr = self._sched_run_simple(MockSlurmScheduler)
        run = mgr.get_last_run()
        self.assertTrue(run.is_failed())
<|endoftext|>


<commit_before>from __future__ import print_function

import os
import sys

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

import mod_wsgi
import apache

def application(environ, start_response):
    headers = []
    headers.append(('Content-Type', 'text/plain'))
    write = start_response('200 OK', headers)

    input = environ['wsgi.input']
    output = StringIO()

    print('PID: %s' % os.getpid(), file=output)
    print('UID: %s' % os.getuid(), file=output)
    print('GID: %s' % os.getgid(), file=output)
    print(file=output)

    print('python.version: %r' % (sys.version,), file=output)
    print('apache.version: %r' % (apache.version,), file=output)
    print('mod_wsgi.version: %r' % (mod_wsgi.version,), file=output)
    print(file=output)

    print('mod_wsgi.process_group: %s' % mod_wsgi.process_group,
            file=output)
    print('mod_wsgi.application_group: %s' % mod_wsgi.application_group,
            file=output)
    print(file=output)

    print('mod_wsgi.maximum_processes: %s' % mod_wsgi.maximum_processes,
            file=output)
    print('mod_wsgi.threads_per_process: %s' % mod_wsgi.threads_per_process,
            file=output)
    print('mod_wsgi.process_metrics: %s' % mod_wsgi.process_metrics(),
            file=output)
    print('mod_wsgi.server_metrics: %s' % mod_wsgi.server_metrics(),
            file=output)
    print(file=output)

    metrics = mod_wsgi.server_metrics()

    if metrics:
        for process in metrics['processes']:
           for worker in process['workers']:
               print(worker['status'], file=output, end='')
        print(file=output)
        print(file=output)

    print('apache.description: %s' % apache.description, file=output)
    print('apache.build_date: %s' % apache.build_date, file=output)
    print('apache.mpm_name: %s' % apache.mpm_name, file=output)
    print('apache.maximum_processes: %s' % apache.maximum_processes,
            file=output)
    print('apache.threads_per_process: %s' % apache.threads_per_process,
            file=output)
    print(file=output)

    print('PATH: %s' % sys.path, file=output)
    print(file=output)

    keys = sorted(environ.keys())
    for key in keys:
        print('%s: %s' % (key, repr(environ[key])), file=output)
    print(file=output)

    keys = sorted(os.environ.keys())
    for key in keys:
        print('%s: %s' % (key, repr(os.environ[key])), file=output)
    print(file=output)

    result = output.getvalue()

    if not isinstance(result, bytes):
        result = result.encode('UTF-8')

    yield result

    yield input.read(int(environ.get('CONTENT_LENGTH', '0')))
<commit_msg>Set charset for output of environ test script.<commit_after>from __future__ import print_function

import os
import sys

try:
    from cStringIO import StringIO
except ImportError:
    from io import StringIO

import mod_wsgi
import apache

def application(environ, start_response):
    headers = []
    headers.append(('Content-Type', 'text/plain; charset="UTF-8"'))
    write = start_response('200 OK', headers)

    input = environ['wsgi.input']
    output = StringIO()

    print('PID: %s' % os.getpid(), file=output)
    print('UID: %s' % os.getuid(), file=output)
    print('GID: %s' % os.getgid(), file=output)
    print(file=output)

    print('python.version: %r' % (sys.version,), file=output)
    print('apache.version: %r' % (apache.version,), file=output)
    print('mod_wsgi.version: %r' % (mod_wsgi.version,), file=output)
    print(file=output)

    print('mod_wsgi.process_group: %s' % mod_wsgi.process_group,
            file=output)
    print('mod_wsgi.application_group: %s' % mod_wsgi.application_group,
            file=output)
    print(file=output)

    print('mod_wsgi.maximum_processes: %s' % mod_wsgi.maximum_processes,
            file=output)
    print('mod_wsgi.threads_per_process: %s' % mod_wsgi.threads_per_process,
            file=output)
    print('mod_wsgi.process_metrics: %s' % mod_wsgi.process_metrics(),
            file=output)
    print('mod_wsgi.server_metrics: %s' % mod_wsgi.server_metrics(),
            file=output)
    print(file=output)

    metrics = mod_wsgi.server_metrics()

    if metrics:
        for process in metrics['processes']:
           for worker in process['workers']:
               print(worker['status'], file=output, end='')
        print(file=output)
        print(file=output)

    print('apache.description: %s' % apache.description, file=output)
    print('apache.build_date: %s' % apache.build_date, file=output)
    print('apache.mpm_name: %s' % apache.mpm_name, file=output)
    print('apache.maximum_processes: %s' % apache.maximum_processes,
            file=output)
    print('apache.threads_per_process: %s' % apache.threads_per_process,
            file=output)
    print(file=output)

    print('PATH: %s' % sys.path, file=output)
    print(file=output)

    keys = sorted(environ.keys())
    for key in keys:
        print('%s: %s' % (key, repr(environ[key])), file=output)
    print(file=output)

    keys = sorted(os.environ.keys())
    for key in keys:
        print('%s: %s' % (key, repr(os.environ[key])), file=output)
    print(file=output)

    result = output.getvalue()

    if not isinstance(result, bytes):
        result = result.encode('UTF-8')

    yield result

    yield input.read(int(environ.get('CONTENT_LENGTH', '0')))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2012-2014 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import

import re
from urlparse import urlparse, urljoin
import mimetypes
import os
import tempfile
import sys

try:
    import requests
    if int(requests.__version__.split('.')[0]) < 2:
        raise ImportError()
except ImportError:
    raise ImportError('Please install python-requests >= 2.0')

from weboob.tools.log import getLogger

from .cookies import WeboobCookieJar

class Profile(object):
    """
    A profile represents the way Browser should act.
    Usually it is to mimic a real browser.
    """

    def setup_session(self, session):
        """
        Change default headers, set up hooks, etc.

        Warning: Do not enable lzma, bzip or bzip2, sdch encodings
        as python-requests does not support it yet.
        Supported as of 2.2: gzip, deflate, compress.
        In doubt, do not change the default Accept-Encoding header
        of python-requests.
        """
        raise NotImplementedError()

class Weboob(Profile):
    """
    It's us!
    Recommended for Weboob-friendly websites only.
    """

    def __init__(self, version):
        self.version = version

    def setup_session(self, session):
        session.headers['User-Agent'] = 'weboob/%s' % self.version

class Firefox(Profile):
    """
    Try to mimic a specific version of Firefox.
    Ideally, it should follow the current ESR Firefox:
    https://www.mozilla.org/en-US/firefox/organizations/all.html
    Do not change the Firefox version without changing the Gecko one!
    """

    def setup_session(self, session):
        """
        Set up headers for a standard Firefox request
        (except for DNT which isn't on by default but is a good idea).

        The goal is to be unidentifiable.
        """
        # Replace all base requests headers
        # https://developer.mozilla.org/en/Gecko_user_agent_string_reference
        # https://bugzilla.mozilla.org/show_bug.cgi?id=572650
        session.headers = {
            'Accept-Language': 'en-us,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0',
            'DNT': '1'}

class Wget(Profile):
    """
    Common alternative user agent.
    Some websites will give you a version with less JavaScript.
    Some others could ban you (after all, wget is not a real browser).
    """
    def __init__(self, version='1.11.4'):
        self.version = version

    def setup_session(self, session):
        # Don't remove base headers, if websites want to block fake browsers,
        # they will probably block any wget user agent anyway.
        session.headers.update({
            'Accept': '*/*',
            'User-Agent': 'Wget/%s' % self.version})

class BaseBrowser(object):
    """
    Simple browser class.
    Act like a browser, and don't try to do too much.
    """

    PROFILE = Firefox()
    TIMEOUT = 10.0
    REFRESH_MAX = 0.0

    VERIFY = True

    PROXIES = None

    MAX_RETRIES = 2

    def __init__(self, logger=None, proxy=None, responses_dirname=None):
        self.logger = getLogger('browser', logger)
        self.PROXIES = proxy
        self._setup_session(self.PROFILE)
        self.url = None
        self.response = None

        self.responses_dirname = responses_dirname
        self.responses_count = 1

    def _save(self, response, warning=False, **kwargs):
        if self.responses_dirname is None:
            self.responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
            print >>sys.stderr, 'Debug data will be saved in this directory: %s' % self.responses_dirname
        elif not os.path.isdir(self.responses_dirname):
            os.makedirs(self.responses_dirname)

        # get the content-type, remove optionnal charset part
        mimetype = response.headers.get('Content-Type', '').split(';')[0]
        # due to http://bugs.python.org/issue1043134
        if mimetype == 'text/plain':
            ext = '.txt'
        else:
            # try to get an extension (and avoid adding 'None')
            ext = mimetypes.guess_extension(mimetype, False) or ''

        path = re.sub('[^A-z0-9\.-_]+', '_', urlparse(response.url).path.rpartition('/')[2])
        if path.endswith(ext):
            ext = ''
        filename = '%02d-%d%s%s%s' % \
            (self.responses_count, response.status_code, '-' if path else '', path, ext)

        response_filepath = os.path.join(self.responses_dirname, filename)
        with open(response_filepath, 'w') as f:
            f.write(response.content)

        request = response.request
        with open(response_filepath + '-request.txt', 'w') as f:
            f.write('%s %s\n\n\n' % (request.method, request.url))
            for key, value in request.headers.iteritems():
                f.write('%s: %s\n' % (key, value))
            if request.body is not None:  # separate '' from None
                f.write('\n\n\n%s' % request.body)
        with open(response_filepath + '-response.txt', 'w') as f:
            f.write('Time: %3.3fs\n' % response.elapsed.total_seconds())
            f.write('%s %s\n\n\n' % (response.status_code, response.reason))
            for key, value in response.headers.iteritems():
                f.write('%s: %s\n' % (key, value))

        match_filepath = os.path.join(self.responses_dirname, 'url_response_match.txt')
        with open(match_filepath, 'a') as f:
            f.write('# %d %s %s\n' % (response.status_code, response.reason, response.headers.get('Content-Type', '')))
            f.write('%s\t%s\n' % (response.url, filename))
        self.responses_count += 1

        msg = u'Response saved to %s' % response_filepath
        if warning:
            self.logger.warning(msg)
        else:
            self.logger.info(msg)

    def _setup_session(self, profile):
        """
        Set up a python-requests session for our usage.
        """
        session = requests.Session()

        session.proxies = self.PROXIES

        session.verify = self.VERIFY and not self.logger.settings['ssl_insecure']

        # defines a max_retries. It's mandatory in case a server is not
        # handling keep alive correctly, like the proxy burp
        a = requests.adapters.HTTPAdapter(max_retries=self.MAX_RETRIES)
        session.mount('http://', a)
        session.mount('https://', a)

        if self.TIMEOUT:
            session.timeout = self.TIMEOUT
        ## weboob only can provide proxy and HTTP auth options
        session.trust_env = False

        profile.setup_session(session)

        if self.logger.settings['save_responses']:
            session.hooks['response'].append(self._save)

        self.session = session

        session.cookies = WeboobCookieJar()

    def location(self, url, **kwargs):
        """
        Like open() but also changes the current URL and response.
        This is the most common method to request web pages.

        Other than that, has the exact same behavior of open().
        """
        response = self.open(url, **kwargs)
        self.response = response
        self.url = self.response.url
        return response

    def open(self, url, referrer=None,
                   allow_redirects=True,
                   stream=None,
                   timeout=None,
                   verify=None,
                   cert=None,
                   proxies=None,
                   **kwargs):
        """
        Make an HTTP request like a browser does:
         * follow redirects (unless disabled)
         * provide referrers (unless disabled)

        Unless a `method` is explicitly provided, it makes a GET request,
        or a POST if data is not None,
        An empty `data` (not None, like '' or {}) *will* make a POST.

        It is a wrapper around session.request().
        All session.request() options are available.
        You should use location() or open() and not session.request(),
        since it has some interesting additions, which are easily
        individually disabled through the arguments.

        Call this instead of location() if you do not want to "visit" the URL
        (for instance, you are downloading a file).

        :param url: URL
        :type url: str

        :param data: POST data
        :type url: str or dict or None

        :param referrer: Force referrer. False to disable sending it, None for guessing
        :type referrer: str or False or None

        :rtype: :class:`requests.Response`
        """
        req = self.build_request(url, referrer, **kwargs)

        preq = self.session.prepare_request(req)
        preq._cookies = WeboobCookieJar.from_cookiejar(preq._cookies)

        if proxies is None:
            proxies = self.PROXIES

        if verify is None:
            verify = self.VERIFY and not self.logger.settings['ssl_insecure']

        if timeout is None:
            timeout = self.TIMEOUT

        # call python-requests
        response = self.session.send(preq,
                                     allow_redirects=allow_redirects,
                                     stream=stream,
                                     timeout=timeout,
                                     verify=verify,
                                     cert=cert,
                                     proxies=proxies)

        if allow_redirects:
            response = self.handle_refresh(response)

        response.raise_for_status()

        return response

    def build_request(self, url, referrer=None, **kwargs):
        """
        Does the same job as open(), but returns a Request without
        submitting it.
        This allows further customization to the Request.
        """
        if isinstance(url, requests.Request):
            req = url
            url = req.url
        else:
            req = requests.Request(url=url, **kwargs)

        # guess method
        if req.method is None:
            if req.data:
                req.method = 'POST'
            else:
                req.method = 'GET'

        if referrer is None:
            referrer = self.get_referrer(self.url, url)
        if referrer:
            # Yes, it is a misspelling.
            req.headers.setdefault('Referer', referrer)

        return req

    REFRESH_RE = re.compile("^(?P<sleep>[\d\.]+)(; url=[\"']?(?P<url>.*?)[\"']?)?$", re.IGNORECASE)

    def handle_refresh(self, response):
        """
        Called by open, to handle Refresh HTTP header.

        It only redirect to the refresh URL if the sleep time is inferior to
        REFRESH_MAX.
        """
        if not 'Refresh' in response.headers:
            return response

        m = self.REFRESH_RE.match(response.headers['Refresh'])
        if m:
            # XXX perhaps we should not redirect if the refresh url is equal to the current url.
            url = m.groupdict().get('url', None) or response.request.url
            sleep = float(m.groupdict()['sleep'])

            if sleep <= self.REFRESH_MAX:
                self.logger.debug('Refresh to %s' % url)
                return self.open(url)
            else:
                self.logger.debug('Do not refresh to %s because %s > REFRESH_MAX(%s)' % (url, sleep, self.REFRESH_MAX))
                return response

        self.logger.warning('Unable to handle refresh "%s"' % response.headers['Refresh'])

        return response

    def get_referrer(self, oldurl, newurl):
        """
        Get the referrer to send when doing a request.
        If we should not send a referrer, it will return None.

        Reference: https://en.wikipedia.org/wiki/HTTP_referer

        :param oldurl: Current absolute URL
        :type oldurl: str or None

        :param newurl: Target absolute URL
        :type newurl: str

        :rtype: str or None
        """
        if oldurl is None:
            return None
        old = urlparse(oldurl)
        new = urlparse(newurl)
        # Do not leak secure URLs to insecure URLs
        if old.scheme == 'https' and new.scheme != 'https':
            return None
        # Reloading the page. Usually no referrer.
        if oldurl == newurl:
            return None
        # TODO maybe implement some *optional* privacy features:
        # * do not leak referrer to other domains (often breaks websites)
        # * send a fake referrer (root of the current domain)
        # * never send the referrer
        # Inspired by the RefControl Firefox addon.
        return oldurl

class UrlNotAllowed(Exception):
    pass

class DomainBrowser(BaseBrowser):
    """
    A browser that handles relative URLs and can have a base URL (usually a domain).

    For instance self.location('/hello') will get http://weboob.org/hello
    if BASEURL is 'http://weboob.org/'.
    """

    BASEURL = None
    """
    Base URL, e.g. 'http://weboob.org/' or 'https://weboob.org/'
    See absurl().
    """

    """
    URLs allowed to load.
    This can be used to force SSL (if the BASEURL is SSL) or any other leakage.
    Set to True to allow only URLs starting by the BASEURL.
    Set it to a list of allowed URLs if you have multiple allowed URLs.
    More complex behavior is possible by overloading url_allowed()
    """
    RESTRICT_URL = False

    def url_allowed(self, url):
        """
        Checks if we are allowed to visit an URL.
        See RESTRICT_URL.

        :param url: Absolute URL
        :type url: str
        :rtype: bool
        """
        if self.BASEURL is None or self.RESTRICT_URL is False:
            return True
        if self.RESTRICT_URL is True:
            return url.startswith(self.BASEURL)
        for restrict_url in self.RESTRICT_URL:
            if url.startswith(restrict_url):
                return True
        return False

    def absurl(self, uri, base=None):
        """
        Get the absolute URL, relative to the base URL.
        If BASEURL is None, it will try to use the current URL.
        If base is False, it will always try to use the current URL.

        :param uri: URI to make absolute. It can be already absolute.
        :type uri: str

        :param base: Base absolute URL.
        :type base: str or None or False

        :rtype: str
        """
        if not base:
            base = self.url
        if base is None or base is True:
            base = self.BASEURL
        return urljoin(base, uri)

    def open(self, req, *args, **kwargs):
        uri = req.url if isinstance(req, requests.Request) else req

        url = self.absurl(uri)
        if not self.url_allowed(url):
            raise UrlNotAllowed(url)

        if isinstance(req, requests.Request):
            req.url = url
        else:
            req = url
        return super(DomainBrowser, self).open(req, *args, **kwargs)

    def go_home(self):
        """
        Go to the "home" page, usually the BASEURL.
        """
        return self.location(self.BASEURL or self.absurl('/'))
<commit_msg>fix compatibility with requests < 2.2<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2012-2014 Laurent Bachelier
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

from __future__ import absolute_import

import re
from urlparse import urlparse, urljoin
import mimetypes
import os
import tempfile
import sys

try:
    import requests
    if int(requests.__version__.split('.')[0]) < 2:
        raise ImportError()
except ImportError:
    raise ImportError('Please install python-requests >= 2.0')

from weboob.tools.log import getLogger

from .cookies import WeboobCookieJar

class Profile(object):
    """
    A profile represents the way Browser should act.
    Usually it is to mimic a real browser.
    """

    def setup_session(self, session):
        """
        Change default headers, set up hooks, etc.

        Warning: Do not enable lzma, bzip or bzip2, sdch encodings
        as python-requests does not support it yet.
        Supported as of 2.2: gzip, deflate, compress.
        In doubt, do not change the default Accept-Encoding header
        of python-requests.
        """
        raise NotImplementedError()

class Weboob(Profile):
    """
    It's us!
    Recommended for Weboob-friendly websites only.
    """

    def __init__(self, version):
        self.version = version

    def setup_session(self, session):
        session.headers['User-Agent'] = 'weboob/%s' % self.version

class Firefox(Profile):
    """
    Try to mimic a specific version of Firefox.
    Ideally, it should follow the current ESR Firefox:
    https://www.mozilla.org/en-US/firefox/organizations/all.html
    Do not change the Firefox version without changing the Gecko one!
    """

    def setup_session(self, session):
        """
        Set up headers for a standard Firefox request
        (except for DNT which isn't on by default but is a good idea).

        The goal is to be unidentifiable.
        """
        # Replace all base requests headers
        # https://developer.mozilla.org/en/Gecko_user_agent_string_reference
        # https://bugzilla.mozilla.org/show_bug.cgi?id=572650
        session.headers = {
            'Accept-Language': 'en-us,en;q=0.5',
            'Accept-Encoding': 'gzip, deflate',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0',
            'DNT': '1'}

class Wget(Profile):
    """
    Common alternative user agent.
    Some websites will give you a version with less JavaScript.
    Some others could ban you (after all, wget is not a real browser).
    """
    def __init__(self, version='1.11.4'):
        self.version = version

    def setup_session(self, session):
        # Don't remove base headers, if websites want to block fake browsers,
        # they will probably block any wget user agent anyway.
        session.headers.update({
            'Accept': '*/*',
            'User-Agent': 'Wget/%s' % self.version})

class BaseBrowser(object):
    """
    Simple browser class.
    Act like a browser, and don't try to do too much.
    """

    PROFILE = Firefox()
    TIMEOUT = 10.0
    REFRESH_MAX = 0.0

    VERIFY = True

    PROXIES = None

    MAX_RETRIES = 2

    def __init__(self, logger=None, proxy=None, responses_dirname=None):
        self.logger = getLogger('browser', logger)
        self.PROXIES = proxy
        self._setup_session(self.PROFILE)
        self.url = None
        self.response = None

        self.responses_dirname = responses_dirname
        self.responses_count = 1

    def _save(self, response, warning=False, **kwargs):
        if self.responses_dirname is None:
            self.responses_dirname = tempfile.mkdtemp(prefix='weboob_session_')
            print >>sys.stderr, 'Debug data will be saved in this directory: %s' % self.responses_dirname
        elif not os.path.isdir(self.responses_dirname):
            os.makedirs(self.responses_dirname)

        # get the content-type, remove optionnal charset part
        mimetype = response.headers.get('Content-Type', '').split(';')[0]
        # due to http://bugs.python.org/issue1043134
        if mimetype == 'text/plain':
            ext = '.txt'
        else:
            # try to get an extension (and avoid adding 'None')
            ext = mimetypes.guess_extension(mimetype, False) or ''

        path = re.sub('[^A-z0-9\.-_]+', '_', urlparse(response.url).path.rpartition('/')[2])
        if path.endswith(ext):
            ext = ''
        filename = '%02d-%d%s%s%s' % \
            (self.responses_count, response.status_code, '-' if path else '', path, ext)

        response_filepath = os.path.join(self.responses_dirname, filename)
        with open(response_filepath, 'w') as f:
            f.write(response.content)

        request = response.request
        with open(response_filepath + '-request.txt', 'w') as f:
            f.write('%s %s\n\n\n' % (request.method, request.url))
            for key, value in request.headers.iteritems():
                f.write('%s: %s\n' % (key, value))
            if request.body is not None:  # separate '' from None
                f.write('\n\n\n%s' % request.body)
        with open(response_filepath + '-response.txt', 'w') as f:
            f.write('Time: %3.3fs\n' % response.elapsed.total_seconds())
            f.write('%s %s\n\n\n' % (response.status_code, response.reason))
            for key, value in response.headers.iteritems():
                f.write('%s: %s\n' % (key, value))

        match_filepath = os.path.join(self.responses_dirname, 'url_response_match.txt')
        with open(match_filepath, 'a') as f:
            f.write('# %d %s %s\n' % (response.status_code, response.reason, response.headers.get('Content-Type', '')))
            f.write('%s\t%s\n' % (response.url, filename))
        self.responses_count += 1

        msg = u'Response saved to %s' % response_filepath
        if warning:
            self.logger.warning(msg)
        else:
            self.logger.info(msg)

    def _setup_session(self, profile):
        """
        Set up a python-requests session for our usage.
        """
        session = requests.Session()

        session.proxies = self.PROXIES

        session.verify = self.VERIFY and not self.logger.settings['ssl_insecure']

        # defines a max_retries. It's mandatory in case a server is not
        # handling keep alive correctly, like the proxy burp
        a = requests.adapters.HTTPAdapter(max_retries=self.MAX_RETRIES)
        session.mount('http://', a)
        session.mount('https://', a)

        if self.TIMEOUT:
            session.timeout = self.TIMEOUT
        ## weboob only can provide proxy and HTTP auth options
        session.trust_env = False

        profile.setup_session(session)

        if self.logger.settings['save_responses']:
            session.hooks['response'].append(self._save)

        self.session = session

        session.cookies = WeboobCookieJar()

    def location(self, url, **kwargs):
        """
        Like open() but also changes the current URL and response.
        This is the most common method to request web pages.

        Other than that, has the exact same behavior of open().
        """
        response = self.open(url, **kwargs)
        self.response = response
        self.url = self.response.url
        return response

    def open(self, url, referrer=None,
                   allow_redirects=True,
                   stream=None,
                   timeout=None,
                   verify=None,
                   cert=None,
                   proxies=None,
                   **kwargs):
        """
        Make an HTTP request like a browser does:
         * follow redirects (unless disabled)
         * provide referrers (unless disabled)

        Unless a `method` is explicitly provided, it makes a GET request,
        or a POST if data is not None,
        An empty `data` (not None, like '' or {}) *will* make a POST.

        It is a wrapper around session.request().
        All session.request() options are available.
        You should use location() or open() and not session.request(),
        since it has some interesting additions, which are easily
        individually disabled through the arguments.

        Call this instead of location() if you do not want to "visit" the URL
        (for instance, you are downloading a file).

        :param url: URL
        :type url: str

        :param data: POST data
        :type url: str or dict or None

        :param referrer: Force referrer. False to disable sending it, None for guessing
        :type referrer: str or False or None

        :rtype: :class:`requests.Response`
        """
        req = self.build_request(url, referrer, **kwargs)

        preq = self.session.prepare_request(req)
        if hasattr(preq, '_cookies'):
            # The _cookies attribute is not present in requests < 2.2. As in
            # previous version it doesn't calls extract_cookies_to_jar(), it is
            # not a problem as we keep our own cookiejar instance.
            preq._cookies = WeboobCookieJar.from_cookiejar(preq._cookies)

        if proxies is None:
            proxies = self.PROXIES

        if verify is None:
            verify = self.VERIFY and not self.logger.settings['ssl_insecure']

        if timeout is None:
            timeout = self.TIMEOUT

        # call python-requests
        response = self.session.send(preq,
                                     allow_redirects=allow_redirects,
                                     stream=stream,
                                     timeout=timeout,
                                     verify=verify,
                                     cert=cert,
                                     proxies=proxies)

        if allow_redirects:
            response = self.handle_refresh(response)

        response.raise_for_status()

        return response

    def build_request(self, url, referrer=None, **kwargs):
        """
        Does the same job as open(), but returns a Request without
        submitting it.
        This allows further customization to the Request.
        """
        if isinstance(url, requests.Request):
            req = url
            url = req.url
        else:
            req = requests.Request(url=url, **kwargs)

        # guess method
        if req.method is None:
            if req.data:
                req.method = 'POST'
            else:
                req.method = 'GET'

        if referrer is None:
            referrer = self.get_referrer(self.url, url)
        if referrer:
            # Yes, it is a misspelling.
            req.headers.setdefault('Referer', referrer)

        return req

    REFRESH_RE = re.compile("^(?P<sleep>[\d\.]+)(; url=[\"']?(?P<url>.*?)[\"']?)?$", re.IGNORECASE)

    def handle_refresh(self, response):
        """
        Called by open, to handle Refresh HTTP header.

        It only redirect to the refresh URL if the sleep time is inferior to
        REFRESH_MAX.
        """
        if not 'Refresh' in response.headers:
            return response

        m = self.REFRESH_RE.match(response.headers['Refresh'])
        if m:
            # XXX perhaps we should not redirect if the refresh url is equal to the current url.
            url = m.groupdict().get('url', None) or response.request.url
            sleep = float(m.groupdict()['sleep'])

            if sleep <= self.REFRESH_MAX:
                self.logger.debug('Refresh to %s' % url)
                return self.open(url)
            else:
                self.logger.debug('Do not refresh to %s because %s > REFRESH_MAX(%s)' % (url, sleep, self.REFRESH_MAX))
                return response

        self.logger.warning('Unable to handle refresh "%s"' % response.headers['Refresh'])

        return response

    def get_referrer(self, oldurl, newurl):
        """
        Get the referrer to send when doing a request.
        If we should not send a referrer, it will return None.

        Reference: https://en.wikipedia.org/wiki/HTTP_referer

        :param oldurl: Current absolute URL
        :type oldurl: str or None

        :param newurl: Target absolute URL
        :type newurl: str

        :rtype: str or None
        """
        if oldurl is None:
            return None
        old = urlparse(oldurl)
        new = urlparse(newurl)
        # Do not leak secure URLs to insecure URLs
        if old.scheme == 'https' and new.scheme != 'https':
            return None
        # Reloading the page. Usually no referrer.
        if oldurl == newurl:
            return None
        # TODO maybe implement some *optional* privacy features:
        # * do not leak referrer to other domains (often breaks websites)
        # * send a fake referrer (root of the current domain)
        # * never send the referrer
        # Inspired by the RefControl Firefox addon.
        return oldurl

class UrlNotAllowed(Exception):
    pass

class DomainBrowser(BaseBrowser):
    """
    A browser that handles relative URLs and can have a base URL (usually a domain).

    For instance self.location('/hello') will get http://weboob.org/hello
    if BASEURL is 'http://weboob.org/'.
    """

    BASEURL = None
    """
    Base URL, e.g. 'http://weboob.org/' or 'https://weboob.org/'
    See absurl().
    """

    """
    URLs allowed to load.
    This can be used to force SSL (if the BASEURL is SSL) or any other leakage.
    Set to True to allow only URLs starting by the BASEURL.
    Set it to a list of allowed URLs if you have multiple allowed URLs.
    More complex behavior is possible by overloading url_allowed()
    """
    RESTRICT_URL = False

    def url_allowed(self, url):
        """
        Checks if we are allowed to visit an URL.
        See RESTRICT_URL.

        :param url: Absolute URL
        :type url: str
        :rtype: bool
        """
        if self.BASEURL is None or self.RESTRICT_URL is False:
            return True
        if self.RESTRICT_URL is True:
            return url.startswith(self.BASEURL)
        for restrict_url in self.RESTRICT_URL:
            if url.startswith(restrict_url):
                return True
        return False

    def absurl(self, uri, base=None):
        """
        Get the absolute URL, relative to the base URL.
        If BASEURL is None, it will try to use the current URL.
        If base is False, it will always try to use the current URL.

        :param uri: URI to make absolute. It can be already absolute.
        :type uri: str

        :param base: Base absolute URL.
        :type base: str or None or False

        :rtype: str
        """
        if not base:
            base = self.url
        if base is None or base is True:
            base = self.BASEURL
        return urljoin(base, uri)

    def open(self, req, *args, **kwargs):
        uri = req.url if isinstance(req, requests.Request) else req

        url = self.absurl(uri)
        if not self.url_allowed(url):
            raise UrlNotAllowed(url)

        if isinstance(req, requests.Request):
            req.url = url
        else:
            req = url
        return super(DomainBrowser, self).open(req, *args, **kwargs)

    def go_home(self):
        """
        Go to the "home" page, usually the BASEURL.
        """
        return self.location(self.BASEURL or self.absurl('/'))
<|endoftext|>


<commit_before>"""
Uploads documents to Axilent.
"""
import markdown
from dox.config import get_cfg, get_keymap, get_keyfields
from dox.client import get_content_library_resource, get_library_client

def _tagging(md,content_key):
    """
    Apply tagging
    """
    tagger = get_library_client()

    # tagging
    tags = md.Meta['tags']
    for tag in tags:
        if tag:
            tagger.tagcontent(project='Axilent Docs',
                              content_type='Article',
                              content_key=content_key,
                              tag=tag)

def upload_document(path,key=None):
    """
    Uploads the specified document, returns the key of the uploaded document.
    """
    # 1. Prepare data for upload
    cfg = get_cfg()
    body_field_name = cfg.get('Connection','body_field')
    data = {}
    md = markdown.Markdown(extensions=['meta','tables'])
    with open(path) as docfile:
        body = md.convert(docfile.read())

        for field_name, field_value in md.Meta.items():
            data[field_name] = field_value[0]

        data[body_field_name] = body

    # Upload Content
    return_key = None
    resource = get_content_library_resource()
    if key:
        resource.put(data={'content':data,
                           'content_type':cfg.get('Connection','content_type'),
                           'project':cfg.get('Connection','project'),
                           'key':key})

        _tagging(md,key)

        return (key,False) # no new document created
    else:
        response = resource.post(data={'content':data,
                                       'project':cfg.get('Connection','project'),
                                       'content_type':cfg.get('Connection','content_type')})
        created_content_type, created_key = response['created_content'].split(':')

        _tagging(md,created_key)

        return (created_key,True) # new document created

def find_key(path,keymap,keyfields):
    """
    Finds a key for the specified document if it exists.
    """
    # First try to pull key from the keymap
    try:
        return keymap[path]
    except KeyError:
        pass

    # Try to find it from a keyfield
    try:
        cfg = get_cfg()
        keyfield_name = cfg.get('Connection','key_field')
        md = markdown.Markdown(extensions=['meta','tables'])
        with open(path) as docfile:
            md.convert(docfile.read())
            keyfield = md.Meta[keyfield_name][0]
            return keyfields[keyfield]
    except KeyError:
        pass

    # Give up
    return None

def extract_keyfield(path):
    """
    Extracts the keyfield value from the document.
    """
    cfg = get_cfg()
    keyfield_name = cfg.get('Connection','key_field')
    md = markdown.Markdown(extensions=['meta','tables'])
    with open(path) as docfile:
        md.convert(docfile.read())
        try:
            keyfield = md.Meta[keyfield_name][0]
            return keyfield
        except KeyError:
            return None
<commit_msg>Removing hard coded content type.<commit_after>"""
Uploads documents to Axilent.
"""
import markdown
from dox.config import get_cfg, get_keymap, get_keyfields
from dox.client import get_content_library_resource, get_library_client

def _tagging(md,content_key,content_type):
    """
    Apply tagging
    """
    tagger = get_library_client()

    # tagging
    tags = md.Meta['tags']
    for tag in tags:
        if tag:
            tagger.tagcontent(project='Axilent Docs',
                              content_type=content_type,
                              content_key=content_key,
                              tag=tag)

def upload_document(path,key=None):
    """
    Uploads the specified document, returns the key of the uploaded document.
    """
    # 1. Prepare data for upload
    cfg = get_cfg()
    body_field_name = cfg.get('Connection','body_field')
    data = {}
    md = markdown.Markdown(extensions=['meta','tables'])
    with open(path) as docfile:
        body = md.convert(docfile.read())

        for field_name, field_value in md.Meta.items():
            data[field_name] = field_value[0]

        data[body_field_name] = body

    # Upload Content
    return_key = None
    resource = get_content_library_resource()
    if key:
        resource.put(data={'content':data,
                           'content_type':cfg.get('Connection','content_type'),
                           'project':cfg.get('Connection','project'),
                           'key':key})

        _tagging(md,key,cfg.get('Connection','content_type'))

        return (key,False) # no new document created
    else:
        response = resource.post(data={'content':data,
                                       'project':cfg.get('Connection','project'),
                                       'content_type':cfg.get('Connection','content_type')})
        created_content_type, created_key = response['created_content'].split(':')

        _tagging(md,created_key,cfg.get('Connection','content_type'))

        return (created_key,True) # new document created

def find_key(path,keymap,keyfields):
    """
    Finds a key for the specified document if it exists.
    """
    # First try to pull key from the keymap
    try:
        return keymap[path]
    except KeyError:
        pass

    # Try to find it from a keyfield
    try:
        cfg = get_cfg()
        keyfield_name = cfg.get('Connection','key_field')
        md = markdown.Markdown(extensions=['meta','tables'])
        with open(path) as docfile:
            md.convert(docfile.read())
            keyfield = md.Meta[keyfield_name][0]
            return keyfields[keyfield]
    except KeyError:
        pass

    # Give up
    return None

def extract_keyfield(path):
    """
    Extracts the keyfield value from the document.
    """
    cfg = get_cfg()
    keyfield_name = cfg.get('Connection','key_field')
    md = markdown.Markdown(extensions=['meta','tables'])
    with open(path) as docfile:
        md.convert(docfile.read())
        try:
            keyfield = md.Meta[keyfield_name][0]
            return keyfield
        except KeyError:
            return None
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# :noTabs=true:

# (c) Copyright Rosetta Commons Member Institutions.
# (c) This file is part of the Rosetta software suite and is made available under license.
# (c) The Rosetta software is developed by the contributing members of the Rosetta Commons.
# (c) For more information, see http://www.rosettacommons.org. Questions about this can be
# (c) addressed to University of Washington CoMotion, email: license@uw.edu.

## @file   build.py
## @brief  Script to build Binder
## @author Sergey Lyskov

from __future__ import print_function

import os, sys, argparse, platform, subprocess, imp, shutil, distutils.dir_util

from collections import OrderedDict

if sys.platform.startswith("linux"): Platform = "linux" # can be linux1, linux2, etc
elif sys.platform == "darwin" : Platform = "macos"
elif sys.platform == "cygwin" : Platform = "cygwin"
elif sys.platform == "win32" : Platform = "windows"
else: Platform = "unknown"
PlatformBits = platform.architecture()[0][:2]

_machine_name_ = os.uname()[1]

_python_version_ = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)  # should be formatted: 2.7, 3.5, 3.6, ...

_pybind11_version_ = '35045eeef8969b7b446c64b192502ac1cbf7c451'

def execute(message, command_line, return_='status', until_successes=False, terminate_on_failure=True, silent=False):
    print(message);  print(command_line)
    while True:

        p = subprocess.Popen(command_line, bufsize=0, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, errors = p.communicate()

        output = output + errors

        if sys.version_info[0] == 2: output = output.decode('utf-8', errors='replace').encode('utf-8', 'replace') # Python-2
        else: output = output.decode('utf-8', errors='replace')  # Python-3

        exit_code = p.returncode

        if exit_code  or  not silent: print(output)

        if exit_code and until_successes: pass  # Thats right - redability COUNT!
        else: break

        print( "Error while executing {}: {}\n".format(message, output) )
        print("Sleeping 60s... then I will retry...")
        time.sleep(60)

    if return_ == 'tuple': return(exit_code, output)

    if exit_code and terminate_on_failure:
        print("\nEncounter error while executing: " + command_line)
        if return_==True: return True
        else: print("\nEncounter error while executing: " + command_line + '\n' + output); sys.exit(1)

    if return_ == 'output': return output
    else: return False

def update_source_file(file_name, data):
    ''' write data to a file but only if file does not exist or it content different from data '''
    if( not os.path.isfile(file_name)  or  open(file_name).read() != data ):
        print('Writing '+ file_name)
        with open(file_name, 'w') as f: f.write(data)

def get_compiler_family():
    ''' Try to guess compiler family using Options.compiler '''
    if 'clang' in Options.compiler: return 'clang'
    if 'gcc'   in Options.compiler: return 'gcc'
    if 'cl'    in Options.compiler: return 'cl'
    return 'unknown'

def install_llvm_tool(name, source_location, prefix, debug, jobs=1, clean=True, gcc_install_prefix=None):
    ''' Install and update (if needed) custom LLVM tool at given prefix (from config).
        Return absolute path to executable on success and terminate with error on failure
    '''
    if not os.path.isdir(prefix): os.makedirs(prefix)

    llvm_version='4.0.0'
    prefix += '/llvm-' + llvm_version
    clang_path = "{prefix}/tools/clang".format(**locals())

    if not os.path.isfile(prefix + '/CMakeLists.txt'): execute('Download llvm source.', 'curl http://releases.llvm.org/{llvm_version}/llvm-{llvm_version}.src.tar.xz | tar -Jxo && mv llvm-{llvm_version}.src {prefix}'.format(llvm_version=llvm_version, prefix=prefix) )

    if not os.path.isdir(clang_path): execute('Download clang source.', 'curl http://releases.llvm.org/{llvm_version}/cfe-{llvm_version}.src.tar.xz | tar -Jxo && mv cfe-{llvm_version}.src {clang_path}'.format(llvm_version=llvm_version, clang_path=clang_path) )

    if not os.path.isdir(prefix+'/tools/clang/tools/extra'): os.makedirs(prefix+'/tools/clang/tools/extra')

    tool_link_path = '{prefix}/tools/clang/tools/extra/{name}'.format(prefix=prefix, name=name)
    if os.path.islink(tool_link_path): os.unlink(tool_link_path)
    os.symlink(source_location, tool_link_path)

    cmake_lists = prefix + '/tools/clang/tools/extra/CMakeLists.txt'
    tool_build_line = 'add_subdirectory({})'.format(name)

    if not os.path.isfile(cmake_lists):
        with open(cmake_lists, 'w') as f: f.write(tool_build_line + '\n')

    build_dir = prefix+'/build_' + llvm_version + '.' + Platform + '.' +_machine_name_ + ('.debug' if debug else '.release')
    if not os.path.isdir(build_dir): os.makedirs(build_dir)
    execute(
        'Building tool: {}...'.format(name),
        'cd {build_dir} && cmake -G Ninja -DCMAKE_BUILD_TYPE={build_type} -DLLVM_ENABLE_EH=1 -DLLVM_ENABLE_RTTI=ON {gcc_install_prefix} .. && ninja bin/binder clang {jobs}'.format( # we need to build Clang so lib/clang/<version>/include is also built
            build_dir=build_dir,
            jobs="-j{}".format(jobs) if jobs else "",
            build_type='Debug' if debug else 'Release',
            gcc_install_prefix='-DGCC_INSTALL_PREFIX='+gcc_install_prefix if gcc_install_prefix else ''),
        silent=True)
    print()

    executable = build_dir + '/bin/' + name
    if not os.path.isfile(executable): print("\nEncounter error while running install_llvm_tool: Build is complete but executable {} is not there!!!".format(executable) ); sys.exit(1)

    return executable

def install_pybind11(prefix, clean=True):
    ''' Download and install PyBind11 library at given prefix. Install version specified by _pybind11_version_ sha1
    '''
    #git_checkout = '( git fetch && git checkout {0} && git reset --hard {0} && git pull )'.format(_pybind11_version_) if clean else 'git checkout {}'.format(_pybind11_version_)
    git_checkout = '( git fetch && git reset --hard {0} )'.format(_pybind11_version_) if clean else 'git checkout {}'.format(_pybind11_version_)

    if not os.path.isdir(prefix): os.makedirs(prefix)
    package_dir = prefix + '/pybind11'

    if not os.path.isdir(package_dir): execute('Clonning pybind11...', 'cd {} && git clone https://github.com/RosettaCommons/pybind11.git'.format(prefix) )
    execute('Checking out PyBind11 revision: {}...'.format(_pybind11_version_), 'cd {package_dir} && ( {git_checkout} )'.format(package_dir=package_dir, git_checkout=git_checkout), silent=True)
    print()

    include = package_dir + '/include/pybind11/pybind11.h'
    if not os.path.isfile(include): print("\nEncounter error while running install_pybind11: Install is complete but include file {} is not there!!!".format(include) ); sys.exit(1)

    return package_dir + '/include'

def main(args):
    ''' Binding demo build/test script '''

    parser = argparse.ArgumentParser()

    parser.add_argument('-j', '--jobs', default=1, const=0, nargs="?", type=int, help="Number of processors to use on when building, use '-j' with no arguments to launch job-per-core. (default: 1) ")
    parser.add_argument("--type", default='Release', choices=['Release', 'Debug', 'MinSizeRel', 'RelWithDebInfo'], help="Specify build type")
    parser.add_argument('--compiler', default='clang', help='Compiler to use, defualt is clang')
    parser.add_argument('--binder', default='', help='Path to Binder tool. If none is given then download, build and install binder into main/source/build. Use "--binder-debug" to control which mode of binder (debug/release) is used.')
    parser.add_argument("--binder-debug", action="store_true", help="Run binder tool in debug mode (only relevant if no '--binder' option was specified)")
    parser.add_argument('--pybind11', default='', help='Path to pybind11 source tree')
    parser.add_argument('--annotate-includes', action="store_true", help='Annotate includes in generated source files')
    parser.add_argument('--trace', action="store_true", help='Binder will add trace output to to generated source files')

    global Options
    Options = parser.parse_args()

    source_path = os.path.abspath('.')

    if not Options.binder: Options.binder = install_llvm_tool('binder', source_path+'/source', source_path + '/build', Options.binder_debug, jobs=Options.jobs)

    if not Options.pybind11: Options.pybind11 = install_pybind11(source_path + '/build')

    print( 'Binder binaries now ready at: {}\nPybind11 clone is ready at: {}'.format(Options.binder, Options.pybind11) )

if __name__ == "__main__":
    main(sys.argv)
<commit_msg>updating `llvm_version`<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# :noTabs=true:

# (c) Copyright Rosetta Commons Member Institutions.
# (c) This file is part of the Rosetta software suite and is made available under license.
# (c) The Rosetta software is developed by the contributing members of the Rosetta Commons.
# (c) For more information, see http://www.rosettacommons.org. Questions about this can be
# (c) addressed to University of Washington CoMotion, email: license@uw.edu.

## @file   build.py
## @brief  Script to build Binder
## @author Sergey Lyskov

from __future__ import print_function

import os, sys, argparse, platform, subprocess, imp, shutil, distutils.dir_util

from collections import OrderedDict

if sys.platform.startswith("linux"): Platform = "linux" # can be linux1, linux2, etc
elif sys.platform == "darwin" : Platform = "macos"
elif sys.platform == "cygwin" : Platform = "cygwin"
elif sys.platform == "win32" : Platform = "windows"
else: Platform = "unknown"
PlatformBits = platform.architecture()[0][:2]

_machine_name_ = os.uname()[1]

_python_version_ = '{}.{}'.format(sys.version_info.major, sys.version_info.minor)  # should be formatted: 2.7, 3.5, 3.6, ...

_pybind11_version_ = '35045eeef8969b7b446c64b192502ac1cbf7c451'

def execute(message, command_line, return_='status', until_successes=False, terminate_on_failure=True, silent=False):
    print(message);  print(command_line)
    while True:

        p = subprocess.Popen(command_line, bufsize=0, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        output, errors = p.communicate()

        output = output + errors

        if sys.version_info[0] == 2: output = output.decode('utf-8', errors='replace').encode('utf-8', 'replace') # Python-2
        else: output = output.decode('utf-8', errors='replace')  # Python-3

        exit_code = p.returncode

        if exit_code  or  not silent: print(output)

        if exit_code and until_successes: pass  # Thats right - redability COUNT!
        else: break

        print( "Error while executing {}: {}\n".format(message, output) )
        print("Sleeping 60s... then I will retry...")
        time.sleep(60)

    if return_ == 'tuple': return(exit_code, output)

    if exit_code and terminate_on_failure:
        print("\nEncounter error while executing: " + command_line)
        if return_==True: return True
        else: print("\nEncounter error while executing: " + command_line + '\n' + output); sys.exit(1)

    if return_ == 'output': return output
    else: return False

def update_source_file(file_name, data):
    ''' write data to a file but only if file does not exist or it content different from data '''
    if( not os.path.isfile(file_name)  or  open(file_name).read() != data ):
        print('Writing '+ file_name)
        with open(file_name, 'w') as f: f.write(data)

def get_compiler_family():
    ''' Try to guess compiler family using Options.compiler '''
    if 'clang' in Options.compiler: return 'clang'
    if 'gcc'   in Options.compiler: return 'gcc'
    if 'cl'    in Options.compiler: return 'cl'
    return 'unknown'

def install_llvm_tool(name, source_location, prefix, debug, jobs=1, clean=True, gcc_install_prefix=None):
    ''' Install and update (if needed) custom LLVM tool at given prefix (from config).
        Return absolute path to executable on success and terminate with error on failure
    '''
    if not os.path.isdir(prefix): os.makedirs(prefix)

    llvm_version='6.0.1'
    prefix += '/llvm-' + llvm_version
    clang_path = "{prefix}/tools/clang".format(**locals())

    if not os.path.isfile(prefix + '/CMakeLists.txt'): execute('Download llvm source.', 'curl http://releases.llvm.org/{llvm_version}/llvm-{llvm_version}.src.tar.xz | tar -Jxo && mv llvm-{llvm_version}.src {prefix}'.format(llvm_version=llvm_version, prefix=prefix) )

    if not os.path.isdir(clang_path): execute('Download clang source.', 'curl http://releases.llvm.org/{llvm_version}/cfe-{llvm_version}.src.tar.xz | tar -Jxo && mv cfe-{llvm_version}.src {clang_path}'.format(llvm_version=llvm_version, clang_path=clang_path) )

    if not os.path.isdir(prefix+'/tools/clang/tools/extra'): os.makedirs(prefix+'/tools/clang/tools/extra')

    tool_link_path = '{prefix}/tools/clang/tools/extra/{name}'.format(prefix=prefix, name=name)
    if os.path.islink(tool_link_path): os.unlink(tool_link_path)
    os.symlink(source_location, tool_link_path)

    cmake_lists = prefix + '/tools/clang/tools/extra/CMakeLists.txt'
    tool_build_line = 'add_subdirectory({})'.format(name)

    if not os.path.isfile(cmake_lists):
        with open(cmake_lists, 'w') as f: f.write(tool_build_line + '\n')

    build_dir = prefix+'/build_' + llvm_version + '.' + Platform + '.' +_machine_name_ + ('.debug' if debug else '.release')
    if not os.path.isdir(build_dir): os.makedirs(build_dir)
    execute(
        'Building tool: {}...'.format(name),
        'cd {build_dir} && cmake -G Ninja -DCMAKE_BUILD_TYPE={build_type} -DLLVM_ENABLE_EH=1 -DLLVM_ENABLE_RTTI=ON {gcc_install_prefix} .. && ninja bin/binder clang {jobs}'.format( # we need to build Clang so lib/clang/<version>/include is also built
            build_dir=build_dir,
            jobs="-j{}".format(jobs) if jobs else "",
            build_type='Debug' if debug else 'Release',
            gcc_install_prefix='-DGCC_INSTALL_PREFIX='+gcc_install_prefix if gcc_install_prefix else ''),
        silent=True)
    print()

    executable = build_dir + '/bin/' + name
    if not os.path.isfile(executable): print("\nEncounter error while running install_llvm_tool: Build is complete but executable {} is not there!!!".format(executable) ); sys.exit(1)

    return executable

def install_pybind11(prefix, clean=True):
    ''' Download and install PyBind11 library at given prefix. Install version specified by _pybind11_version_ sha1
    '''
    #git_checkout = '( git fetch && git checkout {0} && git reset --hard {0} && git pull )'.format(_pybind11_version_) if clean else 'git checkout {}'.format(_pybind11_version_)
    git_checkout = '( git fetch && git reset --hard {0} )'.format(_pybind11_version_) if clean else 'git checkout {}'.format(_pybind11_version_)

    if not os.path.isdir(prefix): os.makedirs(prefix)
    package_dir = prefix + '/pybind11'

    if not os.path.isdir(package_dir): execute('Clonning pybind11...', 'cd {} && git clone https://github.com/RosettaCommons/pybind11.git'.format(prefix) )
    execute('Checking out PyBind11 revision: {}...'.format(_pybind11_version_), 'cd {package_dir} && ( {git_checkout} )'.format(package_dir=package_dir, git_checkout=git_checkout), silent=True)
    print()

    include = package_dir + '/include/pybind11/pybind11.h'
    if not os.path.isfile(include): print("\nEncounter error while running install_pybind11: Install is complete but include file {} is not there!!!".format(include) ); sys.exit(1)

    return package_dir + '/include'

def main(args):
    ''' Binding demo build/test script '''

    parser = argparse.ArgumentParser()

    parser.add_argument('-j', '--jobs', default=1, const=0, nargs="?", type=int, help="Number of processors to use on when building, use '-j' with no arguments to launch job-per-core. (default: 1) ")
    parser.add_argument("--type", default='Release', choices=['Release', 'Debug', 'MinSizeRel', 'RelWithDebInfo'], help="Specify build type")
    parser.add_argument('--compiler', default='clang', help='Compiler to use, defualt is clang')
    parser.add_argument('--binder', default='', help='Path to Binder tool. If none is given then download, build and install binder into main/source/build. Use "--binder-debug" to control which mode of binder (debug/release) is used.')
    parser.add_argument("--binder-debug", action="store_true", help="Run binder tool in debug mode (only relevant if no '--binder' option was specified)")
    parser.add_argument('--pybind11', default='', help='Path to pybind11 source tree')
    parser.add_argument('--annotate-includes', action="store_true", help='Annotate includes in generated source files')
    parser.add_argument('--trace', action="store_true", help='Binder will add trace output to to generated source files')

    global Options
    Options = parser.parse_args()

    source_path = os.path.abspath('.')

    if not Options.binder: Options.binder = install_llvm_tool('binder', source_path+'/source', source_path + '/build', Options.binder_debug, jobs=Options.jobs)

    if not Options.pybind11: Options.pybind11 = install_pybind11(source_path + '/build')

    print( 'Binder binaries now ready at: {}\nPybind11 clone is ready at: {}'.format(Options.binder, Options.pybind11) )

if __name__ == "__main__":
    main(sys.argv)
<|endoftext|>


<commit_before>from exceptions import *
import boto.ec2
import logging
import sys

class Server(object):

    log = logging.getLogger('Servers.Server')
    log.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
            '%(asctime)s [%(name)s] %(levelname)s: %(message)s',
            datefmt='%H:%M:%S')
    ch.setFormatter(formatter)
    log.addHandler(ch)

    def __init__(self, dry=None, verbose=None, size=None, cluster=None,
                    environment=None, ami=None, region=None, role=None,
                    keypair=None, availability_zone=None, security_groups=None):

        self.dry = dry
        self.verbose = verbose
        self.size = size
        self.cluster = cluster
        self.environment = environment
        self.ami = ami
        self.region = region
        self.role = role
        self.keypair = keypair
        self.availability_zone = availability_zone
        self.security_groups = security_groups

    def configure(self):

        if self.dry is None:
            self.dry = False

        self.log.info('Using dry value \'{dry}\''.format(dry = self.dry))

        if self.verbose is None:
            self.verbose = False

        self.log.info('Using verbose value \'{verbose}\''.format(
                        verbose = self.verbose))

        if self.size is None:
            self.size = 'm3.medium'

        self.log.info('Using Instance Type \'{size}\''.format(size = self.size))

        if self.cluster is None:
            raise InvalidCluster('A cluster must be specified.')

        self.log.info('Using Cluster \'{cluster}\''.format(
                        cluster = self.cluster))

        if self.environment is None:
            self.environment = 'test'

        self.log.info('Using Environment \'{environment}\''.format(
                        environment = self.environment))

        if self.region is None:
            self.region = 'us-east-1'

        valid = lambda r: r in [region.name for region in boto.ec2.regions()]

        if not valid(self.region):

            error = '\'{region}\' is not a valid EC2 region'.format(
                        region=self.region)
            raise RegionDoesNotExist(error)

        self.log.info('Using EC2 Region \'{region}\''.format(
                        region = self.region))

        self.establish_ec2_connection()

        if self.ami is None:
            self.ami = 'ami-146e2a7c'

        try:
            self.ec2.get_all_images(image_ids=[self.ami])
        except Exception, e:
            if 'Invalid id' in str(e):
                error = '\'{ami}\' is not a valid AMI'.format(ami = self.ami)
                raise InvalidAMI(error)

        self.log.info('Using EC2 AMI \'{ami}\''.format(ami = self.ami))

        if self.role is None:
            self.role = self.environment[0] + '-' + self.cluster

        self.log.info('Using IAM Role \'{role}\''.format(role = self.role))

        if self.keypair is None:
            self.keypair = 'stage-key'
            if self.environment == 'prod':
                self.keypair = 'bkaiserkey'

        valid = lambda k: k in [pair.name for pair in
                self.ec2.get_all_key_pairs()]

        if not valid(self.keypair):
            error = '\'{keypair}\' is not a valid EC2 keypair'.format(
                        keypair = self.keypair)
            raise InvalidKeyPair(error)

        self.log.info('Using EC2 Key Pair \'{keypair}\''.format(
                        keypair = self.keypair))

        if self.availability_zone is None:
            self.availability_zone = 'c'

        if len(self.availability_zone) == 1:
            self.availability_zone = self.region+self.availability_zone

        valid = lambda z: z in [zone.name for zone in self.ec2.get_all_zones()]

        if not valid(self.availability_zone):
            error = '\'{zone}\' is not a valid EC2 availability zone'.format(
                    zone = self.availability_zone)
            raise InvalidAvailabilityZone(error)

        self.log.info('Using EC2 Availability Zone \'{zone}\''.format(
                        zone = self.availability_zone))

        if self.security_groups is None:
            self.security_groups = ['management', 'chef-nodes']

        self.log.info('Using security groups {groups}'.format(
                        groups=', '.join(self.security_groups)))

    def establish_ec2_connection(self):

        self.log.info('Using EC2 Region \'{region}\''.format(
                        region=self.region))
        self.log.info("Attempting to connect to EC2")

        try:
            self.ec2 = boto.ec2.connect_to_region(self.region)

            if self.verbose:
                self.log.info('Established connection to EC2')
        except Exception, e:
            raise e

    def autorun(self):

        self.configure()
<commit_msg>Added warning messages when parameters aren't defined<commit_after>from exceptions import *
import boto.ec2
import logging
import sys

class Server(object):

    log = logging.getLogger('Servers.Server')
    log.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
            '%(asctime)s [%(name)s] %(levelname)s: %(message)s',
            datefmt='%H:%M:%S')
    ch.setFormatter(formatter)
    log.addHandler(ch)

    def __init__(self, dry=None, verbose=None, size=None, cluster=None,
                    environment=None, ami=None, region=None, role=None,
                    keypair=None, availability_zone=None, security_groups=None):

        self.dry = dry
        self.verbose = verbose
        self.size = size
        self.cluster = cluster
        self.environment = environment
        self.ami = ami
        self.region = region
        self.role = role
        self.keypair = keypair
        self.availability_zone = availability_zone
        self.security_groups = security_groups

    def configure(self):

        if self.dry is None:
            self.dry = False

        self.log.info('Using dry value \'{dry}\''.format(dry = self.dry))

        if self.verbose is None:
            self.verbose = False

        self.log.info('Using verbose value \'{verbose}\''.format(
                        verbose = self.verbose))

        if self.size is None:
            self.log.warn('No Instance Type provided')
            self.size = 'm3.medium'

        self.log.info('Using Instance Type \'{size}\''.format(size = self.size))

        if self.cluster is None:
            self.log.warn('No cluster provided')
            raise InvalidCluster('A cluster must be specified.')

        self.log.info('Using Cluster \'{cluster}\''.format(
                        cluster = self.cluster))

        if self.environment is None:
            self.log.warn('No environment provided')
            self.environment = 'test'

        self.log.info('Using Environment \'{environment}\''.format(
                        environment = self.environment))

        if self.region is None:
            self.log.warn('No region provided')
            self.region = 'us-east-1'

        valid = lambda r: r in [region.name for region in boto.ec2.regions()]

        if not valid(self.region):

            error = '\'{region}\' is not a valid EC2 region'.format(
                        region=self.region)
            raise RegionDoesNotExist(error)

        self.log.info('Using EC2 Region \'{region}\''.format(
                        region = self.region))

        self.establish_ec2_connection()

        if self.ami is None:
            self.log.warn('No AMI provided')
            self.ami = 'ami-146e2a7c'

        try:
            self.ec2.get_all_images(image_ids=[self.ami])
        except Exception, e:
            if 'Invalid id' in str(e):
                error = '\'{ami}\' is not a valid AMI'.format(ami = self.ami)
                raise InvalidAMI(error)

        self.log.info('Using EC2 AMI \'{ami}\''.format(ami = self.ami))

        if self.role is None:
            self.log.warn('No IAM Role provided')
            self.role = self.environment[0] + '-' + self.cluster

        self.log.info('Using IAM Role \'{role}\''.format(role = self.role))

        if self.keypair is None:
            self.log.warn('No EC2 Keypair provided')
            self.keypair = 'stage-key'
            if self.environment == 'prod':
                self.keypair = 'bkaiserkey'

        valid = lambda k: k in [pair.name for pair in
                self.ec2.get_all_key_pairs()]

        if not valid(self.keypair):
            error = '\'{keypair}\' is not a valid EC2 keypair'.format(
                        keypair = self.keypair)
            raise InvalidKeyPair(error)

        self.log.info('Using EC2 Key Pair \'{keypair}\''.format(
                        keypair = self.keypair))

        if self.availability_zone is None:
            self.log.warn('No EC2 availability zone provided')
            self.availability_zone = 'c'

        if len(self.availability_zone) == 1:
            self.availability_zone = self.region+self.availability_zone

        valid = lambda z: z in [zone.name for zone in self.ec2.get_all_zones()]

        if not valid(self.availability_zone):
            error = '\'{zone}\' is not a valid EC2 availability zone'.format(
                    zone = self.availability_zone)
            raise InvalidAvailabilityZone(error)

        self.log.info('Using EC2 Availability Zone \'{zone}\''.format(
                        zone = self.availability_zone))

        if self.security_groups is None:
            self.log.warn('No EC2 security groups provided')
            self.security_groups = ['management', 'chef-nodes']

        self.log.info('Using security groups {groups}'.format(
                        groups=', '.join(self.security_groups)))

    def establish_ec2_connection(self):

        self.log.info('Using EC2 Region \'{region}\''.format(
                        region=self.region))
        self.log.info("Attempting to connect to EC2")

        try:
            self.ec2 = boto.ec2.connect_to_region(self.region)

            if self.verbose:
                self.log.info('Established connection to EC2')
        except Exception, e:
            raise e

    def autorun(self):

        self.configure()
<|endoftext|>