<commit_before>import json
import traceback

import etcd
import gevent.event

from tendrl.commons.event import Event
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.message import Message, ExceptionMessage
from tendrl.commons.objects.job import Job

class JobConsumerThread(gevent.greenlet.Greenlet):

    def __init__(self):
        super(JobConsumerThread, self).__init__()
        self._complete = gevent.event.Event()

    def _run(self):
        Event(
            Message(
                priority="info",
                publisher=NS.publisher_id,
                payload={"message": "%s running" % self.__class__.__name__}
            )
        )
        while not self._complete.is_set():
            gevent.sleep(2)
            try:
                try:
                    jobs = NS.etcd_orm.client.read("/queue")
                except etcd.EtcdKeyNotFound:
                    continue

                for job in jobs.leaves:
                    try:
                        jid = job.key.split('/')[-1]
                        job = Job(job_id=jid).load()
                        if job.locked_by:
                            continue
                        raw_job = {}
                        raw_job["payload"] = json.loads(job.payload.decode('utf-8'))
                    except etcd.EtcdKeyNotFound:
                        continue

                    if raw_job['payload']["type"] == NS.type and \
                            job.status == "new":

                        # Job routing

                        # Flows created by tendrl-api use 'tags' from flow definition to target jobs
                        _tag_match = False
                        if raw_job.get("payload", {}).get("tags", []):
                            NS.node_context = NS.node_context.load()
                            tags = json.loads(NS.node_context.tags)
                            for flow_tag in raw_job['payload']['tags']:
                                if flow_tag in tags:
                                    _tag_match = True

                        # Flows created by tendrl backend use 'node_ids' to target jobs
                        _node_id_match = False
                        if raw_job.get("payload", {}).get("node_ids", []):
                            if NS.node_context.node_id in \
                                    raw_job['payload']['node_ids']:
                                _node_id_match = True

                        if not _tag_match and not _node_id_match:
                            _job_node_ids = ", ".join(raw_job.get("payload",
                                                                  {}).get("node_ids",
                                                                          []))
                            _job_tags = ", ".join(raw_job.get("payload", {}).get("tags", []))
                            _msg = "Node (%s)(tags: %s) will not process job-%s (node_ids: %s)(tags: %s)" % (NS.node_context.node_id,
                                                                                json.loads(NS.node_context.tags),
                                                                                _job_node_ids,
                                                                                _job_tags)
                            Event(
                                Message(
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": _msg}
                                )
                            )
                            continue

                        job_status_key = "/queue/%s/status" % job.job_id
                        job_lock_key = "/queue/%s/locked_by" % job.job_id
                        try:
                            lock_info = dict(node_id=NS.node_context.node_id, fqdn=NS.node_context.fqdn,
                                             tags=NS.node_context.tags)
                            NS.etcd_orm.client.write(job_lock_key, json.dumps(lock_info))
                            NS.etcd_orm.client.write(job_status_key, "processing", prevValue="new")
                        except etcd.EtcdCompareFailed:
                            # job is already being processed by some tendrl agent
                            continue

                        current_ns, flow_name, obj_name = \
                            self._extract_fqdn(raw_job['payload']['run'])

                        if obj_name:
                            runnable_flow = current_ns.ns.get_obj_flow(
                                obj_name, flow_name)
                        else:
                            runnable_flow = current_ns.ns.get_flow(flow_name)
                        try:

                            the_flow = runnable_flow(parameters=raw_job['payload']['parameters'],
                                                     job_id=job.job_id)
                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id=the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Processing Job %s" %
                                             job.job_id
                                             }
                                )
                            )

                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Running Flow %s" %
                                            raw_job['payload']['run']
                                         }
                                )
                            )
                            the_flow.run()
                            try:
                                NS.etcd_orm.client.write(job_status_key, "finished", prevValue="processing")
                            except etcd.EtcdCompareFailed:
                                # This should not happen!
                                raise FlowExecutionFailedError("Cannnot mark job as 'finished', current job status invalid")

                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "JOB[%s]:  Finished Flow %s" %
                                            (job.job_id, raw_job['payload']['run'])
                                         }
                                )
                            )
                        except (FlowExecutionFailedError, Exception) as e:
                            Event(
                                ExceptionMessage(
                                    priority="error",
                                    publisher=NS.publisher_id,
                                    payload={"message": "error",
                                             "exception": e
                                             }
                                )
                            )
                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="error",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Job failed %s: %s" % (e, e.message)}
                                )
                            )
                            try:
                                NS.etcd_orm.client.write(job_status_key, "failed", prevValue="processing")
                            except etcd.EtcdCompareFailed:
                                # This should not happen!
                                raise FlowExecutionFailedError("Cannnot mark job as 'failed', current job status invalid")
                            job = job.load()
                            job.errors = str(e)
                            job.save()
            except Exception as ex:
                Event(
                    ExceptionMessage(
                        priority="error",
                        publisher=NS.publisher_id,
                        payload={"message": "Job /queue empty",
                                 "exception": ex
                                 }
                    )
                )

    def stop(self):
        self._complete.set()

    def _extract_fqdn(self, flow_fqdn):
        ns, flow_name = flow_fqdn.split(".flows.")
        obj_name = None

        # check if the flow is bound to any object
        try:
            ns, obj_name = ns.split(".objects.")
        except ValueError:
            pass

        ns_str = ns.split(".")[-1]
        if "integrations" in ns:
            return getattr(NS.integrations, ns_str), flow_name, obj_name
        else:
            return getattr(NS, ns_str), flow_name, obj_name

<commit_msg>Update __init__.py<commit_after>import json
import traceback

import etcd
import gevent.event

from tendrl.commons.event import Event
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.message import Message, ExceptionMessage
from tendrl.commons.objects.job import Job

class JobConsumerThread(gevent.greenlet.Greenlet):

    def __init__(self):
        super(JobConsumerThread, self).__init__()
        self._complete = gevent.event.Event()

    def _run(self):
        Event(
            Message(
                priority="info",
                publisher=NS.publisher_id,
                payload={"message": "%s running" % self.__class__.__name__}
            )
        )
        while not self._complete.is_set():
            gevent.sleep(2)
            try:
                try:
                    jobs = NS.etcd_orm.client.read("/queue")
                except etcd.EtcdKeyNotFound:
                    continue

                for job in jobs.leaves:
                    try:
                        jid = job.key.split('/')[-1]
                        job = Job(job_id=jid).load()
                        if job.locked_by:
                            continue
                        raw_job = {}
                        raw_job["payload"] = json.loads(job.payload.decode('utf-8'))
                    except etcd.EtcdKeyNotFound:
                        continue

                    if raw_job['payload']["type"] == NS.type and \
                            job.status == "new":

                        # Job routing

                        # Flows created by tendrl-api use 'tags' from flow definition to target jobs
                        _tag_match = False
                        if raw_job.get("payload", {}).get("tags", []):
                            NS.node_context = NS.node_context.load()
                            tags = json.loads(NS.node_context.tags)
                            for flow_tag in raw_job['payload']['tags']:
                                if flow_tag in tags:
                                    _tag_match = True

                        # Flows created by tendrl backend use 'node_ids' to target jobs
                        _node_id_match = False
                        if raw_job.get("payload", {}).get("node_ids", []):
                            if NS.node_context.node_id in \
                                    raw_job['payload']['node_ids']:
                                _node_id_match = True

                        if not _tag_match and not _node_id_match:
                            _job_node_ids = ", ".join(raw_job.get("payload",
                                                                  {}).get("node_ids",
                                                                          []))
                            _job_tags = ", ".join(raw_job.get("payload", {}).get("tags", []))
                            _msg = "Node (%s)(tags: %s) will not process job-%s (node_ids: %s)(tags: %s)" % (NS.node_context.node_id,
                                                                                json.loads(NS.node_context.tags),
                                                                                jid,
                                                                                _job_node_ids,
                                                                                _job_tags)
                            Event(
                                Message(
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": _msg}
                                )
                            )
                            continue

                        job_status_key = "/queue/%s/status" % job.job_id
                        job_lock_key = "/queue/%s/locked_by" % job.job_id
                        try:
                            lock_info = dict(node_id=NS.node_context.node_id, fqdn=NS.node_context.fqdn,
                                             tags=NS.node_context.tags)
                            NS.etcd_orm.client.write(job_lock_key, json.dumps(lock_info))
                            NS.etcd_orm.client.write(job_status_key, "processing", prevValue="new")
                        except etcd.EtcdCompareFailed:
                            # job is already being processed by some tendrl agent
                            continue

                        current_ns, flow_name, obj_name = \
                            self._extract_fqdn(raw_job['payload']['run'])

                        if obj_name:
                            runnable_flow = current_ns.ns.get_obj_flow(
                                obj_name, flow_name)
                        else:
                            runnable_flow = current_ns.ns.get_flow(flow_name)
                        try:

                            the_flow = runnable_flow(parameters=raw_job['payload']['parameters'],
                                                     job_id=job.job_id)
                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id=the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Processing Job %s" %
                                             job.job_id
                                             }
                                )
                            )

                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Running Flow %s" %
                                            raw_job['payload']['run']
                                         }
                                )
                            )
                            the_flow.run()
                            try:
                                NS.etcd_orm.client.write(job_status_key, "finished", prevValue="processing")
                            except etcd.EtcdCompareFailed:
                                # This should not happen!
                                raise FlowExecutionFailedError("Cannnot mark job as 'finished', current job status invalid")

                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="info",
                                    publisher=NS.publisher_id,
                                    payload={"message": "JOB[%s]:  Finished Flow %s" %
                                            (job.job_id, raw_job['payload']['run'])
                                         }
                                )
                            )
                        except (FlowExecutionFailedError, Exception) as e:
                            Event(
                                ExceptionMessage(
                                    priority="error",
                                    publisher=NS.publisher_id,
                                    payload={"message": "error",
                                             "exception": e
                                             }
                                )
                            )
                            Event(
                                Message(
                                    job_id=job.job_id,
                                    flow_id = the_flow.parameters['flow_id'],
                                    priority="error",
                                    publisher=NS.publisher_id,
                                    payload={"message": "Job failed %s: %s" % (e, e.message)}
                                )
                            )
                            try:
                                NS.etcd_orm.client.write(job_status_key, "failed", prevValue="processing")
                            except etcd.EtcdCompareFailed:
                                # This should not happen!
                                raise FlowExecutionFailedError("Cannnot mark job as 'failed', current job status invalid")
                            job = job.load()
                            job.errors = str(e)
                            job.save()
            except Exception as ex:
                Event(
                    ExceptionMessage(
                        priority="error",
                        publisher=NS.publisher_id,
                        payload={"message": "Job /queue empty",
                                 "exception": ex
                                 }
                    )
                )

    def stop(self):
        self._complete.set()

    def _extract_fqdn(self, flow_fqdn):
        ns, flow_name = flow_fqdn.split(".flows.")
        obj_name = None

        # check if the flow is bound to any object
        try:
            ns, obj_name = ns.split(".objects.")
        except ValueError:
            pass

        ns_str = ns.split(".")[-1]
        if "integrations" in ns:
            return getattr(NS.integrations, ns_str), flow_name, obj_name
        else:
            return getattr(NS, ns_str), flow_name, obj_name

<|endoftext|>


<commit_before>import sys, os, shutil
from nose.tools import *
import nipype.pipeline.engine as pe

import logging
logger = logging.getLogger(__name__)

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
from qipipe.pipelines import staging
from test.unit.pipelines.pipelines_helper import get_xnat_subjects, clear_xnat_subjects

ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
"""The test parent directory."""

FIXTURE = os.path.join(ROOT, 'fixtures', 'staging', 'breast')
"""The test fixture directory."""

RESULTS = os.path.join(ROOT, 'results', 'pipelines', 'staging')
"""The test results directory."""

COLLECTION = "Breast"
"""The QIN test collection."""

from nipype import config
cfg = dict(logging=dict(workflow_level='DEBUG', log_directory=RESULTS, log_to_file=True),
    execution=dict(crashdump_dir=RESULTS, create_report=False, keep_inputs=True))
config.update_config(cfg)

class TestStagingWorkflow:
    """Registration pipeline unit tests."""

    def setUp(self):
        shutil.rmtree(RESULTS, True)

    def tearDown(self):
        shutil.rmtree(RESULTS, True)

    def test_staging(self):
        """
        Run the registration pipeline and verify that the registered images are created
        in XNAT.
        """

        logger.debug("Testing the registration pipeline on %s..." % FIXTURE)

        # The test subject => directory dictionary.
        sbj_dir_dict = get_xnat_subjects(COLLECTION, FIXTURE)
        # Delete any existing test subjects.
        clear_xnat_subjects(sbj_dir_dict.iterkeys())

        # The staging destination and work area.
        dest = os.path.join(RESULTS, 'data')
        work = os.path.join(RESULTS, 'work')

        # Run the workflow.
        logger.debug("Executing the registration workflow...")
        sessions = staging.run(COLLECTION, dest=dest, base_dir=work, *sbj_dir_dict.itervalues())

        # Verify the result.
        for sbj in sbj_dir_dict.iterkeys():
            assert_true(sbj.exists(), "Subject not created in XNAT: %s" % sbj.label())
        for sess in sessions:
            assert_true(sess.exists(), "Session not created in XNAT: %s" % sess)

        # Cleanup.
        for sbj in sbj_dir_dict.iterkeys():
            sbj.delete(delete_files=True)

if __name__ == "__main__":
    import nose

    nose.main(defaultTest=__name__)
<commit_msg>Pull out setup/teardown.<commit_after>import sys, os, shutil
from nose.tools import *
import nipype.pipeline.engine as pe

import logging
logger = logging.getLogger(__name__)

sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..'))
from qipipe.pipelines import staging
from test.unit.pipelines.pipelines_helper import get_xnat_subjects, clear_xnat_subjects

ROOT = os.path.normpath(os.path.join(os.path.dirname(__file__), '..', '..'))
"""The test parent directory."""

FIXTURE = os.path.join(ROOT, 'fixtures', 'staging', 'breast')
"""The test fixture directory."""

RESULTS = os.path.join(ROOT, 'results', 'pipelines', 'staging')
"""The test results directory."""

COLLECTION = "Breast"
"""The QIN test collection."""

from nipype import config
cfg = dict(logging=dict(workflow_level='DEBUG', log_directory=RESULTS, log_to_file=True),
    execution=dict(crashdump_dir=RESULTS, create_report=False, keep_inputs=True))
config.update_config(cfg)

class TestStagingWorkflow:
    """Registration pipeline unit tests."""

    def setUp(self):
        shutil.rmtree(RESULTS, True)
        # The test subject => directory dictionary.
        self._sbj_dir_dict = get_xnat_subjects(COLLECTION, FIXTURE)
        # Delete any existing test subjects.
        clear_xnat_subjects(*self._sbj_dir_dict.keys())

    def tearDown(self):
        shutil.rmtree(RESULTS, True)
        clear_xnat_subjects(*self._sbj_dir_dict.keys())

    def test_staging(self):
        """
        Run the staging pipeline and verify that the registered images are created
        in XNAT.
        """

        logger.debug("Testing the registration pipeline on %s..." % FIXTURE)

        # The staging destination and work area.
        dest = os.path.join(RESULTS, 'data')
        work = os.path.join(RESULTS, 'work')

        # Run the workflow.
        logger.debug("Executing the staging workflow...")
        sessions = staging.run(COLLECTION, dest=dest, base_dir=work, *self._sbj_dir_dict.itervalues())

        # Verify the result.
        for sbj in self._sbj_dir_dict.iterkeys():
            assert_true(sbj.exists(), "The subject was not created in XNAT: %s" % sbj.label())
        for sess in sessions:
            assert_true(sess.exists(), "The session not created in XNAT: %s" % sess)

if __name__ == "__main__":
    import nose

    nose.main(defaultTest=__name__)
<|endoftext|>


<commit_before># Copyright (c) 2015 Cloudera, Inc. All rights reserved.

import pytest
from subprocess import check_call
from tests.common.test_vector import *
from tests.common.impala_test_suite import *
from tests.common.skip import IS_S3
from tests.util.filesystem_utils import WAREHOUSE

TEST_DB = 'hidden_files_db'
TEST_TBL = 'hf'

class TestHiddenFiles(ImpalaTestSuite):
  """
  Tests that files with special prefixes/suffixes are considered 'hidden' when
  loading table metadata and running queries.
  """

  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestHiddenFiles, cls).add_test_dimensions()
    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())
    cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
    # Only run in exhaustive mode on hdfs since this test takes a long time.
    if cls.exploration_strategy() != 'exhaustive' and not IS_S3:
      cls.TestMatrix.clear()

  def setup_method(self, method):
    self.cleanup_db(TEST_DB)
    self.client.execute("create database %s location '%s/%s'" % (TEST_DB, WAREHOUSE,
      TEST_DB))
    self.client.execute(
      "create table %s.%s like functional.alltypes" % (TEST_DB, TEST_TBL))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=1)" % (TEST_DB, TEST_TBL))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=2)" % (TEST_DB, TEST_TBL))

    self.__populate_test_table()

  def teardown_method(self, method):
    self.cleanup_db(TEST_DB)

  def __populate_test_table(self):
    """Copy files into the HDFS directories of two partitions of the table.
    The goal is to have both an empty and non-empty partition with hidden files."""

    ALLTYPES_LOC = "%s/alltypes" % WAREHOUSE
    TEST_TBL_LOC = "%s/%s/%s" % (WAREHOUSE, TEST_DB, TEST_TBL)
    # Copy a visible file into one of the partitions.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt" % TEST_TBL_LOC], shell=False)
    # Add hidden files to the non-empty partition. Use upper case hidden suffixes.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/.100101.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/_100101.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.COPYING" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.TMP" % TEST_TBL_LOC], shell=False)
    # Add hidden files to the empty partition. Use lower case hidden suffixes.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/.100201.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/_100201.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.copying" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.tmp" % TEST_TBL_LOC], shell=False)

  @pytest.mark.execute_serially
  def test_hidden_files_load(self, vector):
    """Tests that an incremental refresh ignores hidden files."""
    self.client.execute("invalidate metadata %s.%s" % (TEST_DB, TEST_TBL))
    self.run_test_case('QueryTest/hidden-files', vector)

  @pytest.mark.execute_serially
  def test_hidden_files_refresh(self, vector):
    """Tests that an incremental refresh ignores hidden files."""
    self.client.execute("refresh %s.%S" % (TEST_DB, TEST_TBL))
    self.run_test_case('QueryTest/hidden-files', vector)
<commit_msg>Fix typo in hidden files test<commit_after># Copyright (c) 2015 Cloudera, Inc. All rights reserved.

import pytest
from subprocess import check_call
from tests.common.test_vector import *
from tests.common.impala_test_suite import *
from tests.common.skip import IS_S3
from tests.util.filesystem_utils import WAREHOUSE

TEST_DB = 'hidden_files_db'
TEST_TBL = 'hf'

class TestHiddenFiles(ImpalaTestSuite):
  """
  Tests that files with special prefixes/suffixes are considered 'hidden' when
  loading table metadata and running queries.
  """

  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestHiddenFiles, cls).add_test_dimensions()
    cls.TestMatrix.add_dimension(create_single_exec_option_dimension())
    cls.TestMatrix.add_dimension(create_uncompressed_text_dimension(cls.get_workload()))
    # Only run in exhaustive mode on hdfs since this test takes a long time.
    if cls.exploration_strategy() != 'exhaustive' and not IS_S3:
      cls.TestMatrix.clear()

  def setup_method(self, method):
    self.cleanup_db(TEST_DB)
    self.client.execute("create database %s location '%s/%s'" % (TEST_DB, WAREHOUSE,
      TEST_DB))
    self.client.execute(
      "create table %s.%s like functional.alltypes" % (TEST_DB, TEST_TBL))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=1)" % (TEST_DB, TEST_TBL))
    self.client.execute(
      "alter table %s.%s add partition (year=2010, month=2)" % (TEST_DB, TEST_TBL))

    self.__populate_test_table()

  def teardown_method(self, method):
    self.cleanup_db(TEST_DB)

  def __populate_test_table(self):
    """Copy files into the HDFS directories of two partitions of the table.
    The goal is to have both an empty and non-empty partition with hidden files."""

    ALLTYPES_LOC = "%s/alltypes" % WAREHOUSE
    TEST_TBL_LOC = "%s/%s/%s" % (WAREHOUSE, TEST_DB, TEST_TBL)
    # Copy a visible file into one of the partitions.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt" % TEST_TBL_LOC], shell=False)
    # Add hidden files to the non-empty partition. Use upper case hidden suffixes.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/.100101.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/_100101.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.COPYING" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=1/100101.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=1/100101.txt.TMP" % TEST_TBL_LOC], shell=False)
    # Add hidden files to the empty partition. Use lower case hidden suffixes.
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/.100201.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/_100201.txt" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.copying" % TEST_TBL_LOC], shell=False)
    check_call(["hadoop", "fs", "-cp",
          "%s/year=2010/month=2/100201.txt" % ALLTYPES_LOC,
          "%s/year=2010/month=2/100201.txt.tmp" % TEST_TBL_LOC], shell=False)

  @pytest.mark.execute_serially
  def test_hidden_files_load(self, vector):
    """Tests that an incremental refresh ignores hidden files."""
    self.client.execute("invalidate metadata %s.%s" % (TEST_DB, TEST_TBL))
    self.run_test_case('QueryTest/hidden-files', vector)

  @pytest.mark.execute_serially
  def test_hidden_files_refresh(self, vector):
    """Tests that an incremental refresh ignores hidden files."""
    self.client.execute("refresh %s.%s" % (TEST_DB, TEST_TBL))
    self.run_test_case('QueryTest/hidden-files', vector)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""Unit tests for SerializedDagModel."""

import unittest

from airflow import example_dags as example_dags_module
from airflow.models import DagBag
from airflow.models.serialized_dag import SerializedDagModel as SDM
from airflow.serialization.serialized_objects import SerializedDAG
from airflow.utils.session import create_session

# To move it to a shared module.
def make_example_dags(module):
    """Loads DAGs from a module for test."""
    dagbag = DagBag(module.__path__[0])
    return dagbag.dags

def clear_db_serialized_dags():
    with create_session() as session:
        session.query(SDM).delete()

class SerializedDagModelTest(unittest.TestCase):
    """Unit tests for SerializedDagModel."""

    def setUp(self):
        clear_db_serialized_dags()

    def tearDown(self):
        clear_db_serialized_dags()

    def test_dag_fileloc_hash(self):
        """Verifies the correctness of hashing file path."""
        self.assertTrue(SDM.dag_fileloc_hash('/airflow/dags/test_dag.py') == 60791)

    def _write_example_dags(self):
        example_dags = make_example_dags(example_dags_module)
        for dag in example_dags.values():
            SDM.write_dag(dag)
        return example_dags

    def test_write_dag(self):
        """DAGs can be written into database."""
        example_dags = self._write_example_dags()

        with create_session() as session:
            for dag in example_dags.values():
                self.assertTrue(SDM.has_dag(dag.dag_id))
                result = session.query(
                    SDM.fileloc, SDM.data).filter(SDM.dag_id == dag.dag_id).one()

                self.assertTrue(result.fileloc == dag.full_filepath)
                # Verifies JSON schema.
                SerializedDAG.validate_schema(result.data)

    def test_read_dags(self):
        """DAGs can be read from database."""
        example_dags = self._write_example_dags()
        serialized_dags = SDM.read_all_dags()
        self.assertTrue(len(example_dags) == len(serialized_dags))
        for dag_id, dag in example_dags.items():
            serialized_dag = serialized_dags[dag_id]

            self.assertTrue(serialized_dag.dag_id == dag.dag_id)
            self.assertTrue(set(serialized_dag.task_dict) == set(dag.task_dict))

    def test_remove_dags(self):
        """DAGs can be removed from database."""
        example_dags_list = list(self._write_example_dags().values())
        # Remove SubDags from the list as they are not stored in DB in a separate row
        # and are directly added in Json blob of the main DAG
        filtered_example_dags_list = [dag for dag in example_dags_list if not dag.is_subdag]
        # Tests removing by dag_id.
        dag_removed_by_id = filtered_example_dags_list[0]
        SDM.remove_dag(dag_removed_by_id.dag_id)
        self.assertFalse(SDM.has_dag(dag_removed_by_id.dag_id))

        # Tests removing by file path.
        dag_removed_by_file = filtered_example_dags_list[1]
        example_dag_files = [dag.full_filepath for dag in filtered_example_dags_list]
        example_dag_files.remove(dag_removed_by_file.full_filepath)
        SDM.remove_deleted_dags(example_dag_files)
        self.assertFalse(SDM.has_dag(dag_removed_by_file.dag_id))
<commit_msg>[AIRFLOW-6638] Remove flakiness test from test_serialized_db remove (#7258)<commit_after># -*- coding: utf-8 -*-
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

"""Unit tests for SerializedDagModel."""

import unittest

from airflow import example_dags as example_dags_module
from airflow.models import DagBag
from airflow.models.serialized_dag import SerializedDagModel as SDM
from airflow.serialization.serialized_objects import SerializedDAG
from airflow.utils.session import create_session

# To move it to a shared module.
def make_example_dags(module):
    """Loads DAGs from a module for test."""
    dagbag = DagBag(module.__path__[0])
    return dagbag.dags

def clear_db_serialized_dags():
    with create_session() as session:
        session.query(SDM).delete()

class SerializedDagModelTest(unittest.TestCase):
    """Unit tests for SerializedDagModel."""

    def setUp(self):
        clear_db_serialized_dags()

    def tearDown(self):
        clear_db_serialized_dags()

    def test_dag_fileloc_hash(self):
        """Verifies the correctness of hashing file path."""
        self.assertTrue(SDM.dag_fileloc_hash('/airflow/dags/test_dag.py') == 60791)

    def _write_example_dags(self):
        example_dags = make_example_dags(example_dags_module)
        for dag in example_dags.values():
            SDM.write_dag(dag)
        return example_dags

    def test_write_dag(self):
        """DAGs can be written into database."""
        example_dags = self._write_example_dags()

        with create_session() as session:
            for dag in example_dags.values():
                self.assertTrue(SDM.has_dag(dag.dag_id))
                result = session.query(
                    SDM.fileloc, SDM.data).filter(SDM.dag_id == dag.dag_id).one()

                self.assertTrue(result.fileloc == dag.full_filepath)
                # Verifies JSON schema.
                SerializedDAG.validate_schema(result.data)

    def test_read_dags(self):
        """DAGs can be read from database."""
        example_dags = self._write_example_dags()
        serialized_dags = SDM.read_all_dags()
        self.assertTrue(len(example_dags) == len(serialized_dags))
        for dag_id, dag in example_dags.items():
            serialized_dag = serialized_dags[dag_id]

            self.assertTrue(serialized_dag.dag_id == dag.dag_id)
            self.assertTrue(set(serialized_dag.task_dict) == set(dag.task_dict))

    def test_remove_dags_by_id(self):
        """DAGs can be removed from database."""
        example_dags_list = list(self._write_example_dags().values())
        # Remove SubDags from the list as they are not stored in DB in a separate row
        # and are directly added in Json blob of the main DAG
        filtered_example_dags_list = [dag for dag in example_dags_list if not dag.is_subdag]
        # Tests removing by dag_id.
        dag_removed_by_id = filtered_example_dags_list[0]
        SDM.remove_dag(dag_removed_by_id.dag_id)
        self.assertFalse(SDM.has_dag(dag_removed_by_id.dag_id))

    def test_remove_dags_by_filepath(self):
        """DAGs can be removed from database."""
        example_dags_list = list(self._write_example_dags().values())
        # Remove SubDags from the list as they are not stored in DB in a separate row
        # and are directly added in Json blob of the main DAG
        filtered_example_dags_list = [dag for dag in example_dags_list if not dag.is_subdag]
        # Tests removing by file path.
        dag_removed_by_file = filtered_example_dags_list[0]
        # remove repeated files for those DAGs that define multiple dags in the same file (set comprehension)
        example_dag_files = list({dag.full_filepath for dag in filtered_example_dags_list})
        example_dag_files.remove(dag_removed_by_file.full_filepath)
        SDM.remove_deleted_dags(example_dag_files)
        self.assertFalse(SDM.has_dag(dag_removed_by_file.dag_id))
<|endoftext|>


<commit_before>"""
Tests donut/modules/groups/
"""
from donut.testing.fixtures import client
from donut import app
from donut.modules.groups.helpers import get_position_data
from donut.modules.groups.routes import get_positions
from donut.modules.groups import helpers
from donut.modules.groups import routes

group = {
    "group_id": 1,
    "group_name": "Donut Devteam",
    "group_desc": None,
    "type": ""
}

# Helpers
def test_get_group_list_data(client):
    assert helpers.get_group_list_data(["not_a_real_field"]) == "Invalid field"
    assert helpers.get_group_list_data(["group_name"]) == [{
        "group_name":
        "Donut Devteam"
    }]
    groups = [group]
    assert helpers.get_group_list_data() == groups
    assert helpers.get_group_list_data(None, {"group_id": 1}) == groups
    assert helpers.get_group_list_data(None, {"group_id": 2}) == []

def test_get_group_positions_data(client):
    assert helpers.get_group_positions(1) == [{
        "pos_id": 1,
        "pos_name": "Head"
    }, {
        "pos_id": 2,
        "pos_name": "Secretary"
    }]
    assert helpers.get_group_positions(2) == []

def test_get_position_data(client):
    res = get_position_data()
    assert res is not None

def test_get_group_data(client):
    assert helpers.get_group_data(2) == {}
    assert helpers.get_group_data(1, ["not_a_real_field"]) == "Invalid field"
    assert helpers.get_group_data(1) == group
    assert helpers.get_group_data(1, ["group_name"]) == {
        "group_name": "Donut Devteam"
    }

def test_get_members_by_group(client):
    assert helpers.get_members_by_group(1)[0]["user_id"] == 1
    assert helpers.get_members_by_group(2) == {}

# Test Routes
# Difficult to test because query arguments are undefined outside Flask context
# def test_get_groups_list(client):
#     assert routes.get_groups_list() is not None

def test_get_group_positions(client):
    assert routes.get_group_positions(1) is not None

def get_groups(client):
    assert routes.get_groups(1) is not None

def test_get_group_members(client):
    assert routes.get_group_members(1) is not None

<commit_msg>fixed another merge conflict issue in test_groups; mainly import conflict<commit_after>"""
Tests donut/modules/groups/
"""
from donut.testing.fixtures import client
from donut import app
from donut.modules.groups import helpers
from donut.modules.groups import routes

group = {
    "group_id": 1,
    "group_name": "Donut Devteam",
    "group_desc": None,
    "type": ""
}

# Helpers
def test_get_group_list_data(client):
    assert helpers.get_group_list_data(["not_a_real_field"]) == "Invalid field"
    assert helpers.get_group_list_data(["group_name"]) == [{
        "group_name":
        "Donut Devteam"
    }]
    groups = [group]
    assert helpers.get_group_list_data() == groups
    assert helpers.get_group_list_data(None, {"group_id": 1}) == groups
    assert helpers.get_group_list_data(None, {"group_id": 2}) == []

def test_get_group_positions_data(client):
    assert helpers.get_group_positions(1) == [{
        "pos_id": 1,
        "pos_name": "Head"
    }, {
        "pos_id": 2,
        "pos_name": "Secretary"
    }]
    assert helpers.get_group_positions(2) == []

def test_get_position_data(client):
    res = helpers.get_position_data()
    assert res is not None

def test_get_group_data(client):
    assert helpers.get_group_data(2) == {}
    assert helpers.get_group_data(1, ["not_a_real_field"]) == "Invalid field"
    assert helpers.get_group_data(1) == group
    assert helpers.get_group_data(1, ["group_name"]) == {
        "group_name": "Donut Devteam"
    }

def test_get_members_by_group(client):
    assert helpers.get_members_by_group(1)[0]["user_id"] == 1
    assert helpers.get_members_by_group(2) == {}

# Test Routes
# Difficult to test because query arguments are undefined outside Flask context
# def test_get_groups_list(client):
#     assert routes.get_groups_list() is not None

def test_get_group_positions(client):
    assert routes.get_group_positions(1) is not None

def get_groups(client):
    assert routes.get_groups(1) is not None

def test_get_group_members(client):
    assert routes.get_group_members(1) is not None
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# encoding: utf-8
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )

"""Contains Django model for the scanner types."""

import os
import datetime
import json

from subprocess import Popen

from django.core.validators import validate_comma_separated_integer_list
from django.conf import settings
from django.db import models
from recurrence.fields import RecurrenceField

from .organization_model import Organization
from .group_model import Group
from .regexrule_model import RegexRule
from .userprofile_model import UserProfile

base_dir = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

class Scanner(models.Model):

    """A scanner, i.e. a template for actual scanning jobs."""

    name = models.CharField(max_length=256, unique=True, null=False,
                            verbose_name='Navn')
    organization = models.ForeignKey(Organization, null=False,
                                     verbose_name='Organisation')

    group = models.ForeignKey(Group, null=True, blank=True,
                              verbose_name='Gruppe')
    schedule = RecurrenceField(max_length=1024,
                               verbose_name='Planlagt afvikling')
    do_cpr_scan = models.BooleanField(default=True, verbose_name='CPR')
    do_name_scan = models.BooleanField(default=False, verbose_name='Navn')
    do_address_scan = models.BooleanField(default=False,
                                          verbose_name='Adresse')
    do_ocr = models.BooleanField(default=False, verbose_name='Scan billeder')
    do_last_modified_check = models.BooleanField(default=True,
                                                 verbose_name='Tjek sidst ændret dato')
    do_cpr_modulus11 = models.BooleanField(default=True,
                                           verbose_name='Tjek modulus-11')
    do_cpr_ignore_irrelevant = models.BooleanField(
        default=True,
        verbose_name='Ignorer ugyldige fødselsdatoer')
    columns = models.CharField(validators=[validate_comma_separated_integer_list],
                               max_length=128,
                               null=True,
                               blank=True
                               )

    regex_rules = models.ManyToManyField(RegexRule,
                                         blank=True,
                                         verbose_name='Regex regler')
    recipients = models.ManyToManyField(UserProfile, blank=True,
                                        verbose_name='Modtagere')

    # Spreadsheet annotation and replacement parameters

    # Save a copy of any spreadsheets scanned with annotations
    # in each row where matches were found. If this is enabled and any of
    # the replacement parameters are enabled (e.g. do_cpr_replace), matches
    # will also be replaced with the specified text (e.g. cpr_replace_text).
    output_spreadsheet_file = models.BooleanField(default=False)

    # Replace CPRs?
    do_cpr_replace = models.BooleanField(default=False)
    # Text to replace CPRs with
    cpr_replace_text = models.CharField(max_length=2048, null=True,
                                        blank=True)
    # Replace names?
    do_name_replace = models.BooleanField(default=False)
    # Text to replace names with
    name_replace_text = models.CharField(max_length=2048, null=True,
                                         blank=True)
    # Replace addresses?
    do_address_replace = models.BooleanField(default=False)
    # Text to replace addresses with
    address_replace_text = models.CharField(max_length=2048, null=True,
                                            blank=True)
    is_running = models.BooleanField(default=False)

    @property
    def schedule_description(self):
        """A lambda for creating schedule description strings."""
        rules = [r for r in self.schedule.rrules]  # Use r.to_text() to render
        dates = [d for d in self.schedule.rdates]
        if len(rules) > 0 or len(dates) > 0:
            return u"Ja"
        else:
            return u"Nej"

    # Run error messages
    ALREADY_RUNNING = (
        "Scanneren kunne ikke startes," +
        " fordi der allerede er en scanning i gang for den."
    )
    NO_VALID_DOMAINS = (
        "Scanneren kunne ikke startes," +
        " fordi den ikke har nogen gyldige domæner."
    )

    # DON'T USE DIRECTLY !!!
    # Use process_urls property instead.
    encoded_process_urls = models.CharField(
        max_length=262144,
        null=True,
        blank=True
    )
    # Booleans for control of scanners run from web service.
    do_run_synchronously = models.BooleanField(default=False)
    is_visible = models.BooleanField(default=True)

    def _get_process_urls(self):
        s = self.encoded_process_urls
        if s:
            urls = json.loads(s)
        else:
            urls = []
        return urls

    def _set_process_urls(self, urls):
        self.encoded_process_urls = json.dumps(urls)

    process_urls = property(_get_process_urls, _set_process_urls)

    # First possible start time
    FIRST_START_TIME = datetime.time(18, 0)
    # Amount of quarter-hours that can be added to the start time
    STARTTIME_QUARTERS = 6 * 4

    def get_start_time(self):
        """The time of day the Scanner should be automatically started."""
        added_minutes = 15 * (self.pk % Scanner.STARTTIME_QUARTERS)
        added_hours = int(added_minutes / 60)
        added_minutes -= added_hours * 60
        return Scanner.FIRST_START_TIME.replace(
            hour=Scanner.FIRST_START_TIME.hour + added_hours,
            minute=Scanner.FIRST_START_TIME.minute + added_minutes
        )

    @property
    def has_valid_domains(self):
        return len([d for d in self.domains.all() if d.validation_status]) > 0

    @classmethod
    def modulo_for_starttime(cls, time):
        """Convert a datetime.time object to the corresponding modulo value.

        The modulo value can be used to search the database for scanners that
        should be started at the given time by filtering a query with:
            (WebScanner.pk % WebScanner.STARTTIME_QUARTERS) == <modulo_value>
        """
        if(time < cls.FIRST_START_TIME):
            return None
        hours = time.hour - cls.FIRST_START_TIME.hour
        minutes = 60 * hours + time.minute - cls.FIRST_START_TIME.minute
        return int(minutes / 15)

    @property
    def display_name(self):
        """The name used when displaying the scanner on the web page."""
        return "WebScanner '%s'" % self.name

    def __unicode__(self):
        """Return the name of the scanner."""
        return self.name

    def __str__(self):
        """Return the name of the scanner."""
        return self.__unicode__()

    def run(self, test_only=False, blocking=False, user=None):
        """Run a scan with the Scanner.

        Return the Scan object if we started the scanner.
        Return None if there is already a scanner running,
        or if there was a problem running the scanner.
        If test_only is True, only check if we can run a scan, don't actually
        run one.
        """
        if self.is_running:
            return Scanner.ALREADY_RUNNING

        if not self.has_valid_domains:
            return Scanner.NO_VALID_DOMAINS

        # Create a new Scan
        scan = self.create_scan()
        if isinstance(scan, str):
            return scan
        # Add user as recipient on scan
        if user:
            scan.recipients.add(user.profile)
        # Get path to run script
        SCRAPY_WEBSCANNER_DIR = os.path.join(settings.PROJECT_DIR, "scrapy-webscanner")

        if test_only:
            return scan

        if not os.path.exists(scan.scan_log_dir):
            os.makedirs(scan.scan_log_dir)
        log_file = open(scan.scan_log_file, "a")

        if not os.path.exists(scan.scan_output_files_dir):
            os.makedirs(scan.scan_output_files_dir)

        try:
            process = Popen([os.path.join(SCRAPY_WEBSCANNER_DIR, "run.sh"),
                             str(scan.pk)], cwd=SCRAPY_WEBSCANNER_DIR,
                            stderr=log_file,
                            stdout=log_file)
            if blocking:
                process.communicate()
        except Exception as e:
            print(e)
            return None
        return scan

    class Meta:
        abstract = False
        ordering = ['name']
        db_table = 'os2webscanner_scanner'
<commit_msg>* Added path to SCANNER_DIR when starting exchange scan.<commit_after># -*- coding: UTF-8 -*-
# encoding: utf-8
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )

"""Contains Django model for the scanner types."""

import os
import datetime
import json

from subprocess import Popen

from django.core.validators import validate_comma_separated_integer_list
from django.conf import settings
from django.db import models
from recurrence.fields import RecurrenceField

from .organization_model import Organization
from .group_model import Group
from .regexrule_model import RegexRule
from .userprofile_model import UserProfile
from .exchangescan_model import ExchangeScan

base_dir = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.realpath(__file__))))

class Scanner(models.Model):

    """A scanner, i.e. a template for actual scanning jobs."""

    name = models.CharField(max_length=256, unique=True, null=False,
                            verbose_name='Navn')
    organization = models.ForeignKey(Organization, null=False,
                                     verbose_name='Organisation')

    group = models.ForeignKey(Group, null=True, blank=True,
                              verbose_name='Gruppe')
    schedule = RecurrenceField(max_length=1024,
                               verbose_name='Planlagt afvikling')
    do_cpr_scan = models.BooleanField(default=True, verbose_name='CPR')
    do_name_scan = models.BooleanField(default=False, verbose_name='Navn')
    do_address_scan = models.BooleanField(default=False,
                                          verbose_name='Adresse')
    do_ocr = models.BooleanField(default=False, verbose_name='Scan billeder')
    do_last_modified_check = models.BooleanField(default=True,
                                                 verbose_name='Tjek sidst ændret dato')
    do_cpr_modulus11 = models.BooleanField(default=True,
                                           verbose_name='Tjek modulus-11')
    do_cpr_ignore_irrelevant = models.BooleanField(
        default=True,
        verbose_name='Ignorer ugyldige fødselsdatoer')
    columns = models.CharField(validators=[validate_comma_separated_integer_list],
                               max_length=128,
                               null=True,
                               blank=True
                               )

    regex_rules = models.ManyToManyField(RegexRule,
                                         blank=True,
                                         verbose_name='Regex regler')
    recipients = models.ManyToManyField(UserProfile, blank=True,
                                        verbose_name='Modtagere')

    # Spreadsheet annotation and replacement parameters

    # Save a copy of any spreadsheets scanned with annotations
    # in each row where matches were found. If this is enabled and any of
    # the replacement parameters are enabled (e.g. do_cpr_replace), matches
    # will also be replaced with the specified text (e.g. cpr_replace_text).
    output_spreadsheet_file = models.BooleanField(default=False)

    # Replace CPRs?
    do_cpr_replace = models.BooleanField(default=False)
    # Text to replace CPRs with
    cpr_replace_text = models.CharField(max_length=2048, null=True,
                                        blank=True)
    # Replace names?
    do_name_replace = models.BooleanField(default=False)
    # Text to replace names with
    name_replace_text = models.CharField(max_length=2048, null=True,
                                         blank=True)
    # Replace addresses?
    do_address_replace = models.BooleanField(default=False)
    # Text to replace addresses with
    address_replace_text = models.CharField(max_length=2048, null=True,
                                            blank=True)
    is_running = models.BooleanField(default=False)

    @property
    def schedule_description(self):
        """A lambda for creating schedule description strings."""
        rules = [r for r in self.schedule.rrules]  # Use r.to_text() to render
        dates = [d for d in self.schedule.rdates]
        if len(rules) > 0 or len(dates) > 0:
            return u"Ja"
        else:
            return u"Nej"

    # Run error messages
    ALREADY_RUNNING = (
        "Scanneren kunne ikke startes," +
        " fordi der allerede er en scanning i gang for den."
    )
    NO_VALID_DOMAINS = (
        "Scanneren kunne ikke startes," +
        " fordi den ikke har nogen gyldige domæner."
    )

    # DON'T USE DIRECTLY !!!
    # Use process_urls property instead.
    encoded_process_urls = models.CharField(
        max_length=262144,
        null=True,
        blank=True
    )
    # Booleans for control of scanners run from web service.
    do_run_synchronously = models.BooleanField(default=False)
    is_visible = models.BooleanField(default=True)

    def _get_process_urls(self):
        s = self.encoded_process_urls
        if s:
            urls = json.loads(s)
        else:
            urls = []
        return urls

    def _set_process_urls(self, urls):
        self.encoded_process_urls = json.dumps(urls)

    process_urls = property(_get_process_urls, _set_process_urls)

    # First possible start time
    FIRST_START_TIME = datetime.time(18, 0)
    # Amount of quarter-hours that can be added to the start time
    STARTTIME_QUARTERS = 6 * 4

    def get_start_time(self):
        """The time of day the Scanner should be automatically started."""
        added_minutes = 15 * (self.pk % Scanner.STARTTIME_QUARTERS)
        added_hours = int(added_minutes / 60)
        added_minutes -= added_hours * 60
        return Scanner.FIRST_START_TIME.replace(
            hour=Scanner.FIRST_START_TIME.hour + added_hours,
            minute=Scanner.FIRST_START_TIME.minute + added_minutes
        )

    @property
    def has_valid_domains(self):
        return len([d for d in self.domains.all() if d.validation_status]) > 0

    @classmethod
    def modulo_for_starttime(cls, time):
        """Convert a datetime.time object to the corresponding modulo value.

        The modulo value can be used to search the database for scanners that
        should be started at the given time by filtering a query with:
            (WebScanner.pk % WebScanner.STARTTIME_QUARTERS) == <modulo_value>
        """
        if(time < cls.FIRST_START_TIME):
            return None
        hours = time.hour - cls.FIRST_START_TIME.hour
        minutes = 60 * hours + time.minute - cls.FIRST_START_TIME.minute
        return int(minutes / 15)

    @property
    def display_name(self):
        """The name used when displaying the scanner on the web page."""
        return "WebScanner '%s'" % self.name

    def __unicode__(self):
        """Return the name of the scanner."""
        return self.name

    def __str__(self):
        """Return the name of the scanner."""
        return self.__unicode__()

    def run(self, test_only=False, blocking=False, user=None):
        """Run a scan with the Scanner.

        Return the Scan object if we started the scanner.
        Return None if there is already a scanner running,
        or if there was a problem running the scanner.
        If test_only is True, only check if we can run a scan, don't actually
        run one.
        """
        if self.is_running:
            return Scanner.ALREADY_RUNNING

        if not self.has_valid_domains:
            return Scanner.NO_VALID_DOMAINS

        # Create a new Scan
        scan = self.create_scan()
        if isinstance(scan, str):
            return scan
        # Add user as recipient on scan
        if user:
            scan.recipients.add(user.profile)
        # Get path to run script
        if isinstance(scan, ExchangeScan):
            SCANNER_DIR = os.path.join(settings.PROJECT_DIR, "scrapy-webscanner/mailscan")
        else:
            SCANNER_DIR = os.path.join(settings.PROJECT_DIR, "scrapy-webscanner")

        if test_only:
            return scan

        if not os.path.exists(scan.scan_log_dir):
            os.makedirs(scan.scan_log_dir)
        log_file = open(scan.scan_log_file, "a")

        if not os.path.exists(scan.scan_output_files_dir):
            os.makedirs(scan.scan_output_files_dir)

        try:
            process = Popen([os.path.join(SCANNER_DIR, "run.sh"),
                             str(scan.pk)], cwd=SCANNER_DIR,
                            stderr=log_file,
                            stdout=log_file)
            if blocking:
                process.communicate()
        except Exception as e:
            print(e)
            return None
        return scan

    class Meta:
        abstract = False
        ordering = ['name']
        db_table = 'os2webscanner_scanner'
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
# encoding: utf-8
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )

import os
from io import StringIO

from django.db import models

from .scan_model import Scan
from .webscanner_model import WebScanner
from .webdomain_model import WebDomain

class WebScan(Scan):

    """An actual instance of the web scanning process done by a web scanner."""

    def __init__(self, *args, **kwargs):
        """Initialize a new scan.

        Stores the old status of the scan for later use.
        """
        super(WebScan, self).__init__(*args, **kwargs)
        self._old_status = self.status

    scanner = models.ForeignKey(WebScanner,
                                null=False, verbose_name='webscanner',
                                related_name='webscans')

    domains = models.ManyToManyField(WebDomain,
                                     verbose_name='Domæner')

    do_link_check = models.BooleanField(default=False,
                                        verbose_name='Tjek links')

    do_external_link_check = models.BooleanField(
        default=False,
        verbose_name='Eksterne links'
    )

    do_last_modified_check_head_request = models.BooleanField(
        default=True,
        verbose_name='Brug HTTP HEAD request'
    )

    do_collect_cookies = models.BooleanField(default=False,
                                             verbose_name='Saml cookies')

    # Create method - copies fields from scanner
    @classmethod
    def create(scan_cls, scanner):
        """ Create and copy fields from scanner. """
        scan = scan_cls(
            is_visible=scanner.is_visible,
            whitelisted_names=scanner.organization.name_whitelist,
            blacklisted_names=scanner.organization.name_blacklist,
            whitelisted_addresses=scanner.organization.address_whitelist,
            blacklisted_addresses=scanner.organization.address_blacklist,
            whitelisted_cprs=scanner.organization.cpr_whitelist,
            do_cpr_scan=scanner.do_cpr_scan,
            do_name_scan=scanner.do_name_scan,
            do_address_scan=scanner.do_address_scan,
            do_ocr=scanner.do_ocr,
            do_cpr_modulus11=scanner.do_cpr_modulus11,
            do_cpr_ignore_irrelevant=scanner.do_cpr_ignore_irrelevant,
            do_link_check=scanner.do_link_check,
            do_external_link_check=scanner.do_external_link_check,
            do_last_modified_check=scanner.do_last_modified_check,
            do_last_modified_check_head_request=scanner.
                do_last_modified_check_head_request,
            do_collect_cookies=scanner.do_collect_cookies,
            columns=scanner.columns,
            output_spreadsheet_file=scanner.output_spreadsheet_file,
            do_cpr_replace=scanner.do_cpr_replace,
            cpr_replace_text=scanner.cpr_replace_text,
            do_name_replace=scanner.do_name_replace,
            name_replace_text=scanner.name_replace_text,
            do_address_replace=scanner.do_address_replace,
            address_replace_text=scanner.address_replace_text
        )
        #
        scan.status = Scan.NEW
        scan.scanner = scanner
        scan.save()
        scan.domains.add(*scanner.domains.all())
        scan.regex_rules.add(*scanner.regex_rules.all())
        scan.recipients.add(*scanner.recipients.all())

        return scan

        # Cookie log - undigested cookie strings for inspection.
    def log_cookie(self, string):
        with open(self.cookie_log_file, "a") as f:
            f.write("{0}\n".format(string))

    @property
    def no_of_cookies(self):
        try:
            with open(self.cookie_log_file, "r") as f:
                return sum(1 for line in f)
        except IOError:
            return 0

    @property
    def cookie_log(self):
        try:
            with open(self.cookie_log_file, "r") as f:
                raw_log = f.read()
        except IOError:
            return ""

        cookie_counts = {}
        lines = raw_log.split('\n')
        for l in lines:
            if len(l) == 0:
                continue
            domain = l.split('|')[0]
            cookie = l.split('|')[1]
            if domain in cookie_counts:
                if cookie in cookie_counts[domain]:
                    cookie_counts[domain][cookie] += 1
                else:
                    cookie_counts[domain][cookie] = 1
            else:
                cookie_counts[domain] = {cookie: 1}

        output = StringIO.StringIO()
        for domain in cookie_counts:
            output.write("{0}\n".format(domain))
            for cookie in cookie_counts[domain]:
                output.write("    {0} Antal: {1}\n".format(
                    cookie,
                    cookie_counts[domain][cookie]
                ))

        result = output.getvalue()

        output.close()
        return result

    @property
    def cookie_log_file(self):
        """Return the file path to this scan's cookie log."""
        return os.path.join(self.scan_log_dir, 'cookie_%s.log' % self.pk)

    @property
    def no_of_broken_links(self):
        """Return the number of broken links for this scan."""
        return self.urls.exclude(status_code__isnull=True).count()
<commit_msg>* super method should be empty in python 3 * During start scan scanner is_running is set to True * has_active_scans method moved from super class Scanner (is deprecated)<commit_after># -*- coding: UTF-8 -*-
# encoding: utf-8
# The contents of this file are subject to the Mozilla Public License
# Version 2.0 (the "License"); you may not use this file except in
# compliance with the License. You may obtain a copy of the License at
#    http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS"basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# OS2Webscanner was developed by Magenta in collaboration with OS2 the
# Danish community of open source municipalities (http://www.os2web.dk/).
#
# The code is currently governed by OS2 the Danish community of open
# source municipalities ( http://www.os2web.dk/ )

import os
from io import StringIO

from django.db import models

from .scan_model import Scan
from .webscanner_model import WebScanner
from .webdomain_model import WebDomain

class WebScan(Scan):

    """An actual instance of the web scanning process done by a web scanner."""

    def __init__(self, *args, **kwargs):
        """Initialize a new scan.

        Stores the old status of the scan for later use.
        """
        super().__init__(*args, **kwargs)
        self._old_status = self.status

    scanner = models.ForeignKey(WebScanner,
                                null=False, verbose_name='webscanner',
                                related_name='webscans')

    domains = models.ManyToManyField(WebDomain,
                                     verbose_name='Domæner')

    do_link_check = models.BooleanField(default=False,
                                        verbose_name='Tjek links')

    do_external_link_check = models.BooleanField(
        default=False,
        verbose_name='Eksterne links'
    )

    do_last_modified_check_head_request = models.BooleanField(
        default=True,
        verbose_name='Brug HTTP HEAD request'
    )

    do_collect_cookies = models.BooleanField(default=False,
                                             verbose_name='Saml cookies')

    # Create method - copies fields from scanner
    @classmethod
    def create(scan_cls, scanner):
        """ Create and copy fields from scanner. """
        scan = scan_cls(
            is_visible=scanner.is_visible,
            whitelisted_names=scanner.organization.name_whitelist,
            blacklisted_names=scanner.organization.name_blacklist,
            whitelisted_addresses=scanner.organization.address_whitelist,
            blacklisted_addresses=scanner.organization.address_blacklist,
            whitelisted_cprs=scanner.organization.cpr_whitelist,
            do_cpr_scan=scanner.do_cpr_scan,
            do_name_scan=scanner.do_name_scan,
            do_address_scan=scanner.do_address_scan,
            do_ocr=scanner.do_ocr,
            do_cpr_modulus11=scanner.do_cpr_modulus11,
            do_cpr_ignore_irrelevant=scanner.do_cpr_ignore_irrelevant,
            do_link_check=scanner.do_link_check,
            do_external_link_check=scanner.do_external_link_check,
            do_last_modified_check=scanner.do_last_modified_check,
            do_last_modified_check_head_request=scanner.
                do_last_modified_check_head_request,
            do_collect_cookies=scanner.do_collect_cookies,
            columns=scanner.columns,
            output_spreadsheet_file=scanner.output_spreadsheet_file,
            do_cpr_replace=scanner.do_cpr_replace,
            cpr_replace_text=scanner.cpr_replace_text,
            do_name_replace=scanner.do_name_replace,
            name_replace_text=scanner.name_replace_text,
            do_address_replace=scanner.do_address_replace,
            address_replace_text=scanner.address_replace_text
        )
        #
        scanner.is_running = True
        scanner.save()
        scan.status = Scan.NEW
        scan.scanner = scanner
        scan.save()
        scan.domains.add(*scanner.domains.all())
        scan.regex_rules.add(*scanner.regex_rules.all())
        scan.recipients.add(*scanner.recipients.all())

        return scan

        # Cookie log - undigested cookie strings for inspection.
    def log_cookie(self, string):
        with open(self.cookie_log_file, "a") as f:
            f.write("{0}\n".format(string))

    @property
    def no_of_cookies(self):
        try:
            with open(self.cookie_log_file, "r") as f:
                return sum(1 for line in f)
        except IOError:
            return 0

    @property
    def cookie_log(self):
        try:
            with open(self.cookie_log_file, "r") as f:
                raw_log = f.read()
        except IOError:
            return ""

        cookie_counts = {}
        lines = raw_log.split('\n')
        for l in lines:
            if len(l) == 0:
                continue
            domain = l.split('|')[0]
            cookie = l.split('|')[1]
            if domain in cookie_counts:
                if cookie in cookie_counts[domain]:
                    cookie_counts[domain][cookie] += 1
                else:
                    cookie_counts[domain][cookie] = 1
            else:
                cookie_counts[domain] = {cookie: 1}

        output = StringIO.StringIO()
        for domain in cookie_counts:
            output.write("{0}\n".format(domain))
            for cookie in cookie_counts[domain]:
                output.write("    {0} Antal: {1}\n".format(
                    cookie,
                    cookie_counts[domain][cookie]
                ))

        result = output.getvalue()

        output.close()
        return result

    @property
    def cookie_log_file(self):
        """Return the file path to this scan's cookie log."""
        return os.path.join(self.scan_log_dir, 'cookie_%s.log' % self.pk)

    @property
    def no_of_broken_links(self):
        """Return the number of broken links for this scan."""
        return self.urls.exclude(status_code__isnull=True).count()

    @property
    def has_active_scans(self, scanner):
        """Whether the scanner has active scans."""
        active_scanners = WebScan.objects.filter(scanner=scanner, status__in=(
                Scan.NEW, Scan.STARTED)).count()
        return active_scanners > 0<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import roslib; roslib.load_manifest('dynamaestro')
import rospy
from dynamaestro.msg import VectorStamped

class LQRController(rospy.Subscriber):
    def __init__(self, intopic, outtopic):
        rospy.Subscriber.__init__(self, outtopic, VectorStamped, self.read_state)
        self.intopic = rospy.Publisher(intopic, VectorStamped, queue_size=1)

    def read_state(self, vs):
        self.intopic.publish(VectorStamped(point=[-(vs.point[0] + 2.4142*vs.point[1] + 2.4142*vs.point[2])]))

if __name__ == "__main__":
    rospy.init_node("lqr", anonymous=True)
    lqrc = LQRController("input", "output")
    rospy.spin()
<commit_msg>Improve LQR example for integrator_chains domain<commit_after>#!/usr/bin/env python
from __future__ import print_function

import roslib; roslib.load_manifest('dynamaestro')
import rospy
from dynamaestro.msg import VectorStamped

from control import lqr
import numpy as np

class StateFeedback(rospy.Subscriber):
    def __init__(self, intopic, outtopic, K=None):
        rospy.Subscriber.__init__(self, outtopic, VectorStamped, self.read_state)
        self.intopic = rospy.Publisher(intopic, VectorStamped, queue_size=1)
        self.K = K

    def read_state(self, vs):
        self.intopic.publish(VectorStamped(point=[-np.dot(self.K, np.asarray(vs.point))]))

class LQRController(StateFeedback):
    def __init__(self, intopic, outtopic,
                 A=None, B=None, Q=None, R=None):
        if A is None and B is None:
            A = np.array([[0., 1, 0],
                          [0,  0, 1],
                          [0,  0, 0]])
            B = np.array([[0.], [0], [1]])
        if Q is None and R is None:
            Q = np.diag([1.,1,1])
            R = np.diag([1.])
        K, S, E = lqr(A,B,Q,R)
        StateFeedback.__init__(self, intopic, outtopic, K)

if __name__ == "__main__":
    rospy.init_node("lqr", anonymous=True)

    n = 1  # Number of output dimensions
    m = 3  # Number of derivatives

    A = np.diag(np.ones(m-1), k=1)
    B = np.zeros((m, 1))
    B[-1,0] = 1.0
    Q = np.diag(np.ones(m))
    R = np.diag([1.])

    lqrc = LQRController("input", "output", A, B, Q, R)
    rospy.spin()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Find pull request from given commit hash and open it in a Web browser.

Depends on: git
Supported Services: GitHub, Bitbucket
"""

import argparse
import os
import re
import subprocess
import sys
import webbrowser

_pull_request_url = {
    'github.com': 'https://github.com/{module}/pull/{number}',
    'bitbucket.org': 'https://bitbucket.org/{module}/pull-requests/{number}',
}

def extract_service_and_module(repo_url):
    """Extract service and module from repository url.

    :param str repo_url: repository url

    :return (service, module)
    :rtype (str, str)
    """
    m = re.match('.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)\.git/?$', repo_url)
    if not m:
        m = re.match('.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)/?$', repo_url)
        if not m:
            raise Exception(
                'cannot detect service and module from {}'.format(repo_url))
    service = m.group(1)
    module = m.group(2)
    if service not in _pull_request_url.keys():
        raise Exception(
            'service not supported: {}'.format(service))
    return (service, module)

def _run_command(args):
    """Run command and get its output.

    :param list[str] args: command and arguments

    :return: output of the command
    :rtype: str

    :raise subprocess.CalledProcessError: command failed
    """
    # don't use subprocess.DEVNULL for python2.7
    with open(os.devnull, 'w') as devnull:
        return subprocess.check_output(
            args, stderr=devnull).strip().decode('utf-8')

def get_remote_url(remote):
    """Get remote repository url of current git repository.

    :param str remote: remote name against which pull requests are created

    :return: remote repository url
    :rtype: str
    """
    args = ['git', 'config', 'remote.{}.url'.format(remote)]
    return _run_command(args)

def extract_pull_request_number(commit_logs):
    """Extract first occurance of pull request number from commit logs.

    :param str commit_logs: oneline commit logs

    :return: pull request number
    :rtype: str
    """
    m = re.search('pull request #(\d+)', commit_logs)
    if not m:
        raise Exception(
            'cannot detect pull request number from\n{}'.format(commit_logs))
    return m.group(1)

def get_default_tracking_branch(remote):
    """Get default remote branch of the current repository.

    detect by remote HEAD if present, or use "master" otherwise

    :param str remote: remote name

    :return: branch name
    :rtype: str
    """
    head = 'remotes/{}/HEAD'.format(remote)
    args = ['git', 'rev-parse', '--abbrev-ref', head]
    try:
        ref = _run_command(args)
        if ref == head:
            raise Exception
        return ref
    except Exception:
        return '{}/master'.format(remote)

def get_pull_request_number(revision, remote, base_branch):
    """Get pull request number from commit messages.

    :param str revision: revision string
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull request are merged

    :return: pull request number
    :rtype: str
    """
    if not re.match('^[0-9a-f]+$', revision):
        raise Exception('invalid revision: {}'.format(revision))
    if not base_branch:
        tracking_branch = get_default_tracking_branch(remote)
    else:
        tracking_branch = '{}/{}'.format(remote, base_branch)
    args = ['git', 'merge-base',
            '--is-ancestor',
            revision,
            tracking_branch]
    try:
        _run_command(args)
    except subprocess.CalledProcessError as e:
        if e.returncode == 1:
            raise Exception(
                '{revision} is not merged to {tracking_branch} yet'.format(
                    **{'revision': revision,
                       'tracking_branch': tracking_branch}))
        raise
    args = ['git', 'log',
            '--merges', '--oneline', '--reverse', '--ancestry-path',
            '{revision}...{tracking_branch}'.format(
                **{'revision': revision, 'tracking_branch': tracking_branch})]
    output = subprocess.check_output(args).decode('utf-8')
    return extract_pull_request_number(output)

def get_pull_request_url(service, module, number):
    """Get pull-request URL.

    :param str service: service name
    :param str module: module path ('owner/repo')
    :param int number: pull-request identification number
    """
    if service not in _pull_request_url:
        raise Exception(
            'service not supported: {}'.format(service))
    url = _pull_request_url[service]
    return url.format(**{'module': module, 'number': number})

def openpr(revision, base_branch, remote, print_only):
    """Find pull request from given commit hash and open it in a Web browser.

    :param str revision: revision string of the target commit
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull requests are merged
    :param bool print_only: print pull request url instead of opening it
    """
    number = get_pull_request_number(revision, remote, base_branch)
    repo_url = get_remote_url(remote)
    (service, module) = extract_service_and_module(repo_url)
    pr_url = get_pull_request_url(service, module, number)
    if print_only:
        print(pr_url)
    else:
        webbrowser.open(pr_url)

def main():
    """Parse command line arguments and call openpr()."""
    parser = argparse.ArgumentParser(
        description='Find pull request from given commit hash and open it in a Web browser.')
    parser.add_argument(
        'revision',
        help='revision string of the target commit')
    parser.add_argument(
        '-b', '--base-branch',
        metavar='BRANCH',
        default='',
        help=('branch against which pull requests are merged (default: detect by remote HEAD if '
              'present, or use "master" otherwise'))
    parser.add_argument(
        '-r', '--remote',
        default='origin',
        help='remote name against which pull requests are created (default: origin)')
    parser.add_argument(
        '-p', '--print-only',
        action='store_true',
        help='print pull request url instead of opening it')
    args = parser.parse_args()
    try:
        openpr(args.revision, args.base_branch, args.remote, args.print_only)
        sys.exit(0)
    except Exception as e:
        print(e)
        sys.exit(-1)

if __name__ == '__main__':
    main()
<commit_msg>use r'...'<commit_after>#!/usr/bin/env python
"""Find pull request from given commit hash and open it in a Web browser.

Depends on: git
Supported Services: GitHub, Bitbucket
"""

import argparse
import os
import re
import subprocess
import sys
import webbrowser

_pull_request_url = {
    'github.com': 'https://github.com/{module}/pull/{number}',
    'bitbucket.org': 'https://bitbucket.org/{module}/pull-requests/{number}',
}

def extract_service_and_module(repo_url):
    """Extract service and module from repository url.

    :param str repo_url: repository url

    :return (service, module)
    :rtype (str, str)
    """
    m = re.match(r'.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)\.git/?$', repo_url)
    if not m:
        m = re.match(r'.+[/@]([^\.]+\.[^\.]+)[:/]([^/]+/[^/]+)/?$', repo_url)
        if not m:
            raise Exception(
                'cannot detect service and module from {}'.format(repo_url))
    service = m.group(1)
    module = m.group(2)
    if service not in _pull_request_url.keys():
        raise Exception(
            'service not supported: {}'.format(service))
    return (service, module)

def _run_command(args):
    """Run command and get its output.

    :param list[str] args: command and arguments

    :return: output of the command
    :rtype: str

    :raise subprocess.CalledProcessError: command failed
    """
    # don't use subprocess.DEVNULL for python2.7
    with open(os.devnull, 'w') as devnull:
        return subprocess.check_output(
            args, stderr=devnull).strip().decode('utf-8')

def get_remote_url(remote):
    """Get remote repository url of current git repository.

    :param str remote: remote name against which pull requests are created

    :return: remote repository url
    :rtype: str
    """
    args = ['git', 'config', 'remote.{}.url'.format(remote)]
    return _run_command(args)

def extract_pull_request_number(commit_logs):
    """Extract first occurance of pull request number from commit logs.

    :param str commit_logs: oneline commit logs

    :return: pull request number
    :rtype: str
    """
    m = re.search(r'pull request #(\d+)', commit_logs)
    if not m:
        raise Exception(
            'cannot detect pull request number from\n{}'.format(commit_logs))
    return m.group(1)

def get_default_tracking_branch(remote):
    """Get default remote branch of the current repository.

    detect by remote HEAD if present, or use "master" otherwise

    :param str remote: remote name

    :return: branch name
    :rtype: str
    """
    head = 'remotes/{}/HEAD'.format(remote)
    args = ['git', 'rev-parse', '--abbrev-ref', head]
    try:
        ref = _run_command(args)
        if ref == head:
            raise Exception
        return ref
    except Exception:
        return '{}/master'.format(remote)

def get_pull_request_number(revision, remote, base_branch):
    """Get pull request number from commit messages.

    :param str revision: revision string
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull request are merged

    :return: pull request number
    :rtype: str
    """
    if not re.match(r'^[0-9a-f]+$', revision):
        raise Exception('invalid revision: {}'.format(revision))
    if not base_branch:
        tracking_branch = get_default_tracking_branch(remote)
    else:
        tracking_branch = '{}/{}'.format(remote, base_branch)
    args = ['git', 'merge-base',
            '--is-ancestor',
            revision,
            tracking_branch]
    try:
        _run_command(args)
    except subprocess.CalledProcessError as e:
        if e.returncode == 1:
            raise Exception(
                '{revision} is not merged to {tracking_branch} yet'.format(
                    **{'revision': revision,
                       'tracking_branch': tracking_branch}))
        raise
    args = ['git', 'log',
            '--merges', '--oneline', '--reverse', '--ancestry-path',
            '{revision}...{tracking_branch}'.format(
                **{'revision': revision, 'tracking_branch': tracking_branch})]
    output = subprocess.check_output(args).decode('utf-8')
    return extract_pull_request_number(output)

def get_pull_request_url(service, module, number):
    """Get pull-request URL.

    :param str service: service name
    :param str module: module path ('owner/repo')
    :param int number: pull-request identification number
    """
    if service not in _pull_request_url:
        raise Exception(
            'service not supported: {}'.format(service))
    url = _pull_request_url[service]
    return url.format(**{'module': module, 'number': number})

def openpr(revision, base_branch, remote, print_only):
    """Find pull request from given commit hash and open it in a Web browser.

    :param str revision: revision string of the target commit
    :param str remote: remote name against which pull requests are created
    :param str base_branch: branch against which pull requests are merged
    :param bool print_only: print pull request url instead of opening it
    """
    number = get_pull_request_number(revision, remote, base_branch)
    repo_url = get_remote_url(remote)
    (service, module) = extract_service_and_module(repo_url)
    pr_url = get_pull_request_url(service, module, number)
    if print_only:
        print(pr_url)
    else:
        webbrowser.open(pr_url)

def main():
    """Parse command line arguments and call openpr()."""
    parser = argparse.ArgumentParser(
        description='Find pull request from given commit hash and open it in a Web browser.')
    parser.add_argument(
        'revision',
        help='revision string of the target commit')
    parser.add_argument(
        '-b', '--base-branch',
        metavar='BRANCH',
        default='',
        help=('branch against which pull requests are merged (default: detect by remote HEAD if '
              'present, or use "master" otherwise'))
    parser.add_argument(
        '-r', '--remote',
        default='origin',
        help='remote name against which pull requests are created (default: origin)')
    parser.add_argument(
        '-p', '--print-only',
        action='store_true',
        help='print pull request url instead of opening it')
    args = parser.parse_args()
    try:
        openpr(args.revision, args.base_branch, args.remote, args.print_only)
        sys.exit(0)
    except Exception as e:
        print(e)
        sys.exit(-1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import subprocess
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

browserify_bin = os.path.join(settings.EN_PROJECT_PATH,
                              'node_modules', '.bin', 'browserify')

thisdir = os.path.dirname(__file__)
base_index = os.path.join(thisdir, '..', '..', 'js', 'index-base.js')
admin_index = os.path.join(thisdir, '..', '..', 'js', 'index-base.js')

class Command(BaseCommand):
    help = 'Create browserify bundles.'
    option_list = BaseCommand.option_list + (
        make_option('-f',
                    '--fast',
                    action='store_true',
                    help='Speed up browserify compilation time.'),
    )
    def handle(self, *args, **options):

        if args:
            raise CommandError('This command does not take any args.')

        if not os.path.exists(browserify_bin):
            raise CommandError('Browserify not installed. Run `npm install` '
                               'from the base directory.')

        browserify_args = []

        if options.get('fast', False):
            browserify_args += []

        subprocess.call([browserify_bin] + browserify_args + [
            base_index, '-o',
            os.path.join(settings.STATIC_ROOT, 'base-bundle.js')
        ])

        subprocess.call([browserify_bin] + browserify_args + [
            admin_index, '-o',
            os.path.join(settings.STATIC_ROOT, 'admin-bundle.js')
        ])
<commit_msg>Fix typo in browserify compile command<commit_after>import os
import subprocess
from optparse import make_option

from django.conf import settings
from django.core.management.base import BaseCommand, CommandError

browserify_bin = os.path.join(settings.EN_PROJECT_PATH,
                              'node_modules', '.bin', 'browserify')

thisdir = os.path.dirname(__file__)
base_index = os.path.join(thisdir, '..', '..', 'js', 'index-base.js')
admin_index = os.path.join(thisdir, '..', '..', 'js', 'index-admin.js')

class Command(BaseCommand):
    help = 'Create browserify bundles.'
    option_list = BaseCommand.option_list + (
        make_option('-f',
                    '--fast',
                    action='store_true',
                    help='Speed up browserify compilation time.'),
    )
    def handle(self, *args, **options):

        if args:
            raise CommandError('This command does not take any args.')

        if not os.path.exists(browserify_bin):
            raise CommandError('Browserify not installed. Run `npm install` '
                               'from the base directory.')

        browserify_args = []

        if options.get('fast', False):
            browserify_args += []

        subprocess.call([browserify_bin] + browserify_args + [
            base_index, '-o',
            os.path.join(settings.STATIC_ROOT, 'base-bundle.js')
        ])

        subprocess.call([browserify_bin] + browserify_args + [
            admin_index, '-o',
            os.path.join(settings.STATIC_ROOT, 'admin-bundle.js')
        ])
<|endoftext|>