<commit_before>"""
Copyright (c) 2016 The Hyve B.V.
This code is licensed under the GNU Affero General Public License (AGPL),
version 3, or (at your option) any later version.
"""

import unittest
import logging
import hugoEntrezMap
from import_data_validator import validateData

# globals:
hugo_mapping_file = 'test_data/Homo_sapiens.gene_info.gz'
ncbi_file = open(hugo_mapping_file)
hugo_entrez_map = hugoEntrezMap.parse_ncbi_file(ncbi_file)
# hard-code known clinical attributes
KNOWN_PATIENT_ATTRS = {
    "PATIENT_ID": {"display_name":"Patient Identifier","description":"Identifier to uniquely specify a patient.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "OS_STATUS": {"display_name":"Overall Survival Status","description":"Overall patient survival status.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "OS_MONTHS": {"display_name":"Overall Survival (Months)","description":"Overall survival in months since initial diagnosis.","datatype":"NUMBER","is_patient_attribute":"1","priority":"1"},
    "DFS_STATUS": {"display_name":"Disease Free Status","description":"Disease free status since initial treatment.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "DFS_MONTHS": {"display_name":"Disease Free (Months)","description":"Disease free (months) since initial treatment.","datatype":"NUMBER","is_patient_attribute":"1","priority":"1"},
    "SUBTYPE": {"display_name":"Subtype","description":"Subtype description.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
    "CANCER_TYPE": {"display_name":"Cancer Type","description":"Disease type.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
    "CANCER_TYPE_DETAILED": {"display_name":"Cancer Type Detailed","description":"Cancer Type Detailed.","datatype":"STRING","is_patient_attribute":"0","priority":"1"}
}
KNOWN_SAMPLE_ATTRS = {
    "SAMPLE_ID": {"display_name":"Sample Identifier","description":"A unique sample identifier.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
}

# hard-code known cancer types
KNOWN_CANCER_TYPES = {
    "brca": {"name":"Invasive Breast Carcinoma","color":"HotPink"},
    "prad": {"name":"Prostate Adenocarcinoma","color":"Cyan"}
}

# mock-code sample ids defined in a study
DEFINED_SAMPLE_IDS = ["TCGA-A1-A0SB-01", "TCGA-A1-A0SD-01", "TCGA-A1-A0SE-01", "TCGA-A1-A0SH-01", "TCGA-A2-A04U-01"]

# TODO - something like this could be done for a web-services stub:
# def dumy_request_from_portal_api(service_url, logger):
#     """Send a request to the portal API and return the decoded JSON object."""
#     if logger.isEnabledFor(logging.INFO):
#         url_split = service_url.split('/api/', 1)
#         logger.info("Requesting %s from portal at '%s'",
#                     url_split[1], url_split[0])
#     response = requests.get(service_url)
#     try:
#         response.raise_for_status()
#     except requests.exceptions.HTTPError as e:
#         raise IOError(
#             'Connection error for URL: {url}. Administrator: please check if '
#             '[{url}] is accessible. Message: {msg}'.format(url=service_url,
#                                                            msg=e.message))
#     return response.json()
#
# validateData.request_from_portal_api = dumy_request_from_portal_api

class LogBufferTestCase(unittest.TestCase):

    """Superclass for testcases that want to capture log records emitted.

    Defines a self.logger to log to, and a method get_log_records() to
    collect the list of LogRecords emitted by this logger. In addition,
    defines a function print_log_records to format a list of LogRecords
    to standard output.
    """

    def setUp(self):
        """Set up a logger with a buffering handler."""
        self.logger = logging.getLogger(self.__class__.__name__)
        self.logger.setLevel(logging.INFO)
        handler = logging.handlers.BufferingHandler(capacity=1e6)
        self.orig_handlers = self.logger.handlers
        self.logger.handlers = [handler]

    def tearDown(self):
        """Remove the logger handler (and any buffer it may have)."""
        self.logger.handlers = self.orig_handlers

    def get_log_records(self):
        """Get the log records written to the logger since the last call."""
        recs = self.logger.handlers[0].buffer
        self.logger.handlers[0].flush()
        return recs

    @staticmethod
    def print_log_records(record_list):
        """Pretty-print a list of log records to standard output.

        This can be used if, while writing unit tests, you want to see
        what the messages currently are. The final unit tests committed
        to version control should not print log messages.
        """
        formatter = validateData.LogfileStyleFormatter()
        for record in record_list:
            print formatter.format(record)

class DataFileTestCase(LogBufferTestCase):

    """Superclass for testcases validating a particular data file.

    Provides a validate() method to validate the data file with a
    particular validator class and collect the log records emitted.
    """

    def setUp(self):
        """Set up for validating a file in the test_data directory."""
        super(DataFileTestCase, self).setUp()
        self.orig_study_dir = validateData.STUDY_DIR
        validateData.STUDY_DIR = 'test_data'
        # hard-code known clinical attributes instead of contacting a portal
        self.orig_srv_attrs = validateData.ClinicalValidator.srv_attrs
        mock_srv_attrs = dict(KNOWN_PATIENT_ATTRS)
        mock_srv_attrs.update(KNOWN_SAMPLE_ATTRS)
        validateData.ClinicalValidator.srv_attrs = mock_srv_attrs

    def tearDown(self):
        """Restore the environment to before setUp() was called."""
        validateData.STUDY_DIR = self.orig_study_dir
        validateData.ClinicalValidator.srv_attrs = self.orig_srv_attrs
        super(DataFileTestCase, self).tearDown()

    def validate(self, data_filename, validator_class, extra_meta_fields=None):
        """Validate a file with a Validator and return the log records."""
        meta_dict = {'data_file_path': data_filename}
        if extra_meta_fields is not None:
            meta_dict.update(extra_meta_fields)
        validator = validator_class(hugo_entrez_map, self.logger, meta_dict)
        validator.validate()
        return self.get_log_records()

class PostClinicalDataFileTestCase(DataFileTestCase):

    """Superclass for validating data files to be read after clinical files.

    I.e. DEFINED_SAMPLE_IDS will be initialised with a list of sample
    identifiers defined in the study.
    """

    def setUp(self):
        """Prepare for validating a file by setting the samples defined."""
        super(PostClinicalDataFileTestCase, self).setUp()
        self.orig_defined_sample_ids = validateData.DEFINED_SAMPLE_IDS
        validateData.DEFINED_SAMPLE_IDS = DEFINED_SAMPLE_IDS

    def tearDown(self):
        """Restore the environment to before setUp() was called."""
        validateData.DEFINED_SAMPLE_IDS = self.orig_defined_sample_ids
        super(PostClinicalDataFileTestCase, self).tearDown()

# ----------------------------------------------------------------------------
# Test cases for the various Validator classes found in validateData script

class ColumnOrderTestCase(DataFileTestCase):

    """Test if column order requirements are appropriately validated."""

    def test_column_order_validation_SegValidator(self):
        """seg validator needs its columns in a specific order.

        Here we serve a file with wrong order and expect validator to log this:
        """
        # set level according to this test case:
        self.logger.setLevel(logging.ERROR)
        record_list = self.validate('data_seg_wrong_order.txt',
                                    validateData.SegValidator)
        # we expect 2 errors about columns in wrong order,
        # and one about the file not being parseable:
        self.assertEqual(len(record_list), 3)
        # check if both messages come from checkOrderedRequiredColumns:
        for error in record_list[:2]:
            self.assertEqual("ERROR", error.levelname)
            self.assertEqual("_checkOrderedRequiredColumns", error.funcName)

    def test_column_order_validation_ClinicalValidator(self):
        """ClinicalValidator does NOT need its columns in a specific order.

        Here we serve files with different order and no errors or warnings
        """
        # set level according to this test case:
        self.logger.setLevel(logging.WARNING)
        record_list = self.validate('data_clin_order1.txt',
                                    validateData.ClinicalValidator)
        # we expect no errors or warnings
        self.assertEqual(0, len(record_list))
        # if the file has another order, this is also OK:
        record_list = self.validate('data_clin_order2.txt',
                                    validateData.ClinicalValidator)
        # again, we expect no errors or warnings
        self.assertEqual(0, len(record_list))

class ClinicalColumnDefsTestCase(DataFileTestCase):

    """Tests for validations of the column definitions in a clinical file."""

    def test_correct_definitions(self):
        """Test when all record definitions match with portal."""
        record_list = self.validate('data_clin_coldefs_correct.txt',
                                    validateData.ClinicalValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_wrong_definitions(self):
        """Test when record definitions do not match with portal."""
        record_list = self.validate('data_clin_coldefs_wrong_display_name.txt',
                                    validateData.ClinicalValidator)
        # expecting an info message followed by the error, and another error as
        # the rest of the file cannot be parsed
        self.assertEqual(len(record_list), 3)
        # error about the display name of OS_MONTHS
        self.assertEqual(record_list[1].levelno, logging.ERROR)
        self.assertEqual(record_list[1].column_number, 3)
        self.assertIn('display_name', record_list[1].getMessage().lower())

    def test_unknown_attribute(self):
        """Test when a new attribute is defined in the data file."""
        record_list = self.validate('data_clin_coldefs_unknown_attribute.txt',
                                    validateData.ClinicalValidator)
        # expecting two info messages with a warning in between
        self.assertEqual(len(record_list), 3)
        self.assertEqual(record_list[1].levelno, logging.WARNING)
        self.assertEqual(record_list[1].column_number, 7)
        self.assertIn('will be added', record_list[1].getMessage().lower())

    def test_invalid_definitions(self):
        """Test when new attributes are defined with invalid properties."""
        record_list = self.validate('data_clin_coldefs_invalid_priority.txt',
                                    validateData.ClinicalValidator)
        # expecting an info message followed by the error, and another error as
        # the rest of the file cannot be parsed
        self.assertEqual(len(record_list), 3)
        # error about the non-numeric priority of the SAUSAGE column
        self.assertEqual(record_list[1].levelno, logging.ERROR)
        self.assertEqual(record_list[1].line_number, 5)
        self.assertEqual(record_list[1].column_number, 7)

class CancerTypeValidationTestCase(LogBufferTestCase):

    """Tests for validations of cancer type meta files in a study."""

    def test_new_cancer_type(self):
        """Test when a study defines a new cancer type."""
        #{"id":"luad","name":"Lung Adenocarcinoma","color":"Gainsboro"}
        pass # TODO

    def test_cancer_type_file_format_error(self):
        """Test when a cancer type file does not make sense."""
        pass  # TODO

    def test_cancer_type_matching_portal(self):
        """Test when an existing cancer type is defined exactly as known."""
        pass  # TODO

    def test_cancer_type_disagreeing_with_portal(self):
        """Test when an existing cancer type is redefined by a study."""
        pass  # TODO

    def test_cancer_type_defined_twice(self):
        """Test when a study defines the same cancer type id twice.

        No difference is assumed between matching and disagreeing definitions;
        this validation just fails as it never makes sense to do this.
        """
        pass  # TODO

class GeneIdColumnPresenceTestCase(PostClinicalDataFileTestCase):

    """Tests validating gene-wise files with different combinations of gene id columns."""

    def test_both_name_and_entrez(self):
        """Test when a file has both the Hugo name and Entrez ID columns."""
        record_list = self.validate('data_cna_genecol_presence_both.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_name_only(self):
        """Test when a file has a Hugo name column but none for Entrez IDs."""
        record_list = self.validate('data_cna_genecol_presence_hugo_only.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_entrez_only(self):
        """Test when a file has an Entrez ID column but none for Hugo names."""
        record_list = self.validate('data_cna_genecol_presence_entrez_only.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_neither_name_nor_entrez(self):
        """Test when a file lacks both the Entrez ID and Hugo name columns."""
        record_list = self.validate('data_cna_genecol_presence_neither.txt',
                                    validateData.CNAValidator)
        # two errors after the info: the first makes the file unparsable
        self.assertEqual(len(record_list), 3)
        for record in record_list[1:]:
            self.assertEqual(record.levelno, logging.ERROR)
        self.assertEqual(record_list[1].line_number, 1)
        pass  # TODO
<commit_msg>Add unit tests for cancer type meta file validations<commit_after>"""
Copyright (c) 2016 The Hyve B.V.
This code is licensed under the GNU Affero General Public License (AGPL),
version 3, or (at your option) any later version.
"""

import unittest
import logging
import hugoEntrezMap
from import_data_validator import validateData

# globals:
hugo_mapping_file = 'test_data/Homo_sapiens.gene_info.gz'
ncbi_file = open(hugo_mapping_file)
hugo_entrez_map = hugoEntrezMap.parse_ncbi_file(ncbi_file)
# hard-code known clinical attributes
KNOWN_PATIENT_ATTRS = {
    "PATIENT_ID": {"display_name":"Patient Identifier","description":"Identifier to uniquely specify a patient.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "OS_STATUS": {"display_name":"Overall Survival Status","description":"Overall patient survival status.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "OS_MONTHS": {"display_name":"Overall Survival (Months)","description":"Overall survival in months since initial diagnosis.","datatype":"NUMBER","is_patient_attribute":"1","priority":"1"},
    "DFS_STATUS": {"display_name":"Disease Free Status","description":"Disease free status since initial treatment.","datatype":"STRING","is_patient_attribute":"1","priority":"1"},
    "DFS_MONTHS": {"display_name":"Disease Free (Months)","description":"Disease free (months) since initial treatment.","datatype":"NUMBER","is_patient_attribute":"1","priority":"1"},
    "SUBTYPE": {"display_name":"Subtype","description":"Subtype description.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
    "CANCER_TYPE": {"display_name":"Cancer Type","description":"Disease type.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
    "CANCER_TYPE_DETAILED": {"display_name":"Cancer Type Detailed","description":"Cancer Type Detailed.","datatype":"STRING","is_patient_attribute":"0","priority":"1"}
}
KNOWN_SAMPLE_ATTRS = {
    "SAMPLE_ID": {"display_name":"Sample Identifier","description":"A unique sample identifier.","datatype":"STRING","is_patient_attribute":"0","priority":"1"},
}

# hard-code known cancer types
KNOWN_CANCER_TYPES = {
    "brca": {"name":"Invasive Breast Carcinoma","color":"HotPink"},
    "prad": {"name":"Prostate Adenocarcinoma","color":"Cyan"}
}

# mock-code sample ids defined in a study
DEFINED_SAMPLE_IDS = ["TCGA-A1-A0SB-01", "TCGA-A1-A0SD-01", "TCGA-A1-A0SE-01", "TCGA-A1-A0SH-01", "TCGA-A2-A04U-01"]

# TODO - something like this could be done for a web-services stub:
# def dumy_request_from_portal_api(service_url, logger):
#     """Send a request to the portal API and return the decoded JSON object."""
#     if logger.isEnabledFor(logging.INFO):
#         url_split = service_url.split('/api/', 1)
#         logger.info("Requesting %s from portal at '%s'",
#                     url_split[1], url_split[0])
#     response = requests.get(service_url)
#     try:
#         response.raise_for_status()
#     except requests.exceptions.HTTPError as e:
#         raise IOError(
#             'Connection error for URL: {url}. Administrator: please check if '
#             '[{url}] is accessible. Message: {msg}'.format(url=service_url,
#                                                            msg=e.message))
#     return response.json()
#
# validateData.request_from_portal_api = dumy_request_from_portal_api

class LogBufferTestCase(unittest.TestCase):

    """Superclass for testcases that want to capture log records emitted.

    Defines a self.logger to log to, and a method get_log_records() to
    collect the list of LogRecords emitted by this logger. In addition,
    defines a function print_log_records to format a list of LogRecords
    to standard output.
    """

    def setUp(self):
        """Set up a logger with a buffering handler."""
        self.logger = logging.getLogger(self.__class__.__name__)
        self.logger.setLevel(logging.INFO)
        handler = logging.handlers.BufferingHandler(capacity=1e6)
        self.orig_handlers = self.logger.handlers
        self.logger.handlers = [handler]

    def tearDown(self):
        """Remove the logger handler (and any buffer it may have)."""
        self.logger.handlers = self.orig_handlers

    def get_log_records(self):
        """Get the log records written to the logger since the last call."""
        recs = self.logger.handlers[0].buffer
        self.logger.handlers[0].flush()
        return recs

    @staticmethod
    def print_log_records(record_list):
        """Pretty-print a list of log records to standard output.

        This can be used if, while writing unit tests, you want to see
        what the messages currently are. The final unit tests committed
        to version control should not print log messages.
        """
        formatter = validateData.LogfileStyleFormatter()
        for record in record_list:
            print formatter.format(record)

class DataFileTestCase(LogBufferTestCase):

    """Superclass for testcases validating a particular data file.

    Provides a validate() method to validate the data file with a
    particular validator class and collect the log records emitted.
    """

    def setUp(self):
        """Set up for validating a file in the test_data directory."""
        super(DataFileTestCase, self).setUp()
        self.orig_study_dir = validateData.STUDY_DIR
        validateData.STUDY_DIR = 'test_data'
        # hard-code known clinical attributes instead of contacting a portal
        self.orig_srv_attrs = validateData.ClinicalValidator.srv_attrs
        mock_srv_attrs = dict(KNOWN_PATIENT_ATTRS)
        mock_srv_attrs.update(KNOWN_SAMPLE_ATTRS)
        validateData.ClinicalValidator.srv_attrs = mock_srv_attrs

    def tearDown(self):
        """Restore the environment to before setUp() was called."""
        validateData.STUDY_DIR = self.orig_study_dir
        validateData.ClinicalValidator.srv_attrs = self.orig_srv_attrs
        super(DataFileTestCase, self).tearDown()

    def validate(self, data_filename, validator_class, extra_meta_fields=None):
        """Validate a file with a Validator and return the log records."""
        meta_dict = {'data_file_path': data_filename}
        if extra_meta_fields is not None:
            meta_dict.update(extra_meta_fields)
        validator = validator_class(hugo_entrez_map, self.logger, meta_dict)
        validator.validate()
        return self.get_log_records()

class PostClinicalDataFileTestCase(DataFileTestCase):

    """Superclass for validating data files to be read after clinical files.

    I.e. DEFINED_SAMPLE_IDS will be initialised with a list of sample
    identifiers defined in the study.
    """

    def setUp(self):
        """Prepare for validating a file by setting the samples defined."""
        super(PostClinicalDataFileTestCase, self).setUp()
        self.orig_defined_sample_ids = validateData.DEFINED_SAMPLE_IDS
        validateData.DEFINED_SAMPLE_IDS = DEFINED_SAMPLE_IDS

    def tearDown(self):
        """Restore the environment to before setUp() was called."""
        validateData.DEFINED_SAMPLE_IDS = self.orig_defined_sample_ids
        super(PostClinicalDataFileTestCase, self).tearDown()

# ----------------------------------------------------------------------------
# Test cases for the various Validator classes found in validateData script

class ColumnOrderTestCase(DataFileTestCase):

    """Test if column order requirements are appropriately validated."""

    def test_column_order_validation_SegValidator(self):
        """seg validator needs its columns in a specific order.

        Here we serve a file with wrong order and expect validator to log this:
        """
        # set level according to this test case:
        self.logger.setLevel(logging.ERROR)
        record_list = self.validate('data_seg_wrong_order.txt',
                                    validateData.SegValidator)
        # we expect 2 errors about columns in wrong order,
        # and one about the file not being parseable:
        self.assertEqual(len(record_list), 3)
        # check if both messages come from checkOrderedRequiredColumns:
        for error in record_list[:2]:
            self.assertEqual("ERROR", error.levelname)
            self.assertEqual("_checkOrderedRequiredColumns", error.funcName)

    def test_column_order_validation_ClinicalValidator(self):
        """ClinicalValidator does NOT need its columns in a specific order.

        Here we serve files with different order and no errors or warnings
        """
        # set level according to this test case:
        self.logger.setLevel(logging.WARNING)
        record_list = self.validate('data_clin_order1.txt',
                                    validateData.ClinicalValidator)
        # we expect no errors or warnings
        self.assertEqual(0, len(record_list))
        # if the file has another order, this is also OK:
        record_list = self.validate('data_clin_order2.txt',
                                    validateData.ClinicalValidator)
        # again, we expect no errors or warnings
        self.assertEqual(0, len(record_list))

class ClinicalColumnDefsTestCase(DataFileTestCase):

    """Tests for validations of the column definitions in a clinical file."""

    def test_correct_definitions(self):
        """Test when all record definitions match with portal."""
        record_list = self.validate('data_clin_coldefs_correct.txt',
                                    validateData.ClinicalValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_wrong_definitions(self):
        """Test when record definitions do not match with portal."""
        record_list = self.validate('data_clin_coldefs_wrong_display_name.txt',
                                    validateData.ClinicalValidator)
        # expecting an info message followed by the error, and another error as
        # the rest of the file cannot be parsed
        self.assertEqual(len(record_list), 3)
        # error about the display name of OS_MONTHS
        self.assertEqual(record_list[1].levelno, logging.ERROR)
        self.assertEqual(record_list[1].column_number, 3)
        self.assertIn('display_name', record_list[1].getMessage().lower())

    def test_unknown_attribute(self):
        """Test when a new attribute is defined in the data file."""
        record_list = self.validate('data_clin_coldefs_unknown_attribute.txt',
                                    validateData.ClinicalValidator)
        # expecting two info messages with a warning in between
        self.assertEqual(len(record_list), 3)
        self.assertEqual(record_list[1].levelno, logging.WARNING)
        self.assertEqual(record_list[1].column_number, 7)
        self.assertIn('will be added', record_list[1].getMessage().lower())

    def test_invalid_definitions(self):
        """Test when new attributes are defined with invalid properties."""
        record_list = self.validate('data_clin_coldefs_invalid_priority.txt',
                                    validateData.ClinicalValidator)
        # expecting an info message followed by the error, and another error as
        # the rest of the file cannot be parsed
        self.assertEqual(len(record_list), 3)
        # error about the non-numeric priority of the SAUSAGE column
        self.assertEqual(record_list[1].levelno, logging.ERROR)
        self.assertEqual(record_list[1].line_number, 5)
        self.assertEqual(record_list[1].column_number, 7)

class CancerTypeValidationTestCase(LogBufferTestCase):

    """Tests for validations of cancer type meta files in a study."""

    def setUp(self):
        """Initialize environment, hard-coding known cancer types."""
        super(CancerTypeValidationTestCase, self).setUp()
        self.orig_portal_cancer_types = validateData.PORTAL_CANCER_TYPES
        validateData.PORTAL_CANCER_TYPES = KNOWN_CANCER_TYPES

    def tearDown(self):
        """Restore the environment to before setUp() was called."""
        super(CancerTypeValidationTestCase, self).tearDown()
        validateData.PORTAL_CANCER_TYPES = self.orig_portal_cancer_types

    def test_new_cancer_type(self):
        """Test when a study defines a new cancer type."""
        # {"id":"luad","name":"Lung Adenocarcinoma","color":"Gainsboro"}
        validateData.process_metadata_files(
            'test_data/study_metacancertype_lung', self.logger)
        record_list = self.get_log_records()
        # expecting a warning about a new cancer type being added
        self.assertEqual(len(record_list), 1)
        self.assertEqual(record_list[0].levelno, logging.WARNING)
        self.assertEqual(record_list[0].cause, 'luad')

    def test_cancer_type_file_format_error(self):
        """Test when a new cancer type file does not make sense."""
        self.logger.setLevel(logging.ERROR)
        validateData.process_metadata_files(
            'test_data/study_metacancertype_missing_color', self.logger)
        record_list = self.get_log_records()
        # expecting two errors: one about the missing field and one about the
        # undefined cancer type of the study
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.ERROR)
        self.assertEqual(record_list[0].data_filename, 'meta_cancer_type.txt')
        self.assertIn('dedicated_color', record_list[0].getMessage())
        self.assertEqual(record_list[1].cause, 'luad')

    def test_cancer_type_matching_portal(self):
        """Test when an existing cancer type is defined exactly as known."""
        validateData.process_metadata_files(
            'test_data/study_metacancertype_confirming_existing', self.logger)
        record_list = self.get_log_records()
        # expecting no messages, warnings or errors
        self.assertEqual(len(record_list), 0)

    def test_cancer_type_disagreeing_with_portal(self):
        """Test when an existing cancer type is redefined by a study."""
        validateData.process_metadata_files(
            'test_data/study_metacancertype_redefining', self.logger)
        record_list = self.get_log_records()
        # expecting an error message about the cancer type file, but none about
        # the rest of the study as the portal defines a valid cancer type
        self.assertEqual(len(record_list), 1)
        record = record_list.pop()
        self.assertEqual(record.levelno, logging.ERROR)
        self.assertEqual(record.data_filename, 'meta_cancer_type.txt')
        self.assertEqual(record.cause, 'Breast Cancer')

    def test_cancer_type_defined_twice(self):
        """Test when a study defines the same cancer type id twice.

        No difference is assumed between matching and disagreeing definitions;
        this validation just fails as it never makes sense to do this.
        """
        self.logger.setLevel(logging.ERROR)
        validateData.process_metadata_files(
            'test_data/study_metacancertype_lung_twice', self.logger)
        record_list = self.get_log_records()
        # expecting an error message about the doubly-defined cancer type
        self.assertEqual(len(record_list), 1)
        record = record_list.pop()
        self.assertEqual(record.levelno, logging.ERROR)
        self.assertEqual(record.cause, 'luad')
        self.assertIn('defined a second time', record.getMessage())

class GeneIdColumnPresenceTestCase(PostClinicalDataFileTestCase):

    """Tests validating gene-wise files with different combinations of gene id columns."""

    def test_both_name_and_entrez(self):
        """Test when a file has both the Hugo name and Entrez ID columns."""
        record_list = self.validate('data_cna_genecol_presence_both.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_name_only(self):
        """Test when a file has a Hugo name column but none for Entrez IDs."""
        record_list = self.validate('data_cna_genecol_presence_hugo_only.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_entrez_only(self):
        """Test when a file has an Entrez ID column but none for Hugo names."""
        record_list = self.validate('data_cna_genecol_presence_entrez_only.txt',
                                    validateData.CNAValidator)
        # expecting two info messages: at start and end of file
        self.assertEqual(len(record_list), 2)
        for record in record_list:
            self.assertEqual(record.levelno, logging.INFO)

    def test_neither_name_nor_entrez(self):
        """Test when a file lacks both the Entrez ID and Hugo name columns."""
        record_list = self.validate('data_cna_genecol_presence_neither.txt',
                                    validateData.CNAValidator)
        # two errors after the info: the first makes the file unparsable
        self.assertEqual(len(record_list), 3)
        for record in record_list[1:]:
            self.assertEqual(record.levelno, logging.ERROR)
        self.assertEqual(record_list[1].line_number, 1)
<|endoftext|>


<commit_before>import pandas as pd
import numpy as np
import ForecastEngine as autof
import Bench.TS_datasets as tsds

import logging
import logging.config

#logging.config.fileConfig('logging.conf')

logging.basicConfig(level=logging.INFO)

#get_ipython().magic('matplotlib inline')

b1 = tsds.load_ozone()
df = b1.mPastData

df.head()

lEngine = autof.cForecastEngine()
lEngine

H = b1.mHorizon;
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True;
lEngine.mOptions.mParallelMode = True;
lEngine.mOptions.set_active_autoregressions(['SVR']);

lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
lEngine.getModelInfo();
print(lEngine.mSignalDecomposition.mTrPerfDetails.head());

lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution

lEngine.standrdPlots("outputs/my_ozone_svr_only");

dfapp_in = df.copy();
dfapp_in.tail()

#H = 12
dfapp_out = lEngine.forecast(dfapp_in, H);
#dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H));

print("\n\n<ModelInfo>")
print(lEngine.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.to_json(date_format='iso'))
print("</Forecast>\n\n")

<commit_msg>Reduce the output of this test.<commit_after>import pandas as pd
import numpy as np
import ForecastEngine as autof
import Bench.TS_datasets as tsds

import logging
import logging.config

#logging.config.fileConfig('logging.conf')

logging.basicConfig(level=logging.INFO)

#get_ipython().magic('matplotlib inline')

b1 = tsds.load_ozone()
df = b1.mPastData

df.head()

lEngine = autof.cForecastEngine()
lEngine

H = b1.mHorizon;
# lEngine.mOptions.enable_slow_mode();
lEngine.mOptions.mDebugPerformance = True;
lEngine.mOptions.mParallelMode = True;
lEngine.mOptions.set_active_autoregressions(['SVR']);

lEngine.train(df , b1.mTimeVar , b1.mSignalVar, H);
lEngine.getModelInfo();
print(lEngine.mSignalDecomposition.mTrPerfDetails.head());

lEngine.mSignalDecomposition.mBestModel.mTimeInfo.mResolution

lEngine.standrdPlots("outputs/my_ozone_svr_only");

dfapp_in = df.copy();
dfapp_in.tail()

#H = 12
dfapp_out = lEngine.forecast(dfapp_in, H);
#dfapp_out.to_csv("outputs/ozone_apply_out.csv")
dfapp_out.tail(2 * H)
print("Forecast Columns " , dfapp_out.columns);
Forecast_DF = dfapp_out[[b1.mTimeVar , b1.mSignalVar, b1.mSignalVar + '_Forecast']]
print(Forecast_DF.info())
print("Forecasts\n" , Forecast_DF.tail(H));

print("\n\n<ModelInfo>")
print(lEngine.to_json());
print("</ModelInfo>\n\n")
print("\n\n<Forecast>")
print(Forecast_DF.tail(H).to_json(date_format='iso'))
print("</Forecast>\n\n")

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import pytest

from pytocli import (CommandBuilder, Option, NoValueOption, SingleValueOption,
                     MultiValuesOption, SubCommandBuilder, SubCommand)
from pytocli._base import OptionFactory

class GitSubCommand(SubCommandBuilder):
    @property
    def git(self):
        return self.parent_cmd

class Commit(GitSubCommand):
    """git commit sub command"""
    name = 'commit'
    message = Option(SingleValueOption, '-m',
                     doc='Message to be added on commit')

class Revert(GitSubCommand):
    name = 'revert'

class Git(CommandBuilder):
    name = 'git'
    # Options
    verbose = Option(NoValueOption, '-v', doc='Verbose mode')
    start = Option(SingleValueOption, '-C')
    multi = Option(MultiValuesOption, '--mu')

    # SubCommands
    commit = SubCommand(Commit)
    revert = SubCommand(Revert)

def test_simple_command():
    assert 'git' == str(Git())

def test_no_value():
    assert 'git -v' == str(Git().verbose())

def test_no_value_can_not_receive_value():
    with pytest.raises(ValueError):
        Git().verbose('foo')
    cmd = Git().verbose()
    with pytest.raises(ValueError):
        cmd.verbose('later parameter')

def test_single_value():
    assert 'git -C foo.git' == str(Git().start('foo.git'))

def test_single_values_with_no_arg():
    with pytest.raises(ValueError):
        Git().start()

def test_single_values_with_2_args():
    with pytest.raises(ValueError):
        Git().start(1, 2)

def test_single_values_adding_2_args_on_consecutive_calls():
    with pytest.raises(ValueError):
        Git().start(1).start(2)

def test_multivalue():
    cmd = Git()
    assert 'git --mu 1 2' == str(cmd.multi(1, 2))
    assert 'git --mu 1 2 3' == str(cmd.multi(3))

def test_multivalue_with_no_parameter():
    with pytest.raises(ValueError):
        Git().multi()

def test_chaining():
    assert 'git -v -C foo.git' == str(Git().verbose().start('foo.git'))

def test_doc_on_repr_for_options():
    assert 'Option -v: Verbose mode' == repr(Git.verbose)
    assert 'Option --mu: No doc provided' == repr(Git.multi)

def test_options_tuple():
    assert ('verbose', 'start', 'multi') == Git.options

def test_simple_sub_command():
    assert 'git commit' == str(Git().commit)

def test_simple_command_doc_as_class_attr():
    assert 'CommandBuilder git: No doc provided' == repr(Git())

def test_simple_sub_command_doc_as_class_attr():
    assert ('SubCommandBuilder commit: git commit sub command' ==
            repr(Git.commit))

def test_chaining_sub_command():
    assert ('git -v commit -m "a great msg"' ==
            str(Git().verbose().commit.message('"a great msg"')))

def test_add_option_to_parent():
    commit = Git().commit
    assert ('git commit -m "a great msg"' ==
            str(commit.message('"a great msg"')))
    commit.git.verbose()
    assert 'git -v commit -m "a great msg"' == str(commit)

def test_sub_commands_tuple():
    assert ('commit', 'revert') == Git.sub_commands

def test_option_factory_str_is_abstract():
    with pytest.raises(NotImplementedError):
        str(OptionFactory('foo', 'foo'))

def test_option_factory_repr():
    assert ('OptionFactory Option with name foo and value []' ==
            repr(OptionFactory('foo', 'bar')))

def test_option_factory_add_values_is_abstract():
    with pytest.raises(NotImplementedError):
        OptionFactory('foo', 'foo').add_values()

def test_options_class_attr():
    assert ('verbose', 'start', 'multi') == Git.options

def test_sub_commands_class_attr():
    assert ('commit', 'revert') == Git.sub_commands

class EqualAndSplitStub(CommandBuilder):
    name = 'foo'
    single = Option(SingleValueOption, 'single', equal_sign='#')
    multi = Option(MultiValuesOption, 'multi', equal_sign='=', separator=',')

def test_single_equal_sign():
    assert 'foo single#1' == str(EqualAndSplitStub().single(1))

def test_multi_equal_sign_and_separator():
    assert 'foo multi=2,3,4' == str(EqualAndSplitStub().multi(2, 3, 4))
<commit_msg>Testing SubCommand repr. Coverage 100%.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import pytest

from pytocli import (CommandBuilder, Option, NoValueOption, SingleValueOption,
                     MultiValuesOption, SubCommandBuilder, SubCommand)
from pytocli._base import OptionFactory

class GitSubCommand(SubCommandBuilder):
    @property
    def git(self):
        return self.parent_cmd

class Commit(GitSubCommand):
    """git commit sub command"""
    name = 'commit'
    message = Option(SingleValueOption, '-m',
                     doc='Message to be added on commit')

class Revert(GitSubCommand):
    name = 'revert'

class Git(CommandBuilder):
    name = 'git'
    # Options
    verbose = Option(NoValueOption, '-v', doc='Verbose mode')
    start = Option(SingleValueOption, '-C')
    multi = Option(MultiValuesOption, '--mu')

    # SubCommands
    commit = SubCommand(Commit)
    revert = SubCommand(Revert)

def test_simple_command():
    assert 'git' == str(Git())

def test_no_value():
    assert 'git -v' == str(Git().verbose())

def test_no_value_can_not_receive_value():
    with pytest.raises(ValueError):
        Git().verbose('foo')
    cmd = Git().verbose()
    with pytest.raises(ValueError):
        cmd.verbose('later parameter')

def test_single_value():
    assert 'git -C foo.git' == str(Git().start('foo.git'))

def test_single_values_with_no_arg():
    with pytest.raises(ValueError):
        Git().start()

def test_single_values_with_2_args():
    with pytest.raises(ValueError):
        Git().start(1, 2)

def test_single_values_adding_2_args_on_consecutive_calls():
    with pytest.raises(ValueError):
        Git().start(1).start(2)

def test_multivalue():
    cmd = Git()
    assert 'git --mu 1 2' == str(cmd.multi(1, 2))
    assert 'git --mu 1 2 3' == str(cmd.multi(3))

def test_multivalue_with_no_parameter():
    with pytest.raises(ValueError):
        Git().multi()

def test_chaining():
    assert 'git -v -C foo.git' == str(Git().verbose().start('foo.git'))

def test_doc_on_repr_for_options():
    assert 'Option -v: Verbose mode' == repr(Git.verbose)
    assert 'Option --mu: No doc provided' == repr(Git.multi)

def test_options_tuple():
    assert ('verbose', 'start', 'multi') == Git.options

def test_simple_sub_command():
    assert 'git commit' == str(Git().commit)

def test_simple_command_doc_as_class_attr():
    assert 'CommandBuilder git: No doc provided' == repr(Git())

def test_simple_sub_command_repr():
    assert ('SubCommandBuilder commit: git commit sub command' ==
            repr(Commit(Git())))

def test_simple_sub_command_doc_as_class_attr():
    assert ('SubCommandBuilder commit: git commit sub command' ==
            repr(Git.commit))

def test_chaining_sub_command():
    assert ('git -v commit -m "a great msg"' ==
            str(Git().verbose().commit.message('"a great msg"')))

def test_add_option_to_parent():
    commit = Git().commit
    assert ('git commit -m "a great msg"' ==
            str(commit.message('"a great msg"')))
    commit.git.verbose()
    assert 'git -v commit -m "a great msg"' == str(commit)

def test_sub_commands_tuple():
    assert ('commit', 'revert') == Git.sub_commands

def test_option_factory_str_is_abstract():
    with pytest.raises(NotImplementedError):
        str(OptionFactory('foo', 'foo'))

def test_option_factory_repr():
    assert ('OptionFactory Option with name foo and value []' ==
            repr(OptionFactory('foo', 'bar')))

def test_option_factory_add_values_is_abstract():
    with pytest.raises(NotImplementedError):
        OptionFactory('foo', 'foo').add_values()

def test_options_class_attr():
    assert ('verbose', 'start', 'multi') == Git.options

def test_sub_commands_class_attr():
    assert ('commit', 'revert') == Git.sub_commands

class EqualAndSplitStub(CommandBuilder):
    name = 'foo'
    single = Option(SingleValueOption, 'single', equal_sign='#')
    multi = Option(MultiValuesOption, 'multi', equal_sign='=', separator=',')

def test_single_equal_sign():
    assert 'foo single#1' == str(EqualAndSplitStub().single(1))

def test_multi_equal_sign_and_separator():
    assert 'foo multi=2,3,4' == str(EqualAndSplitStub().multi(2, 3, 4))
<|endoftext|>


<commit_before>from __future__ import absolute_import
import os
import uuid

from couchdbkit.exceptions import ResourceNotFound
from django.test.testcases import TestCase
from mock import patch

from corehq.apps.app_manager.exceptions import AppEditingError
from corehq.apps.app_manager.models import (
    Application,
    ReportModule, ReportAppConfig, Module, RemoteAppDetails, LinkedApplication)
from corehq.apps.linked_domain.exceptions import ActionNotPermitted
from corehq.apps.linked_domain.remote_accessors import _convert_app_from_remote_linking_source, \
    _get_missing_multimedia, _fetch_remote_media
from corehq.apps.app_manager.tests.util import TestXmlMixin
from corehq.apps.app_manager.views.utils import overwrite_app, _get_form_id_map
from corehq.apps.hqmedia.models import CommCareImage, CommCareMultimedia
from corehq.apps.linked_domain.util import convert_app_for_remote_linking

class BaseLinkedAppsTest(TestCase, TestXmlMixin):
    file_path = ('data',)

    @classmethod
    def setUpClass(cls):
        super(BaseLinkedAppsTest, cls).setUpClass()
        cls.master_app_with_report_modules = Application.new_app('domain', "Master Application")
        module = cls.master_app_with_report_modules.add_module(ReportModule.new_module('Reports', None))
        module.report_configs = [
            ReportAppConfig(report_id='id', header={'en': 'CommBugz'}),
        ]

        cls.plain_master_app = Application.new_app('domain', "Master Application")
        cls.plain_master_app.linked_whitelist = ['domain-2']
        cls.plain_master_app.save()

        cls.linked_app = LinkedApplication.new_app('domain-2', "Linked Application")
        cls.linked_app.save()

    @classmethod
    def tearDownClass(cls):
        cls.linked_app.delete()
        cls.plain_master_app.delete()
        super(BaseLinkedAppsTest, cls).tearDownClass()

    def setUp(self):
        # re-fetch app
        self.linked_app = LinkedApplication.get(self.linked_app._id)

class TestLinkedApps(BaseLinkedAppsTest):
    def test_missing_ucrs(self):
        with self.assertRaises(AppEditingError):
            overwrite_app(self.linked_app, self.master_app_with_report_modules, {})

    def test_report_mapping(self):
        report_map = {'id': 'mapped_id'}
        overwrite_app(self.linked_app, self.master_app_with_report_modules, report_map)
        linked_app = Application.get(self.linked_app._id)
        self.assertEqual(linked_app.modules[0].report_configs[0].report_id, 'mapped_id')

    def test_overwrite_app_maintain_ids(self):
        module = self.plain_master_app.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        module = self.linked_app.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        id_map_before = _get_form_id_map(self.linked_app)

        overwrite_app(self.linked_app, self.plain_master_app, {})
        self.assertEqual(
            id_map_before,
            _get_form_id_map(LinkedApplication.get(self.linked_app._id))
        )

    def test_get_master_version(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        self.assertIsNone(self.linked_app.get_master_version())

        copy = self.plain_master_app.make_build()
        copy.save()
        self.addCleanup(copy.delete)

        self.assertIsNone(self.linked_app.get_master_version())

        self.plain_master_app.save()  # increment version number
        copy1 = self.plain_master_app.make_build()
        copy1.is_released = True
        copy1.save()
        self.addCleanup(copy1.delete)

        self.assertEqual(copy1.version, self.linked_app.get_master_version())

    def test_get_latest_master_release(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        self.assertIsNone(self.linked_app.get_latest_master_release())

        copy = self.plain_master_app.make_build()
        copy.save()
        self.addCleanup(copy.delete)

        self.assertIsNone(self.linked_app.get_latest_master_release())

        self.plain_master_app.save()  # increment version number
        copy1 = self.plain_master_app.make_build()
        copy1.is_released = True
        copy1.save()
        self.addCleanup(copy1.delete)

        latest_master_release = self.linked_app.get_latest_master_release()
        self.assertEqual(copy1.get_id, latest_master_release.get_id)
        self.assertEqual(copy1._rev, latest_master_release._rev)

    def test_get_latest_master_release_not_permitted(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        release = self.plain_master_app.make_build()
        release.is_released = True
        release.save()
        self.addCleanup(release.delete)

        latest_master_release = self.linked_app.get_latest_master_release()
        self.assertEqual(release.get_id, latest_master_release.get_id)

        original = self.plain_master_app.linked_whitelist
        self.plain_master_app.linked_whitelist = []
        self.plain_master_app.save()

        def _revert():
            self.plain_master_app.linked_whitelist = original
            self.plain_master_app.save()

        self.addCleanup(_revert)

        with self.assertRaises(ActionNotPermitted):
            self.linked_app.get_latest_master_release()

class TestRemoteLinkedApps(BaseLinkedAppsTest):

    @classmethod
    def setUpClass(cls):
        super(TestRemoteLinkedApps, cls).setUpClass()
        image_data = cls._get_image_data('commcare-hq-logo.png')
        cls.image = CommCareImage.get_by_data(image_data)
        cls.image.attach_data(image_data, original_filename='logo.png')
        cls.image.add_domain(cls.plain_master_app.domain)

    @classmethod
    def tearDownClass(cls):
        cls.image.delete()
        super(TestRemoteLinkedApps, cls).tearDownClass()

    @staticmethod
    def _get_image_data(filename):
        image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'images', filename)
        with open(image_path, 'r') as f:
            return f.read()

    def test_remote_app(self):
        module = self.master_app_with_report_modules.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        linked_app = _mock_pull_remote_master(
            self.master_app_with_report_modules, self.linked_app, {'id': 'mapped_id'}
        )
        master_id_map = _get_form_id_map(self.master_app_with_report_modules)
        linked_id_map = _get_form_id_map(linked_app)
        for xmlns, master_form_id in master_id_map.items():
            linked_form_id = linked_id_map[xmlns]
            self.assertEqual(
                self.master_app_with_report_modules.get_form(master_form_id).source,
                linked_app.get_form(linked_form_id).source
            )

    def test_get_missing_media_list(self):
        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)

        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        with patch('corehq.apps.hqmedia.models.CommCareMultimedia.get', side_effect=ResourceNotFound):
            missing_media = _get_missing_multimedia(self.master_app_with_report_modules)

        media_item = list(self.master_app_with_report_modules.multimedia_map.values())[0]
        self.assertEqual(missing_media, [('case_list_image.jpg', media_item)])

    def test_add_domain_to_media(self):
        self.image.valid_domains.remove(self.master_app_with_report_modules.domain)
        self.image.save()

        image = CommCareImage.get(self.image._id)
        self.assertNotIn(self.master_app_with_report_modules.domain, image.valid_domains)

        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)
        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        missing_media = _get_missing_multimedia(self.master_app_with_report_modules)
        self.assertEqual(missing_media, [])

        image = CommCareImage.get(self.image._id)
        self.assertIn(self.master_app_with_report_modules.domain, image.valid_domains)

    def test_fetch_missing_media(self):
        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)
        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        remote_app_details = RemoteAppDetails(
            'http://localhost:8000', 'test_domain', 'user', 'key', self.master_app_with_report_modules._id
        )
        data = 'this is a test'
        media_details = list(self.master_app_with_report_modules.multimedia_map.values())[0]
        media_details['multimedia_id'] = uuid.uuid4().hex
        media_details['media_type'] = 'CommCareMultimedia'
        with patch('corehq.apps.app_manager.remote_link_accessors._fetch_remote_media_content') as mock:
            mock.return_value = data
            _fetch_remote_media('domain', [('case_list_image.jpg', media_details)], remote_app_details)

        media = CommCareMultimedia.get(media_details['multimedia_id'])
        self.addCleanup(media.delete)
        content = media.fetch_attachment(list(media.blobs.keys())[0])
        self.assertEqual(data, content)

def _mock_pull_remote_master(master_app, linked_app, report_map=None):
    master_source = convert_app_for_remote_linking(master_app)
    master_app = _convert_app_from_remote_linking_source(master_source)
    overwrite_app(linked_app, master_app, report_map or {})
    return Application.get(linked_app._id)
<commit_msg>fix test patching<commit_after>from __future__ import absolute_import
import os
import uuid

from couchdbkit.exceptions import ResourceNotFound
from django.test.testcases import TestCase
from mock import patch

from corehq.apps.app_manager.exceptions import AppEditingError
from corehq.apps.app_manager.models import (
    Application,
    ReportModule, ReportAppConfig, Module, RemoteAppDetails, LinkedApplication)
from corehq.apps.linked_domain.exceptions import ActionNotPermitted
from corehq.apps.linked_domain.remote_accessors import _convert_app_from_remote_linking_source, \
    _get_missing_multimedia, _fetch_remote_media
from corehq.apps.app_manager.tests.util import TestXmlMixin
from corehq.apps.app_manager.views.utils import overwrite_app, _get_form_id_map
from corehq.apps.hqmedia.models import CommCareImage, CommCareMultimedia
from corehq.apps.linked_domain.util import convert_app_for_remote_linking

class BaseLinkedAppsTest(TestCase, TestXmlMixin):
    file_path = ('data',)

    @classmethod
    def setUpClass(cls):
        super(BaseLinkedAppsTest, cls).setUpClass()
        cls.master_app_with_report_modules = Application.new_app('domain', "Master Application")
        module = cls.master_app_with_report_modules.add_module(ReportModule.new_module('Reports', None))
        module.report_configs = [
            ReportAppConfig(report_id='id', header={'en': 'CommBugz'}),
        ]

        cls.plain_master_app = Application.new_app('domain', "Master Application")
        cls.plain_master_app.linked_whitelist = ['domain-2']
        cls.plain_master_app.save()

        cls.linked_app = LinkedApplication.new_app('domain-2', "Linked Application")
        cls.linked_app.save()

    @classmethod
    def tearDownClass(cls):
        cls.linked_app.delete()
        cls.plain_master_app.delete()
        super(BaseLinkedAppsTest, cls).tearDownClass()

    def setUp(self):
        # re-fetch app
        self.linked_app = LinkedApplication.get(self.linked_app._id)

class TestLinkedApps(BaseLinkedAppsTest):
    def test_missing_ucrs(self):
        with self.assertRaises(AppEditingError):
            overwrite_app(self.linked_app, self.master_app_with_report_modules, {})

    def test_report_mapping(self):
        report_map = {'id': 'mapped_id'}
        overwrite_app(self.linked_app, self.master_app_with_report_modules, report_map)
        linked_app = Application.get(self.linked_app._id)
        self.assertEqual(linked_app.modules[0].report_configs[0].report_id, 'mapped_id')

    def test_overwrite_app_maintain_ids(self):
        module = self.plain_master_app.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        module = self.linked_app.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        id_map_before = _get_form_id_map(self.linked_app)

        overwrite_app(self.linked_app, self.plain_master_app, {})
        self.assertEqual(
            id_map_before,
            _get_form_id_map(LinkedApplication.get(self.linked_app._id))
        )

    def test_get_master_version(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        self.assertIsNone(self.linked_app.get_master_version())

        copy = self.plain_master_app.make_build()
        copy.save()
        self.addCleanup(copy.delete)

        self.assertIsNone(self.linked_app.get_master_version())

        self.plain_master_app.save()  # increment version number
        copy1 = self.plain_master_app.make_build()
        copy1.is_released = True
        copy1.save()
        self.addCleanup(copy1.delete)

        self.assertEqual(copy1.version, self.linked_app.get_master_version())

    def test_get_latest_master_release(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        self.assertIsNone(self.linked_app.get_latest_master_release())

        copy = self.plain_master_app.make_build()
        copy.save()
        self.addCleanup(copy.delete)

        self.assertIsNone(self.linked_app.get_latest_master_release())

        self.plain_master_app.save()  # increment version number
        copy1 = self.plain_master_app.make_build()
        copy1.is_released = True
        copy1.save()
        self.addCleanup(copy1.delete)

        latest_master_release = self.linked_app.get_latest_master_release()
        self.assertEqual(copy1.get_id, latest_master_release.get_id)
        self.assertEqual(copy1._rev, latest_master_release._rev)

    def test_get_latest_master_release_not_permitted(self):
        self.linked_app.master = self.plain_master_app.get_id
        self.linked_app.master_domain = self.plain_master_app.domain

        release = self.plain_master_app.make_build()
        release.is_released = True
        release.save()
        self.addCleanup(release.delete)

        latest_master_release = self.linked_app.get_latest_master_release()
        self.assertEqual(release.get_id, latest_master_release.get_id)

        original = self.plain_master_app.linked_whitelist
        self.plain_master_app.linked_whitelist = []
        self.plain_master_app.save()

        def _revert():
            self.plain_master_app.linked_whitelist = original
            self.plain_master_app.save()

        self.addCleanup(_revert)

        with self.assertRaises(ActionNotPermitted):
            self.linked_app.get_latest_master_release()

class TestRemoteLinkedApps(BaseLinkedAppsTest):

    @classmethod
    def setUpClass(cls):
        super(TestRemoteLinkedApps, cls).setUpClass()
        image_data = cls._get_image_data('commcare-hq-logo.png')
        cls.image = CommCareImage.get_by_data(image_data)
        cls.image.attach_data(image_data, original_filename='logo.png')
        cls.image.add_domain(cls.plain_master_app.domain)

    @classmethod
    def tearDownClass(cls):
        cls.image.delete()
        super(TestRemoteLinkedApps, cls).tearDownClass()

    @staticmethod
    def _get_image_data(filename):
        image_path = os.path.join('corehq', 'apps', 'hqwebapp', 'static', 'hqwebapp', 'images', filename)
        with open(image_path, 'r') as f:
            return f.read()

    def test_remote_app(self):
        module = self.master_app_with_report_modules.add_module(Module.new_module('M1', None))
        module.new_form('f1', None, self.get_xml('very_simple_form'))

        linked_app = _mock_pull_remote_master(
            self.master_app_with_report_modules, self.linked_app, {'id': 'mapped_id'}
        )
        master_id_map = _get_form_id_map(self.master_app_with_report_modules)
        linked_id_map = _get_form_id_map(linked_app)
        for xmlns, master_form_id in master_id_map.items():
            linked_form_id = linked_id_map[xmlns]
            self.assertEqual(
                self.master_app_with_report_modules.get_form(master_form_id).source,
                linked_app.get_form(linked_form_id).source
            )

    def test_get_missing_media_list(self):
        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)

        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        with patch('corehq.apps.hqmedia.models.CommCareMultimedia.get', side_effect=ResourceNotFound):
            missing_media = _get_missing_multimedia(self.master_app_with_report_modules)

        media_item = list(self.master_app_with_report_modules.multimedia_map.values())[0]
        self.assertEqual(missing_media, [('case_list_image.jpg', media_item)])

    def test_add_domain_to_media(self):
        self.image.valid_domains.remove(self.master_app_with_report_modules.domain)
        self.image.save()

        image = CommCareImage.get(self.image._id)
        self.assertNotIn(self.master_app_with_report_modules.domain, image.valid_domains)

        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)
        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        missing_media = _get_missing_multimedia(self.master_app_with_report_modules)
        self.assertEqual(missing_media, [])

        image = CommCareImage.get(self.image._id)
        self.assertIn(self.master_app_with_report_modules.domain, image.valid_domains)

    def test_fetch_missing_media(self):
        image_path = 'jr://file/commcare/case_list_image.jpg'
        self.master_app_with_report_modules.get_module(0).set_icon('en', image_path)
        self.master_app_with_report_modules.create_mapping(self.image, image_path, save=False)

        remote_app_details = RemoteAppDetails(
            'http://localhost:8000', 'test_domain', 'user', 'key', self.master_app_with_report_modules._id
        )
        data = 'this is a test'
        media_details = list(self.master_app_with_report_modules.multimedia_map.values())[0]
        media_details['multimedia_id'] = uuid.uuid4().hex
        media_details['media_type'] = 'CommCareMultimedia'
        with patch('corehq.apps.linked_domain.remote_accessors._fetch_remote_media_content') as mock:
            mock.return_value = data
            _fetch_remote_media('domain', [('case_list_image.jpg', media_details)], remote_app_details)

        media = CommCareMultimedia.get(media_details['multimedia_id'])
        self.addCleanup(media.delete)
        content = media.fetch_attachment(list(media.blobs.keys())[0])
        self.assertEqual(data, content)

def _mock_pull_remote_master(master_app, linked_app, report_map=None):
    master_source = convert_app_for_remote_linking(master_app)
    master_app = _convert_app_from_remote_linking_source(master_source)
    overwrite_app(linked_app, master_app, report_map or {})
    return Application.get(linked_app._id)
<|endoftext|>


<commit_before>#-*- coding:utf-8 -*-
#
# This file is part of MoCGF - a code generation framework
# 201500225 Joerg Raedler jraedler@udk-berlin.de
#

import sys, os, os.path, tempfile, argparse, logging, configparser
import MoCGF
from MoCGF.Controller import Controller
from PyQt4 import QtCore, QtGui, uic

descr = """
MoCGF is a framework to generate source code from data sources.
It is developed in the EnEff-BIM project to generate Modelica code from SimModel data.
MoCGF-cli is the command line interface to MoCGF. MoCGF-gui provides a graphical interface."""

# view log in Qt - inspired by
# http://stackoverflow.com/questions/14349563/how-to-get-non-blocking-real-time-behavior-from-python-logging-module-output-t

class QtLogHandler(logging.Handler):

    def __init__(self):
        logging.Handler.__init__(self)

    def emit(self, record):
        record = self.format(record)
        if record:
            XStream.stdout().write('%s\n'%record)

class XStream(QtCore.QObject):
    _stdout = None
    _stderr = None
    _ostdout = None
    _ostderr = None
    messageWritten = QtCore.pyqtSignal(str)

    def flush( self ):
        pass

    def fileno( self ):
        return -1

    def write( self, msg ):
        if not self.signalsBlocked():
            self.messageWritten.emit(msg)

    @staticmethod
    def stdout():
        if not XStream._stdout:
            XStream._stdout = XStream()
            XStream._ostdout = sys.stdout
            sys.stdout = XStream._stdout
        return XStream._stdout

    @staticmethod
    def stderr():
        if not XStream._stderr:
            XStream._stderr = XStream()
            XStream._ostderr = sys.stderr
            sys.stderr = XStream._stderr
        return XStream._stderr

    @staticmethod
    def reset():
        if XStream._stdout:
            sys.stdout = XStream._ostdout
        if XStream._stderr:
            sys.stdout = XStream._ostderr

class LogViewer(QtGui.QWidget):
    def __init__(self, resPath, *arg, **kwarg):
        QtGui.QWidget.__init__(self)
        # load the ui
        self.ui = uic.loadUi(os.path.join(resPath, 'MessageBrowser.ui'), self)
        XStream.stdout().messageWritten.connect(self.textBrowser.insertPlainText)
        XStream.stderr().messageWritten.connect(self.textBrowser.insertPlainText)
        self.logHandler = QtLogHandler()
        self.logHandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
        self.logger = logging.getLogger('MoCGF')
        i = int(kwarg['logLevel'] / 10 ) - 1 # convert level to QComboBox index
        self.levelSelect.setCurrentIndex(i)
        self.levelSelect.currentIndexChanged.connect(self.setLevel)
        self.clearButton.pressed.connect(self.clearLog)
        self.saveButton.pressed.connect(self.saveLog)

    def setLevel(self, i):
        l = (i+1) * 10 # convert QComboBox index to level :-)
        self.logger.setLevel(l)

    def clearLog(self):
        self.textBrowser.clear()

    def saveLog(self):
        f = QtGui.QFileDialog.getSaveFileName(self, 'Select Output File', '*')
        if not f:
            return
        try:
            open(f, 'w').write(self.textBrowser.toPlainText())
        except Exception as e:
            QtGui.QMessageBox.critical(self, 'Error during save', 'Message log could not be saved!')
            self.logger.exception('Could not save message log')

class MoCGFWidget(QtGui.QWidget):
    def __init__(self, app, resPath, cfg, *arg, **kwarg):
        QtGui.QWidget.__init__(self)
        self.app = app
        # load the Icons
        sys.path.insert(0, resPath)
        import Icons_rc
        # load the ui
        self.ui = uic.loadUi(os.path.join(resPath, 'MoCGF-GUI.ui'), self)
        self.setWindowTitle('MoCGF GUI | Version: %s' % (MoCGF.__version__))
        self.cfg = cfg
        # create a logView?
        if 'logLevel' in kwarg and kwarg['logLevel'] > 0:
            self.logView = LogViewer(resPath, *arg, **kwarg)
            self.moCGFMainView.addTab(self.logView, 'Messages')
            kwarg['logHandler'] = self.logView.logHandler
        # create a controller
        self.mocgf = Controller(*arg, **kwarg)
        # apis
        self.apiList.itemSelectionChanged.connect(self.activateAPI)
        self.apiView.anchorClicked.connect(self.openURL)
        for a in sorted(self.mocgf.apis):
            self.apiList.addItem(a)
        self.apiList.item(0).setSelected(True)

        # generators
        self.activeGenerator=None
        self.generatorListReloadButton.pressed.connect(self.updateGeneratorList)
        self.generatorExplorerButton.pressed.connect(self.exploreGenerators)
        self.generatorList.itemSelectionChanged.connect(self.activateGenerator)
        self.generatorView.anchorClicked.connect(self.openURL)
        self.generateButton.clicked.connect(self.executeGenerator)
        self.updateGeneratorList(rescan=False)

        # buttons
        self.uriLoadButton.clicked.connect(self.getUriList)
        self.outputLoadButton.clicked.connect(self.getOutputFile)

        # set preferences from cfg
        if cfg.has_section('PREFERENCES'):
            p = cfg['PREFERENCES']
            self.uriInput.setText(p.get('uriList', ''))
            self.outputInput.setText(p.get('outputFile', ''))
            g = p.get('generator', '')
            for x in range(self.generatorList.count()):
                i = self.generatorList.item(x)
                if i.text() == g:
                    i.setSelected(1)
                    break
        # end of preferences

    # general methods
    def openURL(self, url):
        """open an link target from the generator or api view"""
        scheme = url.scheme()
        if scheme == 'file':
            # print(url)
            QtGui.QDesktopServices.openUrl(url)
        elif scheme == 'api':
            a = url.authority().replace('___', '::').lower()
            i = self.apiList.findItems(a, QtCore.Qt.MatchFixedString)[0]
            self.apiList.setCurrentItem(i)
            self.apiList.itemActivated.emit(i)
            self.moCGFMainView.setCurrentIndex(1)
        else:
            print('Unknown URL scheme:', url)

    def getUriList(self):
        flist = QtGui.QFileDialog.getOpenFileNames(self, 'Select Data Sources')
        if flist:
            self.uriInput.setText(', '.join(flist))

    def getOutputFile(self):
        f = QtGui.QFileDialog.getSaveFileName(self, 'Select Output File', '*')
        if f:
            self.outputInput.setText(f)

    # API methods
    def activateAPI(self):
        items = self.apiList.selectedItems()
        if items:
            api = items[0].text()
            if MoCGF.py27:
                api = unicode(api)
        self.apiView.clear()
        self.apiView.setText(self.mocgf.apiInfoText(api, 'html'))

    # generator methods
    def exploreGenerators(self):
        QtGui.QDesktopServices.openUrl(QtCore.QUrl.fromLocalFile(self.mocgf.generatorPath))

    def updateGeneratorList(self, rescan=True):
        # get old selection
        selItems = self.generatorList.selectedItems()
        selected = None
        if selItems:
            selected = selItems[0].text()
            if MoCGF.py27:
                seleected = unicode(selected)
        if rescan:
            self.mocgf.rescanGenerators()
        self.generatorList.clear()
        i = 0
        for n in sorted(self.mocgf.generators):
            self.generatorList.addItem(n)
            if n == selected:
                # select previously selected generator
                self.generatorList.item(i).setSelected(True)
            i += 1
        if not self.generatorList.selectedItems():
            # nothing selected, select the first in the list
            self.generatorList.item(0).setSelected(True)

    def activateGenerator(self):
        sel = self.generatorList.selectedItems()
        if sel:
            gen = sel[0].text()
            if MoCGF.py27:
                gen = unicode(gen)
        else:
            return
        self.activeGenerator = self.mocgf.generators[gen]
        self.generatorView.clear()
        self.generatorView.setText(self.activeGenerator.infoText('html'))

    def executeGenerator(self):
        line = self.uriInput.text()
        if MoCGF.py27:
            line = unicode(line)
        uriList = [u.strip() for u in line.split(',')]
        outputFile = self.outputInput.text()
        if not outputFile:
            tmp, outputFile = tempfile.mkstemp(suffix='.txt', text=True)
            self.outputInput.setText(outputFile)
        x = self.activeGenerator.execute(uriList)
        o = open(outputFile, 'w')
        o.write(x.read())
        o.close()
        if self.openOutputButton.isChecked():
            QtGui.QDesktopServices.openUrl(QtCore.QUrl.fromLocalFile(outputFile))

    def closeEvent(self, e):
        # first reset output streams to standard settings
        XStream.reset()
        # save preferences to config file
        if hasattr(self.cfg, 'path'):
            uriList = self.uriInput.text()
            outputFile = self.outputInput.text()
            generator = ''
            tmp = self.generatorList.selectedItems()
            if tmp:
                generator = tmp[0].text()
                if MoCGF.py27:
                    generator = unicode(generator)
            try:
                c = self.cfg
                p = 'PREFERENCES'
                if not c.has_section(p):
                    c.add_section(p)
                c.set(p, 'uriList', uriList)
                c.set(p, 'outputFile', outputFile)
                c.set(p, 'generator', generator)
                with open(c.path, 'w') as configfile:
                    c.write(configfile)
            except:
                # silently ignore errors
                pass
        e.accept()

def main():
    """main function when MoCGF is used with the Qt gui"""

    parser = argparse.ArgumentParser(description=descr)
    grp = parser.add_argument_group('path settings')
    grp.add_argument('-p', '--search-path', metavar='PATH', help='search path for generators (separated by ;)')
    grp = parser.add_argument_group('general information')
    grp.add_argument('-d', '--debug', metavar='LEVEL', help='set debug level and activate messages tab (1...5)')

    args = parser.parse_args()

    # first read config file for default values
    defaults = {
        'GeneratorPath': os.environ.get('MOCGF_GENERATORS', ''),
        'LogLevel' : '0',
    }
    cfg = configparser.ConfigParser(defaults)
    homeVar = {'win32':'USERPROFILE', 'linux':'HOME', 'linux2':'HOME', 'darwin':'HOME'}.get(sys.platform)
    cfgFile = os.path.join(os.environ.get(homeVar, ''), '.MoCGF.cfg')
    if os.path.isfile(cfgFile):
        cfg.read(cfgFile)
        cfg.path = cfgFile

    # generatorPath
    gp = args.search_path or cfg['DEFAULT']['GeneratorPath']
    generatorPath = [p for p in gp.split(';') if p]

    # logLevel
    logLevel = 10 * cfg.getint('DEFAULT', 'LogLevel')
    if args.debug:
        logLevel = 10 * int(args.debug)

    # FIXME: resPath may need to be adjusted after installation,
    # or use pkg_resources? See:
    # http://stackoverflow.com/questions/779495/python-access-data-in-package-subdirectory
    resPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'res')

    app = QtGui.QApplication(sys.argv)
    mw = MoCGFWidget(app, resPath, cfg, generatorPath, logLevel=logLevel)
    mw.show()
    r = app.exec_()
    return(r)
<commit_msg>fixed bug with relative output file names<commit_after>#-*- coding:utf-8 -*-
#
# This file is part of MoCGF - a code generation framework
# 201500225 Joerg Raedler jraedler@udk-berlin.de
#

import sys, os, os.path, tempfile, argparse, logging, configparser
import MoCGF
from MoCGF.Controller import Controller
from PyQt4 import QtCore, QtGui, uic

descr = """
MoCGF is a framework to generate source code from data sources.
It is developed in the EnEff-BIM project to generate Modelica code from SimModel data.
MoCGF-cli is the command line interface to MoCGF. MoCGF-gui provides a graphical interface."""

# view log in Qt - inspired by
# http://stackoverflow.com/questions/14349563/how-to-get-non-blocking-real-time-behavior-from-python-logging-module-output-t

class QtLogHandler(logging.Handler):

    def __init__(self):
        logging.Handler.__init__(self)

    def emit(self, record):
        record = self.format(record)
        if record:
            XStream.stdout().write('%s\n'%record)

class XStream(QtCore.QObject):
    _stdout = None
    _stderr = None
    _ostdout = None
    _ostderr = None
    messageWritten = QtCore.pyqtSignal(str)

    def flush( self ):
        pass

    def fileno( self ):
        return -1

    def write( self, msg ):
        if not self.signalsBlocked():
            self.messageWritten.emit(msg)

    @staticmethod
    def stdout():
        if not XStream._stdout:
            XStream._stdout = XStream()
            XStream._ostdout = sys.stdout
            sys.stdout = XStream._stdout
        return XStream._stdout

    @staticmethod
    def stderr():
        if not XStream._stderr:
            XStream._stderr = XStream()
            XStream._ostderr = sys.stderr
            sys.stderr = XStream._stderr
        return XStream._stderr

    @staticmethod
    def reset():
        if XStream._stdout:
            sys.stdout = XStream._ostdout
        if XStream._stderr:
            sys.stdout = XStream._ostderr

class LogViewer(QtGui.QWidget):
    def __init__(self, resPath, *arg, **kwarg):
        QtGui.QWidget.__init__(self)
        # load the ui
        self.ui = uic.loadUi(os.path.join(resPath, 'MessageBrowser.ui'), self)
        XStream.stdout().messageWritten.connect(self.textBrowser.insertPlainText)
        XStream.stderr().messageWritten.connect(self.textBrowser.insertPlainText)
        self.logHandler = QtLogHandler()
        self.logHandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
        self.logger = logging.getLogger('MoCGF')
        i = int(kwarg['logLevel'] / 10 ) - 1 # convert level to QComboBox index
        self.levelSelect.setCurrentIndex(i)
        self.levelSelect.currentIndexChanged.connect(self.setLevel)
        self.clearButton.pressed.connect(self.clearLog)
        self.saveButton.pressed.connect(self.saveLog)

    def setLevel(self, i):
        l = (i+1) * 10 # convert QComboBox index to level :-)
        self.logger.setLevel(l)

    def clearLog(self):
        self.textBrowser.clear()

    def saveLog(self):
        f = QtGui.QFileDialog.getSaveFileName(self, 'Select Output File', '*')
        if not f:
            return
        try:
            open(f, 'w').write(self.textBrowser.toPlainText())
        except Exception as e:
            QtGui.QMessageBox.critical(self, 'Error during save', 'Message log could not be saved!')
            self.logger.exception('Could not save message log')

class MoCGFWidget(QtGui.QWidget):
    def __init__(self, app, resPath, cfg, *arg, **kwarg):
        QtGui.QWidget.__init__(self)
        self.app = app
        # load the Icons
        sys.path.insert(0, resPath)
        import Icons_rc
        # load the ui
        self.ui = uic.loadUi(os.path.join(resPath, 'MoCGF-GUI.ui'), self)
        self.setWindowTitle('MoCGF GUI | Version: %s' % (MoCGF.__version__))
        self.cfg = cfg
        # create a logView?
        if 'logLevel' in kwarg and kwarg['logLevel'] > 0:
            self.logView = LogViewer(resPath, *arg, **kwarg)
            self.moCGFMainView.addTab(self.logView, 'Messages')
            kwarg['logHandler'] = self.logView.logHandler
        # create a controller
        self.mocgf = Controller(*arg, **kwarg)
        # apis
        self.apiList.itemSelectionChanged.connect(self.activateAPI)
        self.apiView.anchorClicked.connect(self.openURL)
        for a in sorted(self.mocgf.apis):
            self.apiList.addItem(a)
        self.apiList.item(0).setSelected(True)

        # generators
        self.activeGenerator=None
        self.generatorListReloadButton.pressed.connect(self.updateGeneratorList)
        self.generatorExplorerButton.pressed.connect(self.exploreGenerators)
        self.generatorList.itemSelectionChanged.connect(self.activateGenerator)
        self.generatorView.anchorClicked.connect(self.openURL)
        self.generateButton.clicked.connect(self.executeGenerator)
        self.updateGeneratorList(rescan=False)

        # buttons
        self.uriLoadButton.clicked.connect(self.getUriList)
        self.outputLoadButton.clicked.connect(self.getOutputFile)

        # set preferences from cfg
        if cfg.has_section('PREFERENCES'):
            p = cfg['PREFERENCES']
            self.uriInput.setText(p.get('uriList', ''))
            self.outputInput.setText(p.get('outputFile', ''))
            g = p.get('generator', '')
            for x in range(self.generatorList.count()):
                i = self.generatorList.item(x)
                if i.text() == g:
                    i.setSelected(1)
                    break
        # end of preferences

    # general methods
    def openURL(self, url):
        """open an link target from the generator or api view"""
        scheme = url.scheme()
        if scheme == 'file':
            # print(url)
            QtGui.QDesktopServices.openUrl(url)
        elif scheme == 'api':
            a = url.authority().replace('___', '::').lower()
            i = self.apiList.findItems(a, QtCore.Qt.MatchFixedString)[0]
            self.apiList.setCurrentItem(i)
            self.apiList.itemActivated.emit(i)
            self.moCGFMainView.setCurrentIndex(1)
        else:
            print('Unknown URL scheme:', url)

    def getUriList(self):
        flist = QtGui.QFileDialog.getOpenFileNames(self, 'Select Data Sources')
        if flist:
            self.uriInput.setText(', '.join(flist))

    def getOutputFile(self):
        f = QtGui.QFileDialog.getSaveFileName(self, 'Select Output File', '*')
        if f:
            self.outputInput.setText(f)

    # API methods
    def activateAPI(self):
        items = self.apiList.selectedItems()
        if items:
            api = items[0].text()
            if MoCGF.py27:
                api = unicode(api)
        self.apiView.clear()
        self.apiView.setText(self.mocgf.apiInfoText(api, 'html'))

    # generator methods
    def exploreGenerators(self):
        QtGui.QDesktopServices.openUrl(QtCore.QUrl.fromLocalFile(self.mocgf.generatorPath))

    def updateGeneratorList(self, rescan=True):
        # get old selection
        selItems = self.generatorList.selectedItems()
        selected = None
        if selItems:
            selected = selItems[0].text()
            if MoCGF.py27:
                seleected = unicode(selected)
        if rescan:
            self.mocgf.rescanGenerators()
        self.generatorList.clear()
        i = 0
        for n in sorted(self.mocgf.generators):
            self.generatorList.addItem(n)
            if n == selected:
                # select previously selected generator
                self.generatorList.item(i).setSelected(True)
            i += 1
        if not self.generatorList.selectedItems():
            # nothing selected, select the first in the list
            self.generatorList.item(0).setSelected(True)

    def activateGenerator(self):
        sel = self.generatorList.selectedItems()
        if sel:
            gen = sel[0].text()
            if MoCGF.py27:
                gen = unicode(gen)
        else:
            return
        self.activeGenerator = self.mocgf.generators[gen]
        self.generatorView.clear()
        self.generatorView.setText(self.activeGenerator.infoText('html'))

    def executeGenerator(self):
        line = self.uriInput.text()
        if MoCGF.py27:
            line = unicode(line)
        uriList = [u.strip() for u in line.split(',')]
        outputFile = str(self.outputInput.text())
        if not outputFile:
            tmp, outputFile = tempfile.mkstemp(suffix='.txt', text=True)
            self.outputInput.setText(outputFile)
        if not os.path.isabs(outputFile):
            outputFile = os.path.abspath(outputFile)
        x = self.activeGenerator.execute(uriList)
        o = open(outputFile, 'w')
        o.write(x.read())
        o.close()
        if self.openOutputButton.isChecked():
            QtGui.QDesktopServices.openUrl(QtCore.QUrl.fromLocalFile(outputFile))

    def closeEvent(self, e):
        # first reset output streams to standard settings
        XStream.reset()
        # save preferences to config file
        if hasattr(self.cfg, 'path'):
            uriList = self.uriInput.text()
            outputFile = self.outputInput.text()
            generator = ''
            tmp = self.generatorList.selectedItems()
            if tmp:
                generator = tmp[0].text()
                if MoCGF.py27:
                    generator = unicode(generator)
            try:
                c = self.cfg
                p = 'PREFERENCES'
                if not c.has_section(p):
                    c.add_section(p)
                c.set(p, 'uriList', uriList)
                c.set(p, 'outputFile', outputFile)
                c.set(p, 'generator', generator)
                with open(c.path, 'w') as configfile:
                    c.write(configfile)
            except:
                # silently ignore errors
                pass
        e.accept()

def main():
    """main function when MoCGF is used with the Qt gui"""

    parser = argparse.ArgumentParser(description=descr)
    grp = parser.add_argument_group('path settings')
    grp.add_argument('-p', '--search-path', metavar='PATH', help='search path for generators (separated by ;)')
    grp = parser.add_argument_group('general information')
    grp.add_argument('-d', '--debug', metavar='LEVEL', help='set debug level and activate messages tab (1...5)')

    args = parser.parse_args()

    # first read config file for default values
    defaults = {
        'GeneratorPath': os.environ.get('MOCGF_GENERATORS', ''),
        'LogLevel' : '0',
    }
    cfg = configparser.ConfigParser(defaults)
    homeVar = {'win32':'USERPROFILE', 'linux':'HOME', 'linux2':'HOME', 'darwin':'HOME'}.get(sys.platform)
    cfgFile = os.path.join(os.environ.get(homeVar, ''), '.MoCGF.cfg')
    if os.path.isfile(cfgFile):
        cfg.read(cfgFile)
        cfg.path = cfgFile

    # generatorPath
    gp = args.search_path or cfg['DEFAULT']['GeneratorPath']
    generatorPath = [p for p in gp.split(';') if p]

    # logLevel
    logLevel = 10 * cfg.getint('DEFAULT', 'LogLevel')
    if args.debug:
        logLevel = 10 * int(args.debug)

    # FIXME: resPath may need to be adjusted after installation,
    # or use pkg_resources? See:
    # http://stackoverflow.com/questions/779495/python-access-data-in-package-subdirectory
    resPath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'res')

    app = QtGui.QApplication(sys.argv)
    mw = MoCGFWidget(app, resPath, cfg, generatorPath, logLevel=logLevel)
    mw.show()
    r = app.exec_()
    return(r)
<|endoftext|>


<commit_before>class BioMolecule(object):
    """
    A generic molecule
    """
    def __init__(self, id, name, mass=None):
        self._id = id
        self.name = name
        self.mass = mass

    @property
    def id(self):
        return self._id

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value

    @property
    def mass(self):
        return self._mass

    @mass.setter
    def mass(self, value):
        # if self.mass and \
        #    not isinstance(value, float) or \
        #    not isinstance(value, int):
        #     raise Exception("mass must be numeric")
#        else:
        self._mass = value

class MRNA(BioMolecule):
    def __init__(self, id, name, sequence, mass=None):
        super(MRNA, self).__init__(id, name, mass)
        self.sequence = sequence

    @property
    def sequence(self):
        return self._sequence

    @sequence.setter
    def sequence(self, value):
        if not isinstance(value, str):
            raise Exception("sequence must be a string")
        # TODO: check for valid nucleotides here
        self._sequence = value
        self.calculate_mass()

    def calculate_mass(self):
        self.mass = 0
        NA_mass = {'a': 1.0, 'u': 2.2, 'g':2.1, 'c':1.3}
        for na in self.sequence:
            self.mass += NA_mass[na]

class Protein(BioMolecule):
    number_of_proteins = 0

    def __init__(self, id, name, sequence, mass=None):
        super(Protein, self).__init__(id, name, mass)
        self.sequence = sequence
        self.number_of_proteins += 1

    def __add__(self, AS):
        self.sequence += AS

    def calculate_mass(self):
        AA_mass = {'a': 1.0, 'v': 2.9, 'f':3.0}
        for aa in self.sequence:
            self.mass += AA_mass[aa]

class Ribosome(BioMolecule):
    def __init__(self, id, name):
        super(Ribosome, self).__init__(id, name)
        self.bound = False
        self.position = None

    def initiate(self, mrna):
        pass

    def elongate(self, mRNA):
        code = {'aug': 'a', 'ugg': 'v', 'ccc': 'f'}
        prot = Protein()
        na_seq = mRNA.sequence
        aa_seq = ''
        for codon_num in range(len(na_seq)/3):
            aa = code[na_seq[0:3]]
            aa_seq += aa
            na_seq = na_seq[3:]

        prot.sequence = aa_seq
        return prot

<commit_msg>adds a working version of translation for exercise<commit_after>import random
import sys

class BioMolecule(object):
    """
    A generic molecule
    """
    def __init__(self, id, name, mass=None):
        self._id = id
        self.name = name
        self.mass = mass

    @property
    def id(self):
        return self._id

    @property
    def name(self):
        return self._name

    @name.setter
    def name(self, value):
        self._name = value

    @property
    def mass(self):
        return self._mass

    @mass.setter
    def mass(self, value):
        # if self.mass and \
        #    not isinstance(value, float) or \
        #    not isinstance(value, int):
        #     raise Exception("mass must be numeric")
#        else:
        self._mass = value

class MRNA(BioMolecule):
    def __init__(self, id, name, sequence, mass=None):
        super(MRNA, self).__init__(id, name, mass)
        self.sequence = sequence
        self.binding = [0]*len(sequence)

    def __getitem__(self, value):
        return self.binding[value]

    def __setitem__(self, key, value):
        self.binding[key] = value

    @property
    def sequence(self):
        return self._sequence

    @sequence.setter
    def sequence(self, value):
        if not isinstance(value, str):
            raise Exception("sequence must be a string")
        # TODO: check for valid nucleotides here
        self._sequence = value.upper()
        self.calculate_mass()

    def calculate_mass(self):
        self.mass = 0
        NA_mass = {'A': 1.0, 'U': 2.2, 'G':2.1, 'C':1.3}
        for na in self.sequence:
            self.mass += NA_mass[na]

class Protein(BioMolecule):
    number_of_proteins = 0

    def __init__(self, id, name, sequence, mass=None):
        super(Protein, self).__init__(id, name, mass)
        self.sequence = sequence
        self.number_of_proteins += 1

    def __add__(self, AS):
        self.sequence += AS

    def calculate_mass(self):
        AA_mass = {'a': 1.0, 'v': 2.9, 'f':3.0}
        for aa in self.sequence:
            self.mass += AA_mass[aa]

class Ribosome(BioMolecule):
    code = dict([('UCA','S'), ('UCG','S'), ('UCC','S'), ('UCU','S'),
                 ('UUU','F'), ('UUC','F'), ('UUA','L'), ('UUG','L'),
                 ('UAU','Y'), ('UAC','Y'), ('UAA','*'), ('UAG','*'),
                 ('UGU','C'), ('UGC','C'), ('UGA','*'), ('UGG','W'),
                 ('CUA','L'), ('CUG','L'), ('CUC','L'), ('CUU','L'),
                 ('CCA','P'), ('CCG','P'), ('CCC','P'), ('CCU','P'),
                 ('CAU','H'), ('CAC','H'), ('CAA','Q'), ('CAG','Q'),
                 ('CGA','R'), ('CGG','R'), ('CGC','R'), ('CGU','R'),
                 ('AUU','I'), ('AUC','I'), ('AUA','I'), ('AUG','M'),
                 ('ACA','T'), ('ACG','T'), ('ACC','T'), ('ACU','T'),
                 ('AAU','N'), ('AAC','N'), ('AAA','K'), ('AAG','K'),
                 ('AGU','S'), ('AGC','S'), ('AGA','R'), ('AGG','R'),
                 ('GUA','V'), ('GUG','V'), ('GUC','V'), ('GUU','V'),
                 ('GCA','A'), ('GCG','A'), ('GCC','A'), ('GCU','A'),
                 ('GAU','D'), ('GAC','D'), ('GAA','E'), ('GAG','E'),
                 ('GGA','G'), ('GGG','G'), ('GGC','G'), ('GGU','G')])

    def __init__(self, id, name):
        super(Ribosome, self).__init__(id, name)
        self.bound = False
        self.position = None

    def initiate(self, mrna):
        if not self.bound and not mrna[0]:  #  no mrna bound yet and target mrna still free at pos 0
            self.bound = mrna
            self.nascent_prot = Protein(self.bound.id,
                                        "Protein_{0}".format(self.bound.id),
                                        [])
            self.position = 0
            self.bound[0] = 1

    def elongate(self):
        if not self.bound: # can't elongate
            return False
        codon = self.bound.sequence[self.position:self.position+3]
        aa = self.code[codon]

        if aa == "*": # terminate at stop codon
            return self.terminate()

        if not self.bound[self.position + 1]: # if the next rna position is free
            self.bound[self.position] = 0
            self.bound[self.position+1] = 1
            self.position += 1
            self.nascent_prot + aa
        return 0

    def terminate(self):
        self.bound[self.position] = 0
        self.bound = False
        return self.nascent_prot

class Cell(object):
    def __init__(self):
        self.ribosomes = [Ribosome(i, 'Ribo_{0}'.format(i)) for i in xrange(200)]
        self.mrnas = [MRNA(i, 'MRNA_{0}'.format(i), "UUUUUUUUUUAA") for i in xrange(20)]
        self.proteins = [[] for x in range(20)]

    def step(self):
        for r in self.ribosomes:
            if not r.bound:
                r.initiate(self.mrnas[random.randint(0,len(self.mrnas)-1)])
            else:
                prot = r.elongate()
                if prot:
                    self.proteins[prot.id].append(prot)

    def simulate(self, steps, p=True):
        for s in xrange(steps):
            self.step()
            if p:
                print [len(x) for x in self.proteins]

if __name__ == "__main__":
    c = Cell()
    c.simulate()
<|endoftext|>


<commit_before># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals
import uuid
import os
from django.test import TestCase

from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.form_processor.tests.utils import use_sql_backend
from corehq.util.test_utils import TestFileMixin

class XMLElementTest(TestCase, TestFileMixin):
    file_path = ('data',)
    root = os.path.dirname(__file__)

    def test_various_encodings(self):
        tests = (
            ('utf-8', 'हिन्दी चट्टानों'),
            ('UTF-8', 'हिन्दी चट्टानों'),
            ('ASCII', 'hello'),
        )
        xml_template = self.get_xml('encoding')

        for encoding, value in tests:
            xml_data = xml_template.format(
                encoding=encoding,
                form_id=uuid.uuid4().hex,
                sample_value=value.encode(encoding),
            )
            xform = submit_form_locally(xml_data, 'test-domain').xform
            self.assertEqual(value, xform.form_data['test'])
            elem = xform.get_xml_element()
            self.assertEqual(value, elem.find('{http://commcarehq.org/couchforms-tests}test').text)

@use_sql_backend
class XMLElementTestSQL(XMLElementTest):
    pass
<commit_msg>fix couchforms/tests/test_xml.py<commit_after># coding=utf-8
from __future__ import absolute_import
from __future__ import unicode_literals
import uuid
import os
from django.test import TestCase

from corehq.apps.receiverwrapper.util import submit_form_locally
from corehq.form_processor.tests.utils import use_sql_backend
from corehq.util.test_utils import TestFileMixin

class XMLElementTest(TestCase, TestFileMixin):
    file_path = ('data',)
    root = os.path.dirname(__file__)

    def test_various_encodings(self):
        tests = (
            ('utf-8', 'हिन्दी चट्टानों'),
            ('UTF-8', 'हिन्दी चट्टानों'),
            ('ASCII', 'hello'),
        )
        xml_template = self.get_xml('encoding').decode('utf-8')

        for encoding, value in tests:
            xml_data = xml_template.format(
                encoding=encoding,
                form_id=uuid.uuid4().hex,
                sample_value=value,
            )
            xform = submit_form_locally(xml_data.encode('utf-8'), 'test-domain').xform
            self.assertEqual(value, xform.form_data['test'])
            elem = xform.get_xml_element()
            self.assertEqual(value, elem.find('{http://commcarehq.org/couchforms-tests}test').text)

@use_sql_backend
class XMLElementTestSQL(XMLElementTest):
    pass
<|endoftext|>