<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

import os
import unittest

from db.alchemy.db_utils import get_uiapi_writer_session
from openquake.output.risk import LossCurveDBWriter, LossMapDBWriter
from openquake.shapes import Site, Curve

from db_tests import helpers

# The data below was captured (and subsequently modified for testing purposes)
# by running
#
#   bin/openquake --config_file=smoketests/classical_psha_simple/config.gem
#
# and putting a breakpoint in openquake/output/risk.py:CurveXMLWriter.write()
RISK_LOSS_CURVE_DATA = [
    (Site(-118.077721, 33.852034),
     (Curve([(3.18e-06, 1.0), (8.81e-06, 1.0), (1.44e-05, 1.0),
             (2.00e-05, 1.0)]),
      {u'assetValue': 5.07, u'assetID': u'a5625',
       u'listDescription': u'Collection of exposure values for ...',
       u'structureCategory': u'RM1L', u'lon': -118.077721,
       u'assetDescription': u'LA building',
       u'vulnerabilityFunctionReference': u'HAZUS_RM1L_LC',
       u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(7.18e-06, 1.0), (1.91e-05, 1.0), (3.12e-05, 1.0),
             (4.32e-05, 1.0)]),
     {u'assetValue': 5.63, u'assetID': u'a5629',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'URML',
      u'lon': -118.077721, u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_URML_LC',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(5.48e-06, 1.0), (1.45e-05, 1.0), (2.36e-05, 1.0),
             (3.27e-05, 1.0)]),
     {u'assetValue': 11.26, u'assetID': u'a5630',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'URML', u'lon': -118.077721,
      u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_URML_LS',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(9.77e-06, 1.0), (2.64e-05, 1.0), (4.31e-05, 1.0),
             (5.98e-05, 1.0)]),
     {u'assetValue': 5.5, u'assetID': u'a5636',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'C3L', u'lon': -118.077721,
      u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_C3L_MC',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),
]

class LossCurveDBWriterTestCase(unittest.TestCase, helpers.DbTestMixin):
    """
    Unit tests for the LossCurveDBWriter class, which serializes
    loss curves to the database.
    """
    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def setUp(self):
        self.job = self.setup_classic_job()
        self.session = get_uiapi_writer_session()
        output_path = self.generate_output_path(self.job)
        self.display_name = os.path.basename(output_path)

        self.writer = LossCurveDBWriter(self.session, output_path, self.job.id)

    def test_serialize(self):
        """All the records are inserted correctly."""
        output = self.writer.output

        # Call the function under test.
        self.writer.serialize(RISK_LOSS_CURVE_DATA)

        # output record
        self.assertEqual(1, len(self.job.output_set))

        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_curve", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # loss curve record
        self.assertEqual(1, len(output.losscurve_set))

        [loss_curve] = output.losscurve_set

        self.assertEqual(loss_curve.unit, 'EUR')
        self.assertEqual(loss_curve.end_branch_label, None)
        self.assertEqual(loss_curve.category, None)

        # loss curve data records
        self.assertEqual(4, len(loss_curve.losscurvedata_set))

        inserted_data = []

        for lcd in loss_curve.losscurvedata_set:
            loc = lcd.location.coords(self.session)

            data = (Site(loc[0], loc[1]),
                    (Curve(zip(lcd.losses, lcd.poes)),
                    {u'assetID': lcd.asset_ref}))

            inserted_data.append(data)

        def normalize(values):
            result = []
            for site, (curve, asset) in values:
                result.append((site,
                               (curve,
                                {'assetID': asset['assetID']})))

            return sorted(result, key=lambda v: v[1][1]['assetID'])

        self.assertEquals(normalize(RISK_LOSS_CURVE_DATA),
                          normalize(inserted_data))

SITE_A = Site(-117.0, 38.0)
SITE_A_ASSET_ONE = {'assetID': 'a1711'}
SITE_A_ASSET_TWO = {'assetID': 'a1712'}

SITE_B = Site(-118.0, 39.0)
SITE_B_ASSET_ONE = {'assetID': 'a1713'}

LOSS_MAP_METADATA = {
    'nrmlID': 'test_nrml_id',
    'riskResultID': 'test_rr_id',
    'lossMapID': 'test_lm_id',
    'endBranchLabel': 'test_ebl',
    'lossCategory': 'economic_loss',
    'unit': 'EUR'}

DETERMINISTIC_LOSS_MAP_METADATA = LOSS_MAP_METADATA.copy()
DETERMINISTIC_LOSS_MAP_METADATA.update({
    'deterministic': True})

SITE_A_DETERMINISTIC_LOSS_ONE = {'mean_loss': 0, 'stddev_loss': 100}
SITE_A_DETERMINISTIC_LOSS_TWO = {'mean_loss': 5, 'stddev_loss': 2000.0}

SITE_B_DETERMINISTIC_LOSS_ONE = {'mean_loss': 120000.0, 'stddev_loss': 2000.0}

SAMPLE_DETERMINISTIC_LOSS_MAP_DATA = [
    DETERMINISTIC_LOSS_MAP_METADATA,
    (SITE_A, [(SITE_A_DETERMINISTIC_LOSS_ONE, SITE_A_ASSET_ONE),
    (SITE_A_DETERMINISTIC_LOSS_TWO, SITE_A_ASSET_TWO)]),
    (SITE_B, [(SITE_B_DETERMINISTIC_LOSS_ONE, SITE_B_ASSET_ONE)])]

NONDETERMINISTIC_LOSS_MAP_METADATA = LOSS_MAP_METADATA.copy()
NONDETERMINISTIC_LOSS_MAP_METADATA.update({
    'poe': 0.6,
    'deterministic': False})

SITE_A_NONDETERMINISTIC_LOSS_ONE = {'value': 12}
SITE_A_NONDETERMINISTIC_LOSS_TWO = {'value': 66}

SITE_B_NONDETERMINISTIC_LOSS_ONE = {'value': 1000.0}

SAMPLE_NONDETERMINISTIC_LOSS_MAP_DATA = [
    NONDETERMINISTIC_LOSS_MAP_METADATA,
    (SITE_A, [(SITE_A_NONDETERMINISTIC_LOSS_ONE, SITE_A_ASSET_ONE),
    (SITE_A_NONDETERMINISTIC_LOSS_TWO, SITE_A_ASSET_TWO)]),
    (SITE_B, [(SITE_B_NONDETERMINISTIC_LOSS_ONE, SITE_B_ASSET_ONE)])]

class LossMapDBWriterTestCase(unittest.TestCase, helpers.DbTestMixin):
    """
    Unit tests for the LossMapDBWriter class, which serializes
    loss maps to the database.
    """
    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def setUp(self):
        self.job = self.setup_classic_job()
        self.session = get_uiapi_writer_session()
        output_path = self.generate_output_path(self.job)
        self.display_name = os.path.basename(output_path)

        self.writer = LossMapDBWriter(self.session, output_path, self.job.id)

    def test_serialize_deterministic(self):
        """
        All the records for deterministic loss maps are inserted correctly.
        """

        output = self.writer.output

        # Call the function under test.
        data = SAMPLE_DETERMINISTIC_LOSS_MAP_DATA
        self.writer.serialize(data)

        # Output record
        self.assertEqual(1, len(self.job.output_set))
        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_map", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # LossMap record
        self.assertEqual(1, len(output.lossmap_set))
        [metadata] = output.lossmap_set
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['deterministic'],
                         metadata.deterministic)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['endBranchLabel'],
                         metadata.end_branch_label)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['lossCategory'],
                         metadata.category)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['unit'],
                         metadata.unit)
        self.assertEqual(None, metadata.poe)

        # LossMapData records
        self.assertEqual(3, len(metadata.lossmapdata_set))
        [data_a, data_b, data_c] = metadata.lossmapdata_set

        self.assertEqual(SITE_A, Site(*data_a.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_ONE['assetID'], data_a.asset_ref)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_ONE['mean_loss'],
                        data_a.value)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_ONE['stddev_loss'],
                         data_a.std_dev)

        self.assertEqual(SITE_A, Site(*data_b.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_TWO['assetID'], data_b.asset_ref)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_TWO['mean_loss'],
                         data_b.value)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_TWO['stddev_loss'],
                         data_b.std_dev)

        self.assertEqual(SITE_B, Site(*data_c.location.coords(self.session)))
        self.assertEqual(SITE_B_ASSET_ONE['assetID'], data_c.asset_ref)
        self.assertEqual(SITE_B_DETERMINISTIC_LOSS_ONE['mean_loss'],
                         data_c.value)
        self.assertEqual(SITE_B_DETERMINISTIC_LOSS_ONE['stddev_loss'],
                         data_c.std_dev)

    def test_serialize_nondeterministic(self):
        """
        All the records for non-deterministic loss maps are inserted correctly.
        """

        output = self.writer.output

        # Call the function under test.
        data = SAMPLE_NONDETERMINISTIC_LOSS_MAP_DATA
        self.writer.serialize(data)

        # Output record
        self.assertEqual(1, len(self.job.output_set))
        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_map", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # LossMap record
        self.assertEqual(1, len(output.lossmap_set))
        [metadata] = output.lossmap_set
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['deterministic'],
                         metadata.deterministic)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['endBranchLabel'],
                         metadata.end_branch_label)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['lossCategory'],
                         metadata.category)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['unit'],
                         metadata.unit)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['poe'],
                         metadata.poe)

        # LossMapData records
        self.assertEqual(3, len(metadata.lossmapdata_set))
        [data_a, data_b, data_c] = metadata.lossmapdata_set

        self.assertEqual(SITE_A, Site(*data_a.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_ONE['assetID'], data_a.asset_ref)
        self.assertEqual(SITE_A_NONDETERMINISTIC_LOSS_ONE['value'],
                         data_a.value)

        self.assertEqual(SITE_A, Site(*data_b.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_TWO['assetID'], data_b.asset_ref)
        self.assertEqual(SITE_A_NONDETERMINISTIC_LOSS_TWO['value'],
                         data_b.value)

        self.assertEqual(SITE_B, Site(*data_c.location.coords(self.session)))
        self.assertEqual(SITE_B_ASSET_ONE['assetID'], data_c.asset_ref)
        self.assertEqual(SITE_B_NONDETERMINISTIC_LOSS_ONE['value'],
                         data_c.value)
<commit_msg>Do not depend on set order (assumes object insert order is the same as the passed-in list).<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2011, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License version 3
# only, as published by the Free Software Foundation.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License version 3 for more details
# (a copy is included in the LICENSE file that accompanied this code).
#
# You should have received a copy of the GNU Lesser General Public License
# version 3 along with OpenQuake.  If not, see
# <http://www.gnu.org/licenses/lgpl-3.0.txt> for a copy of the LGPLv3 License.

import os
import unittest

from db.alchemy.db_utils import get_uiapi_writer_session
from openquake.output.risk import LossCurveDBWriter, LossMapDBWriter
from openquake.shapes import Site, Curve

from db_tests import helpers

# The data below was captured (and subsequently modified for testing purposes)
# by running
#
#   bin/openquake --config_file=smoketests/classical_psha_simple/config.gem
#
# and putting a breakpoint in openquake/output/risk.py:CurveXMLWriter.write()
RISK_LOSS_CURVE_DATA = [
    (Site(-118.077721, 33.852034),
     (Curve([(3.18e-06, 1.0), (8.81e-06, 1.0), (1.44e-05, 1.0),
             (2.00e-05, 1.0)]),
      {u'assetValue': 5.07, u'assetID': u'a5625',
       u'listDescription': u'Collection of exposure values for ...',
       u'structureCategory': u'RM1L', u'lon': -118.077721,
       u'assetDescription': u'LA building',
       u'vulnerabilityFunctionReference': u'HAZUS_RM1L_LC',
       u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(7.18e-06, 1.0), (1.91e-05, 1.0), (3.12e-05, 1.0),
             (4.32e-05, 1.0)]),
     {u'assetValue': 5.63, u'assetID': u'a5629',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'URML',
      u'lon': -118.077721, u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_URML_LC',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(5.48e-06, 1.0), (1.45e-05, 1.0), (2.36e-05, 1.0),
             (3.27e-05, 1.0)]),
     {u'assetValue': 11.26, u'assetID': u'a5630',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'URML', u'lon': -118.077721,
      u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_URML_LS',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),

    (Site(-118.077721, 33.852034),
     (Curve([(9.77e-06, 1.0), (2.64e-05, 1.0), (4.31e-05, 1.0),
             (5.98e-05, 1.0)]),
     {u'assetValue': 5.5, u'assetID': u'a5636',
      u'listDescription': u'Collection of exposure values for ...',
      u'structureCategory': u'C3L', u'lon': -118.077721,
      u'assetDescription': u'LA building',
      u'vulnerabilityFunctionReference': u'HAZUS_C3L_MC',
      u'listID': u'LA01', u'assetValueUnit': 'EUR', u'lat': 33.852034})),
]

class LossCurveDBWriterTestCase(unittest.TestCase, helpers.DbTestMixin):
    """
    Unit tests for the LossCurveDBWriter class, which serializes
    loss curves to the database.
    """
    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def setUp(self):
        self.job = self.setup_classic_job()
        self.session = get_uiapi_writer_session()
        output_path = self.generate_output_path(self.job)
        self.display_name = os.path.basename(output_path)

        self.writer = LossCurveDBWriter(self.session, output_path, self.job.id)

    def test_serialize(self):
        """All the records are inserted correctly."""
        output = self.writer.output

        # Call the function under test.
        self.writer.serialize(RISK_LOSS_CURVE_DATA)

        # output record
        self.assertEqual(1, len(self.job.output_set))

        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_curve", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # loss curve record
        self.assertEqual(1, len(output.losscurve_set))

        [loss_curve] = output.losscurve_set

        self.assertEqual(loss_curve.unit, 'EUR')
        self.assertEqual(loss_curve.end_branch_label, None)
        self.assertEqual(loss_curve.category, None)

        # loss curve data records
        self.assertEqual(4, len(loss_curve.losscurvedata_set))

        inserted_data = []

        for lcd in loss_curve.losscurvedata_set:
            loc = lcd.location.coords(self.session)

            data = (Site(loc[0], loc[1]),
                    (Curve(zip(lcd.losses, lcd.poes)),
                    {u'assetID': lcd.asset_ref}))

            inserted_data.append(data)

        def normalize(values):
            result = []
            for site, (curve, asset) in values:
                result.append((site,
                               (curve,
                                {'assetID': asset['assetID']})))

            return sorted(result, key=lambda v: v[1][1]['assetID'])

        self.assertEquals(normalize(RISK_LOSS_CURVE_DATA),
                          normalize(inserted_data))

SITE_A = Site(-117.0, 38.0)
SITE_A_ASSET_ONE = {'assetID': 'a1711'}
SITE_A_ASSET_TWO = {'assetID': 'a1712'}

SITE_B = Site(-118.0, 39.0)
SITE_B_ASSET_ONE = {'assetID': 'a1713'}

LOSS_MAP_METADATA = {
    'nrmlID': 'test_nrml_id',
    'riskResultID': 'test_rr_id',
    'lossMapID': 'test_lm_id',
    'endBranchLabel': 'test_ebl',
    'lossCategory': 'economic_loss',
    'unit': 'EUR'}

DETERMINISTIC_LOSS_MAP_METADATA = LOSS_MAP_METADATA.copy()
DETERMINISTIC_LOSS_MAP_METADATA.update({
    'deterministic': True})

SITE_A_DETERMINISTIC_LOSS_ONE = {'mean_loss': 0, 'stddev_loss': 100}
SITE_A_DETERMINISTIC_LOSS_TWO = {'mean_loss': 5, 'stddev_loss': 2000.0}

SITE_B_DETERMINISTIC_LOSS_ONE = {'mean_loss': 120000.0, 'stddev_loss': 2000.0}

SAMPLE_DETERMINISTIC_LOSS_MAP_DATA = [
    DETERMINISTIC_LOSS_MAP_METADATA,
    (SITE_A, [(SITE_A_DETERMINISTIC_LOSS_ONE, SITE_A_ASSET_ONE),
    (SITE_A_DETERMINISTIC_LOSS_TWO, SITE_A_ASSET_TWO)]),
    (SITE_B, [(SITE_B_DETERMINISTIC_LOSS_ONE, SITE_B_ASSET_ONE)])]

NONDETERMINISTIC_LOSS_MAP_METADATA = LOSS_MAP_METADATA.copy()
NONDETERMINISTIC_LOSS_MAP_METADATA.update({
    'poe': 0.6,
    'deterministic': False})

SITE_A_NONDETERMINISTIC_LOSS_ONE = {'value': 12}
SITE_A_NONDETERMINISTIC_LOSS_TWO = {'value': 66}

SITE_B_NONDETERMINISTIC_LOSS_ONE = {'value': 1000.0}

SAMPLE_NONDETERMINISTIC_LOSS_MAP_DATA = [
    NONDETERMINISTIC_LOSS_MAP_METADATA,
    (SITE_A, [(SITE_A_NONDETERMINISTIC_LOSS_ONE, SITE_A_ASSET_ONE),
    (SITE_A_NONDETERMINISTIC_LOSS_TWO, SITE_A_ASSET_TWO)]),
    (SITE_B, [(SITE_B_NONDETERMINISTIC_LOSS_ONE, SITE_B_ASSET_ONE)])]

class LossMapDBWriterTestCase(unittest.TestCase, helpers.DbTestMixin):
    """
    Unit tests for the LossMapDBWriter class, which serializes
    loss maps to the database.
    """
    def tearDown(self):
        if hasattr(self, "job") and self.job:
            self.teardown_job(self.job)
        if hasattr(self, "output") and self.output:
            self.teardown_output(self.output)

    def setUp(self):
        self.job = self.setup_classic_job()
        self.session = get_uiapi_writer_session()
        output_path = self.generate_output_path(self.job)
        self.display_name = os.path.basename(output_path)

        self.writer = LossMapDBWriter(self.session, output_path, self.job.id)

    def test_serialize_deterministic(self):
        """
        All the records for deterministic loss maps are inserted correctly.
        """

        output = self.writer.output

        # Call the function under test.
        data = SAMPLE_DETERMINISTIC_LOSS_MAP_DATA
        self.writer.serialize(data)

        # Output record
        self.assertEqual(1, len(self.job.output_set))
        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_map", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # LossMap record
        self.assertEqual(1, len(output.lossmap_set))
        [metadata] = output.lossmap_set
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['deterministic'],
                         metadata.deterministic)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['endBranchLabel'],
                         metadata.end_branch_label)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['lossCategory'],
                         metadata.category)
        self.assertEqual(DETERMINISTIC_LOSS_MAP_METADATA['unit'],
                         metadata.unit)
        self.assertEqual(None, metadata.poe)

        # LossMapData records
        self.assertEqual(3, len(metadata.lossmapdata_set))
        [data_a, data_b, data_c] = sorted(metadata.lossmapdata_set,
                                          key=lambda d: d.id)

        self.assertEqual(SITE_A, Site(*data_a.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_ONE['assetID'], data_a.asset_ref)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_ONE['mean_loss'],
                        data_a.value)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_ONE['stddev_loss'],
                         data_a.std_dev)

        self.assertEqual(SITE_A, Site(*data_b.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_TWO['assetID'], data_b.asset_ref)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_TWO['mean_loss'],
                         data_b.value)
        self.assertEqual(SITE_A_DETERMINISTIC_LOSS_TWO['stddev_loss'],
                         data_b.std_dev)

        self.assertEqual(SITE_B, Site(*data_c.location.coords(self.session)))
        self.assertEqual(SITE_B_ASSET_ONE['assetID'], data_c.asset_ref)
        self.assertEqual(SITE_B_DETERMINISTIC_LOSS_ONE['mean_loss'],
                         data_c.value)
        self.assertEqual(SITE_B_DETERMINISTIC_LOSS_ONE['stddev_loss'],
                         data_c.std_dev)

    def test_serialize_nondeterministic(self):
        """
        All the records for non-deterministic loss maps are inserted correctly.
        """

        output = self.writer.output

        # Call the function under test.
        data = SAMPLE_NONDETERMINISTIC_LOSS_MAP_DATA
        self.writer.serialize(data)

        # Output record
        self.assertEqual(1, len(self.job.output_set))
        [output] = self.job.output_set
        self.assertTrue(output.db_backed)
        self.assertTrue(output.path is None)
        self.assertEqual(self.display_name, output.display_name)
        self.assertEqual("loss_map", output.output_type)
        self.assertTrue(self.job is output.oq_job)

        # LossMap record
        self.assertEqual(1, len(output.lossmap_set))
        [metadata] = output.lossmap_set
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['deterministic'],
                         metadata.deterministic)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['endBranchLabel'],
                         metadata.end_branch_label)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['lossCategory'],
                         metadata.category)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['unit'],
                         metadata.unit)
        self.assertEqual(NONDETERMINISTIC_LOSS_MAP_METADATA['poe'],
                         metadata.poe)

        # LossMapData records
        self.assertEqual(3, len(metadata.lossmapdata_set))
        [data_a, data_b, data_c] = sorted(metadata.lossmapdata_set,
                                          key=lambda d: d.id)

        self.assertEqual(SITE_A, Site(*data_a.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_ONE['assetID'], data_a.asset_ref)
        self.assertEqual(SITE_A_NONDETERMINISTIC_LOSS_ONE['value'],
                         data_a.value)

        self.assertEqual(SITE_A, Site(*data_b.location.coords(self.session)))
        self.assertEqual(SITE_A_ASSET_TWO['assetID'], data_b.asset_ref)
        self.assertEqual(SITE_A_NONDETERMINISTIC_LOSS_TWO['value'],
                         data_b.value)

        self.assertEqual(SITE_B, Site(*data_c.location.coords(self.session)))
        self.assertEqual(SITE_B_ASSET_ONE['assetID'], data_c.asset_ref)
        self.assertEqual(SITE_B_NONDETERMINISTIC_LOSS_ONE['value'],
                         data_c.value)
<|endoftext|>


<commit_before>from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.core.paginator import Paginator, InvalidPage
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.cache import never_cache
from rosetta.polib import pofile
from rosetta.poutil import find_pos, pagination_range
from rosetta import settings as rosetta_settings
import re, os, rosetta, datetime

def home(request):
    """
    Displays a list of messages to be translated
    """

    def fix_nls(in_,out_):
        """Fixes submitted translations by filtering carriage returns and pairing
        newlines at the begging and end of the translated string with the original
        """
        if 0 == len(in_) or 0 == len(out_):
            return out_

        if "\r" in out_ and "\r" not in in_:
            out_=out_.replace("\r",'')

        if "\n" == in_[0] and "\n" != out_[0]:
            out_ = "\n" + out_
        elif "\n" != in_[0] and "\n" == out_[0]:
            out_ = out_.lstrip()
        if "\n" == in_[-1] and "\n" != out_[-1]:
            out_ = out_ + "\n"
        elif "\n" != in_[-1] and "\n" == out_[-1]:
            out_ = out_.rstrip()
        return out_

    version = rosetta.get_version(True)
    if 'rosetta_i18n_fn' in request.session:
        rosetta_i18n_fn=request.session.get('rosetta_i18n_fn')
        rosetta_i18n_pofile = request.session.get('rosetta_i18n_pofile')
        rosetta_i18n_lang_code = request.session['rosetta_i18n_lang_code']
        rosetta_i18n_lang_bidi = (rosetta_i18n_lang_code in settings.LANGUAGES_BIDI)
        rosetta_i18n_write = request.session.get('rosetta_i18n_write', True)

        if 'filter' in request.GET:
            if request.GET.get('filter') == 'untranslated' or request.GET.get('filter') == 'translated' or request.GET.get('filter') == 'both':
                filter_ = request.GET.get('filter')
                request.session['rosetta_i18n_filter'] = filter_
                return HttpResponseRedirect(reverse('rosetta-home'))
        elif 'rosetta_i18n_filter' in request.session:
            rosetta_i18n_filter = request.session.get('rosetta_i18n_filter')
        else:
            rosetta_i18n_filter = 'both'

        if '_next' in request.POST:
            rx=re.compile(r'^m_([0-9]+)')
            rx_plural=re.compile(r'^m_([0-9]+)_([0-9]+)')
            file_change = False
            for k in request.POST.keys():
                if rx_plural.match(k):
                    id=int(rx_plural.match(k).groups()[0])
                    idx=int(rx_plural.match(k).groups()[1])
                    rosetta_i18n_pofile[id].msgstr_plural[str(idx)] = fix_nls(rosetta_i18n_pofile[id].msgid_plural[idx], request.POST.get(k))
                    file_change = True
                elif rx.match(k):
                    id=int(rx.match(k).groups()[0])
                    rosetta_i18n_pofile[id].msgstr = fix_nls(rosetta_i18n_pofile[id].msgid, request.POST.get(k))
                    file_change = True
                if file_change and 'fuzzy' in rosetta_i18n_pofile[id].flags:
                    rosetta_i18n_pofile[id].flags.remove('fuzzy')

            if file_change and rosetta_i18n_write:
                try:
                    rosetta_i18n_pofile.metadata['Last-Translator'] = str("%s %s <%s>" %(request.user.first_name,request.user.last_name,request.user.email))
                    rosetta_i18n_pofile.metadata['X-Translated-Using'] = str("django-rosetta %s" % rosetta.get_version(False))
                    rosetta_i18n_pofile.metadata['PO-Revision-Date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M%z')
                except UnicodeDecodeError:
                    pass
                try:
                    rosetta_i18n_pofile.save()
                    rosetta_i18n_pofile.save_as_mofile(rosetta_i18n_fn.replace('.po','.mo'))

                    # Try auto-reloading
                    if (hasattr(rosetta_settings,'WSGI_AUTO_RELOAD') and rosetta_settings.WSGI_AUTO_RELOAD or \
                        hasattr(settings,'WSGI_AUTO_RELOAD') and settings.WSGI_AUTO_RELOAD) and \
                        request.environ.has_key('mod_wsgi.application_group') and \
                        request.environ.has_key('SCRIPT_FILENAME') and \
                        int(request.environ.get('mod_wsgi.script_reloading', '0')):
                            try:
                                os.utime(request.environ.get('SCRIPT_FILENAME'), None)
                            except OSError:
                                pass

                except:
                    request.session['rosetta_i18n_write'] = False

                request.session['rosetta_i18n_pofile']=rosetta_i18n_pofile

                # Retain query arguments
                query_arg = ''
                if 'query' in request.REQUEST:
                    query_arg = '?query=%s' %request.REQUEST.get('query')
                if 'page' in request.GET:
                    if query_arg:
                        query_arg = query_arg + '&'
                    else:
                        query_arg = '?'
                    query_arg = query_arg + 'page=%d' % int(request.GET.get('page'))

                return HttpResponseRedirect(reverse('rosetta-home') + query_arg)

        rosetta_i18n_lang_name = _(request.session.get('rosetta_i18n_lang_name'))
        rosetta_i18n_lang_code = request.session.get('rosetta_i18n_lang_code')

        if 'query' in request.REQUEST and request.REQUEST.get('query','').strip():
            query=request.REQUEST.get('query').strip()
            rx=re.compile(query, re.IGNORECASE)
            paginator = Paginator([e for e in rosetta_i18n_pofile if rx.search(smart_unicode(e.msgstr)+smart_unicode(e.msgid)+u''.join([o[0] for o in e.occurrences]))], rosetta_settings.MESSAGES_PER_PAGE)
        else:
            if rosetta_i18n_filter == 'both':
                paginator = Paginator(rosetta_i18n_pofile, rosetta_settings.MESSAGES_PER_PAGE)
            elif rosetta_i18n_filter == 'untranslated':
                paginator = Paginator(rosetta_i18n_pofile.untranslated_entries(), rosetta_settings.MESSAGES_PER_PAGE)
            elif rosetta_i18n_filter == 'translated':
                paginator = Paginator(rosetta_i18n_pofile.translated_entries(), rosetta_settings.MESSAGES_PER_PAGE)

        if 'page' in request.GET and int(request.GET.get('page')) <= paginator.num_pages and int(request.GET.get('page')) > 0:
            page = int(request.GET.get('page'))
        else:
            page = 1
        messages = paginator.page(page).object_list
        needs_pagination = paginator.num_pages > 1
        if needs_pagination:
            if paginator.num_pages >= 10:
                page_range = pagination_range(1, paginator.num_pages, page)
            else:
                page_range = range(1,1+paginator.num_pages)
        ADMIN_MEDIA_PREFIX = settings.ADMIN_MEDIA_PREFIX
        ENABLE_TRANSLATION_SUGGESTIONS = rosetta_settings.ENABLE_TRANSLATION_SUGGESTIONS

        return render_to_response('rosetta/pofile.html', locals())

    else:
        return list_languages(request)
home=user_passes_test(lambda user:can_translate(user),'/admin/')(home)
home=never_cache(home)

def download_file(request):
    import zipfile, tempfile, os
    # original filename
    rosetta_i18n_fn=request.session.get('rosetta_i18n_fn', None)
    # in-session modified catalog
    rosetta_i18n_pofile = request.session.get('rosetta_i18n_pofile', None)
    # language code
    rosetta_i18n_lang_code = request.session.get('rosetta_i18n_lang_code', None)

    if not rosetta_i18n_lang_code or not rosetta_i18n_pofile or not rosetta_i18n_fn:
        return HttpResponseRedirect(reverse('rosetta-home'))
    try:
        if len(rosetta_i18n_fn.split('/')) >= 5:
            offered_fn = '_'.join(rosetta_i18n_fn.split('/')[-5:])
        else:
            offered_fn = rosetta_i18n_fn.split('/')[-1]
        # filenames
        tmpdir=tempfile.gettempdir()
        zip_fn = str(os.path.join(tmpdir,'%s.%s.zip' %(offered_fn,rosetta_i18n_lang_code)))
        po_fn = str(os.path.join(tmpdir, rosetta_i18n_fn.split('/')[-1]))
        mo_fn = str(po_fn.replace('.po','.mo')) # not so smart, huh
        rosetta_i18n_pofile.save(po_fn)
        rosetta_i18n_pofile.save_as_mofile(mo_fn)
        zf = zipfile.ZipFile(zip_fn,'w')
        zf.write(po_fn, str(po_fn.split('/')[-1]))
        zf.write(mo_fn, str(mo_fn.split('/')[-1]))
        zf.close()

        response = HttpResponse(file(zip_fn).read())
        response['Content-Disposition'] = 'attachment; filename=%s.%s.zip' %(offered_fn,rosetta_i18n_lang_code)
        response['Content-Type'] = 'application/x-zip'

        os.unlink(zip_fn)
        os.unlink(po_fn)
        os.unlink(mo_fn)

        return response
    except Exception, e:
        return HttpResponseRedirect(reverse('rosetta-home'))
        #return HttpResponse(e, mimetype="text/plain")
download_file=user_passes_test(lambda user:can_translate(user),'/admin/')(download_file)
download_file=never_cache(download_file)

def list_languages(request):
    """
    Lists the languages for the current project, the gettext catalog files
    that can be translated and their translation progress
    """
    languages = []
    do_django = 'django' in request.GET
    do_rosetta = 'rosetta' in request.GET
    has_pos = False
    for language in settings.LANGUAGES:
        pos = find_pos(language[0],include_djangos=do_django,include_rosetta=do_rosetta)
        has_pos = has_pos or len(pos)
        languages.append(
            (language[0],
            _(language[1]),
            [(os.path.realpath(l), pofile(l)) for l in  pos],
            )
        )
    ADMIN_MEDIA_PREFIX = settings.ADMIN_MEDIA_PREFIX
    version = rosetta.get_version(True)
    return render_to_response('rosetta/languages.html', locals())
list_languages=user_passes_test(lambda user:can_translate(user),'/admin/')(list_languages)
list_languages=never_cache(list_languages)

def lang_sel(request,langid,idx):
    """
    Selects a file to be translated
    """
    if langid not in [l[0] for l in settings.LANGUAGES]:
        raise Http404
    else:

        do_django = 'django' in request.GET
        do_rosetta = 'rosetta' in request.GET

        file_ = find_pos(langid,include_djangos=do_django,include_rosetta=do_rosetta)[int(idx)]

        request.session['rosetta_i18n_lang_code'] = langid
        request.session['rosetta_i18n_lang_name'] = unicode([l[1] for l in settings.LANGUAGES if l[0] == langid][0])
        request.session['rosetta_i18n_fn'] = file_
        po = pofile(file_)
        for i in range(len(po)):
            po[i].id = i

        request.session['rosetta_i18n_pofile'] = po
        try:
            os.utime(file_,None)
            request.session['rosetta_i18n_write'] = True
        except OSError:
            request.session['rosetta_i18n_write'] = False

        return HttpResponseRedirect(reverse('rosetta-home'))
lang_sel=user_passes_test(lambda user:can_translate(user),'/admin/')(lang_sel)
lang_sel=never_cache(lang_sel)

def can_translate(user):
    if not user.is_authenticated():
        return False
    elif user.is_superuser:
        return True
    else:
        try:
            from django.contrib.auth.models import Group
            translators = Group.objects.get(name='translators')
            return translators in user.groups.all()
        except Group.DoesNotExist:
            return False

<commit_msg>even better test to see whether WSGI supports reloading<commit_after>from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.core.paginator import Paginator, InvalidPage
from django.core.urlresolvers import reverse
from django.http import Http404, HttpResponseRedirect, HttpResponse
from django.shortcuts import render_to_response
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _
from django.views.decorators.cache import never_cache
from rosetta.polib import pofile
from rosetta.poutil import find_pos, pagination_range
from rosetta import settings as rosetta_settings
import re, os, rosetta, datetime

def home(request):
    """
    Displays a list of messages to be translated
    """

    def fix_nls(in_,out_):
        """Fixes submitted translations by filtering carriage returns and pairing
        newlines at the begging and end of the translated string with the original
        """
        if 0 == len(in_) or 0 == len(out_):
            return out_

        if "\r" in out_ and "\r" not in in_:
            out_=out_.replace("\r",'')

        if "\n" == in_[0] and "\n" != out_[0]:
            out_ = "\n" + out_
        elif "\n" != in_[0] and "\n" == out_[0]:
            out_ = out_.lstrip()
        if "\n" == in_[-1] and "\n" != out_[-1]:
            out_ = out_ + "\n"
        elif "\n" != in_[-1] and "\n" == out_[-1]:
            out_ = out_.rstrip()
        return out_

    version = rosetta.get_version(True)
    if 'rosetta_i18n_fn' in request.session:
        rosetta_i18n_fn=request.session.get('rosetta_i18n_fn')
        rosetta_i18n_pofile = request.session.get('rosetta_i18n_pofile')
        rosetta_i18n_lang_code = request.session['rosetta_i18n_lang_code']
        rosetta_i18n_lang_bidi = (rosetta_i18n_lang_code in settings.LANGUAGES_BIDI)
        rosetta_i18n_write = request.session.get('rosetta_i18n_write', True)

        if 'filter' in request.GET:
            if request.GET.get('filter') == 'untranslated' or request.GET.get('filter') == 'translated' or request.GET.get('filter') == 'both':
                filter_ = request.GET.get('filter')
                request.session['rosetta_i18n_filter'] = filter_
                return HttpResponseRedirect(reverse('rosetta-home'))
        elif 'rosetta_i18n_filter' in request.session:
            rosetta_i18n_filter = request.session.get('rosetta_i18n_filter')
        else:
            rosetta_i18n_filter = 'both'

        if '_next' in request.POST:
            rx=re.compile(r'^m_([0-9]+)')
            rx_plural=re.compile(r'^m_([0-9]+)_([0-9]+)')
            file_change = False
            for k in request.POST.keys():
                if rx_plural.match(k):
                    id=int(rx_plural.match(k).groups()[0])
                    idx=int(rx_plural.match(k).groups()[1])
                    rosetta_i18n_pofile[id].msgstr_plural[str(idx)] = fix_nls(rosetta_i18n_pofile[id].msgid_plural[idx], request.POST.get(k))
                    file_change = True
                elif rx.match(k):
                    id=int(rx.match(k).groups()[0])
                    rosetta_i18n_pofile[id].msgstr = fix_nls(rosetta_i18n_pofile[id].msgid, request.POST.get(k))
                    file_change = True
                if file_change and 'fuzzy' in rosetta_i18n_pofile[id].flags:
                    rosetta_i18n_pofile[id].flags.remove('fuzzy')

            if file_change and rosetta_i18n_write:
                try:
                    rosetta_i18n_pofile.metadata['Last-Translator'] = str("%s %s <%s>" %(request.user.first_name,request.user.last_name,request.user.email))
                    rosetta_i18n_pofile.metadata['X-Translated-Using'] = str("django-rosetta %s" % rosetta.get_version(False))
                    rosetta_i18n_pofile.metadata['PO-Revision-Date'] = datetime.datetime.now().strftime('%Y-%m-%d %H:%M%z')
                except UnicodeDecodeError:
                    pass
                try:
                    rosetta_i18n_pofile.save()
                    rosetta_i18n_pofile.save_as_mofile(rosetta_i18n_fn.replace('.po','.mo'))

                    # Try auto-reloading via the WSGI daemon mode reload mechanism
                    if (hasattr(rosetta_settings,'WSGI_AUTO_RELOAD') and rosetta_settings.WSGI_AUTO_RELOAD or \
                        hasattr(settings,'WSGI_AUTO_RELOAD') and settings.WSGI_AUTO_RELOAD) and \
                        request.environ.has_key('mod_wsgi.process_group') and \
                        request.environ.get('mod_wsgi.process_group',None) and \
                        request.environ.has_key('SCRIPT_FILENAME') and \
                        int(request.environ.get('mod_wsgi.script_reloading', '0')):
                            try:
                                os.utime(request.environ.get('SCRIPT_FILENAME'), None)
                            except OSError:
                                pass

                except:
                    request.session['rosetta_i18n_write'] = False

                request.session['rosetta_i18n_pofile']=rosetta_i18n_pofile

                # Retain query arguments
                query_arg = ''
                if 'query' in request.REQUEST:
                    query_arg = '?query=%s' %request.REQUEST.get('query')
                if 'page' in request.GET:
                    if query_arg:
                        query_arg = query_arg + '&'
                    else:
                        query_arg = '?'
                    query_arg = query_arg + 'page=%d' % int(request.GET.get('page'))

                return HttpResponseRedirect(reverse('rosetta-home') + query_arg)

        rosetta_i18n_lang_name = _(request.session.get('rosetta_i18n_lang_name'))
        rosetta_i18n_lang_code = request.session.get('rosetta_i18n_lang_code')

        if 'query' in request.REQUEST and request.REQUEST.get('query','').strip():
            query=request.REQUEST.get('query').strip()
            rx=re.compile(query, re.IGNORECASE)
            paginator = Paginator([e for e in rosetta_i18n_pofile if rx.search(smart_unicode(e.msgstr)+smart_unicode(e.msgid)+u''.join([o[0] for o in e.occurrences]))], rosetta_settings.MESSAGES_PER_PAGE)
        else:
            if rosetta_i18n_filter == 'both':
                paginator = Paginator(rosetta_i18n_pofile, rosetta_settings.MESSAGES_PER_PAGE)
            elif rosetta_i18n_filter == 'untranslated':
                paginator = Paginator(rosetta_i18n_pofile.untranslated_entries(), rosetta_settings.MESSAGES_PER_PAGE)
            elif rosetta_i18n_filter == 'translated':
                paginator = Paginator(rosetta_i18n_pofile.translated_entries(), rosetta_settings.MESSAGES_PER_PAGE)

        if 'page' in request.GET and int(request.GET.get('page')) <= paginator.num_pages and int(request.GET.get('page')) > 0:
            page = int(request.GET.get('page'))
        else:
            page = 1
        messages = paginator.page(page).object_list
        needs_pagination = paginator.num_pages > 1
        if needs_pagination:
            if paginator.num_pages >= 10:
                page_range = pagination_range(1, paginator.num_pages, page)
            else:
                page_range = range(1,1+paginator.num_pages)
        ADMIN_MEDIA_PREFIX = settings.ADMIN_MEDIA_PREFIX
        ENABLE_TRANSLATION_SUGGESTIONS = rosetta_settings.ENABLE_TRANSLATION_SUGGESTIONS

        return render_to_response('rosetta/pofile.html', locals())

    else:
        return list_languages(request)
home=user_passes_test(lambda user:can_translate(user),'/admin/')(home)
home=never_cache(home)

def download_file(request):
    import zipfile, tempfile, os
    # original filename
    rosetta_i18n_fn=request.session.get('rosetta_i18n_fn', None)
    # in-session modified catalog
    rosetta_i18n_pofile = request.session.get('rosetta_i18n_pofile', None)
    # language code
    rosetta_i18n_lang_code = request.session.get('rosetta_i18n_lang_code', None)

    if not rosetta_i18n_lang_code or not rosetta_i18n_pofile or not rosetta_i18n_fn:
        return HttpResponseRedirect(reverse('rosetta-home'))
    try:
        if len(rosetta_i18n_fn.split('/')) >= 5:
            offered_fn = '_'.join(rosetta_i18n_fn.split('/')[-5:])
        else:
            offered_fn = rosetta_i18n_fn.split('/')[-1]
        # filenames
        tmpdir=tempfile.gettempdir()
        zip_fn = str(os.path.join(tmpdir,'%s.%s.zip' %(offered_fn,rosetta_i18n_lang_code)))
        po_fn = str(os.path.join(tmpdir, rosetta_i18n_fn.split('/')[-1]))
        mo_fn = str(po_fn.replace('.po','.mo')) # not so smart, huh
        rosetta_i18n_pofile.save(po_fn)
        rosetta_i18n_pofile.save_as_mofile(mo_fn)
        zf = zipfile.ZipFile(zip_fn,'w')
        zf.write(po_fn, str(po_fn.split('/')[-1]))
        zf.write(mo_fn, str(mo_fn.split('/')[-1]))
        zf.close()

        response = HttpResponse(file(zip_fn).read())
        response['Content-Disposition'] = 'attachment; filename=%s.%s.zip' %(offered_fn,rosetta_i18n_lang_code)
        response['Content-Type'] = 'application/x-zip'

        os.unlink(zip_fn)
        os.unlink(po_fn)
        os.unlink(mo_fn)

        return response
    except Exception, e:
        return HttpResponseRedirect(reverse('rosetta-home'))
        #return HttpResponse(e, mimetype="text/plain")
download_file=user_passes_test(lambda user:can_translate(user),'/admin/')(download_file)
download_file=never_cache(download_file)

def list_languages(request):
    """
    Lists the languages for the current project, the gettext catalog files
    that can be translated and their translation progress
    """
    languages = []
    do_django = 'django' in request.GET
    do_rosetta = 'rosetta' in request.GET
    has_pos = False
    for language in settings.LANGUAGES:
        pos = find_pos(language[0],include_djangos=do_django,include_rosetta=do_rosetta)
        has_pos = has_pos or len(pos)
        languages.append(
            (language[0],
            _(language[1]),
            [(os.path.realpath(l), pofile(l)) for l in  pos],
            )
        )
    ADMIN_MEDIA_PREFIX = settings.ADMIN_MEDIA_PREFIX
    version = rosetta.get_version(True)
    return render_to_response('rosetta/languages.html', locals())
list_languages=user_passes_test(lambda user:can_translate(user),'/admin/')(list_languages)
list_languages=never_cache(list_languages)

def lang_sel(request,langid,idx):
    """
    Selects a file to be translated
    """
    if langid not in [l[0] for l in settings.LANGUAGES]:
        raise Http404
    else:

        do_django = 'django' in request.GET
        do_rosetta = 'rosetta' in request.GET

        file_ = find_pos(langid,include_djangos=do_django,include_rosetta=do_rosetta)[int(idx)]

        request.session['rosetta_i18n_lang_code'] = langid
        request.session['rosetta_i18n_lang_name'] = unicode([l[1] for l in settings.LANGUAGES if l[0] == langid][0])
        request.session['rosetta_i18n_fn'] = file_
        po = pofile(file_)
        for i in range(len(po)):
            po[i].id = i

        request.session['rosetta_i18n_pofile'] = po
        try:
            os.utime(file_,None)
            request.session['rosetta_i18n_write'] = True
        except OSError:
            request.session['rosetta_i18n_write'] = False

        return HttpResponseRedirect(reverse('rosetta-home'))
lang_sel=user_passes_test(lambda user:can_translate(user),'/admin/')(lang_sel)
lang_sel=never_cache(lang_sel)

def can_translate(user):
    if not user.is_authenticated():
        return False
    elif user.is_superuser:
        return True
    else:
        try:
            from django.contrib.auth.models import Group
            translators = Group.objects.get(name='translators')
            return translators in user.groups.all()
        except Group.DoesNotExist:
            return False

<|endoftext|>


<commit_before>import datetime
from flask import Flask, g, render_template
from flask.ext.sqlalchemy import SQLAlchemy
import logging

telomere = Flask(__name__)
telomere.config.from_object('app.settings')
telomere.secret_key = telomere.config['SECRET_KEY']

if not telomere.debug:
    import logging
    from logging.handlers import SMTPHandler
    mail_handler = SMTPHandler(telomere.config['SMTP_SERVER'],
                               telomere.config['APPLICATION_EMAIL_ADDRESSES'],
                               telomere.config['ADMIN_EMAIL_ADDRESSES'],
                               telomere.config['ERROR_EMAIL_SUBJECT'])
    mail_handler.setLevel(logging.ERROR)
    telomere.logger.addHandler(mail_handler)

@telomere.errorhandler(500)
@telomere.errorhandler(Exception)
def internal_error(exception):
    telomere.logger.error(sys.exc_info())
    return render_template('500.html'), 500

db = SQLAlchemy(telomere)

import app.database
database.init_db()

@telomere.before_request
def set_date():
    g.year = datetime.datetime.now().year

import app.helpers.templateFilters

from app.views import *
<commit_msg>Try to get some extra information about the exception<commit_after>import datetime
from flask import Flask, g, render_template
from flask.ext.sqlalchemy import SQLAlchemy
import logging

telomere = Flask(__name__)
telomere.config.from_object('app.settings')
telomere.secret_key = telomere.config['SECRET_KEY']

if not telomere.debug:
    import logging
    from logging.handlers import SMTPHandler
    mail_handler = SMTPHandler(telomere.config['SMTP_SERVER'],
                               telomere.config['APPLICATION_EMAIL_ADDRESSES'],
                               telomere.config['ADMIN_EMAIL_ADDRESSES'],
                               telomere.config['ERROR_EMAIL_SUBJECT'])
    mail_handler.setLevel(logging.ERROR)
    telomere.logger.addHandler(mail_handler)

@telomere.errorhandler(500)
@telomere.errorhandler(Exception)
def internal_error(exception):
    telomere.logger.error(sys.exc_info()[0])
    return render_template('500.html'), 500

db = SQLAlchemy(telomere)

import app.database
database.init_db()

@telomere.before_request
def set_date():
    g.year = datetime.datetime.now().year

import app.helpers.templateFilters

from app.views import *
<|endoftext|>


<commit_before>
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
from StringIO import StringIO
from selenium.common.exceptions import NoSuchElementException
import openpyxl
import argparse as arg
import sys
import csv
import os

def init_driver():
    driver = webdriver.Firefox()
    driver.wait = WebDriverWait(driver,1)
    return driver

def lookup(driver, query):
    driver.get("http://www.opensecrets.org/indivs/")
    try:
        button = driver.wait.until(EC.element_to_be_clickable(
            (By.ID, "name")))  #find donor search box
        button.click()           #Click input box
        _input = driver.wait.until(EC.element_to_be_clickable(
                (By.ID, "name")))    #Iput text saving space
        _input.send_keys(query)  #Send "Yoon" text
        _id = driver.wait.until(EC.element_to_be_clickable(
                (By.NAME, "submit"))) #Find search botton
        _id.click()                     #Click saerch botton
    except TimeoutException:     #Error handling
        print("Box or Button not found in google.com")

def updateDriver(driver,root, name):
    isEnd = True
    try:
        for child in root:
            url = child.xpath("@href")
            if len(url) == 1:
               isEnd = True
               text = child.text.strip()
               if text.strip() == "Next":
                 isEnd = False
                 print(url[0])
                 print(child.text)
                 url = "http://www.opensecrets.org/indivs/"+url[0]
                 try:
                     driver.get(url)
                 except:
                     print '%s Not Found' %name
                     driver.quit()
    except:
         print '%s Not Found' %name
         isEnd = True

    return isEnd

def getXML(driver):
    parser = etree.HTMLParser()
    try:
        html = driver.execute_script("return document.documentElement.outerHTML")
        tree = etree.parse(StringIO(html), parser)
        root = tree.find("//*[@class='pageCtrl']")
    except NoSuchElementException:
        driver.quit()
        print("Name not found")
        sys.exit(0)
    return root

def scrap(driver):
    driver.current_url  #Getting current url
    data = []           #Container for table data
    for tr in driver.find_elements_by_xpath('//table[@id="top"]//tr'): #loop table id top
        tds = tr.find_elements_by_tag_name('td')
        if tds:
            data.append([td.text for td in tds])
    return data

def iter_scrap(driver,name):
    container = []
    endPage = False
    while not endPage:
        root = getXML(driver)
        print( driver.current_url )
        container.append(scrap(driver))
        endPage = updateDriver(driver,root,name)
        print( 'Is end page? %s' %endPage)
    return container

def flatten(xs):
    result = []
    if isinstance(xs, (list, tuple)):
        for x in xs:
            result.extend(flatten(x))
    else:
        result.append(xs)
    return result

def save_file(name,data):
    name = name.replace(' ','_')
    save_root = "./save"
    if not os.path.exists(save_root):
        os.makedirs(save_root)
    name = name+".csv"
    csvfile = "./save/"+name
    with open(csvfile, "w") as output:
        for infos in data:
            for info in infos:
                info[0] = info[0].replace('\n',' ')
            writer = csv.writer(output, lineterminator='\n')
            writer.writerows(infos)
    print( 'Saved as'+name)

def joinName(arg):
    len_ = len(arg)
    name =''
    for i in range(1,len_):
        name = name + arg[i]
        name = name + ' '
    return name

def importCEO(exle, start_row, max_row, limit):
    wb = openpyxl.load_workbook(exle)
    sheet_name = wb.get_sheet_names()
    sheet = wb.get_sheet_by_name(sheet_name[0])
    con = []
    for i in range(start_row,max_row):
	    cell = 'A'+str(i)
	    n = sheet[cell].value
	    if n:
		con.append(n)
    con2 = []
    for co in con:
	tmp = co.split()
	tmp2 = ''
	for tm in tmp:
	    if len(tm)>2 or not limit:
		tmp2 = tmp2 + tm
                tmp2 = tmp2 + ' '
	con2.append(tmp2.strip())

    return con2

if __name__ == "__main__":
    parser = arg.ArgumentParser()
    parser.add_argument('--limit', default = False, type = bool, help = 'Ignore len 2 string in excel file')
    parser.add_argument('--max_row', default = 3000, type = int,help = 'Limit reding max row from excel sheet')
    parser.add_argument('--start_row', default = 1, type = int, help = 'Start from which row in excel sheet')
    parser.add_argument('--filename', default = '', type = str, help = 'Name of excel file to read')
    args = parser.parse_args()
    con = importCEO(args.filename,args.start_row,args.max_row,args.limit)
    for co in con:
	driver = init_driver()
	name = co
	lookup(driver, name)
	data = iter_scrap(driver,name)
	driver.quit()
	save_file(name,data)

<commit_msg>fix range error<commit_after>
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
from StringIO import StringIO
from selenium.common.exceptions import NoSuchElementException
import openpyxl
import argparse as arg
import sys
import csv
import os

def init_driver():
    driver = webdriver.Firefox()
    driver.wait = WebDriverWait(driver,1)
    return driver

def lookup(driver, query):
    driver.get("http://www.opensecrets.org/indivs/")
    try:
        button = driver.wait.until(EC.element_to_be_clickable(
            (By.ID, "name")))  #find donor search box
        button.click()           #Click input box
        _input = driver.wait.until(EC.element_to_be_clickable(
                (By.ID, "name")))    #Iput text saving space
        _input.send_keys(query)  #Send "Yoon" text
        _id = driver.wait.until(EC.element_to_be_clickable(
                (By.NAME, "submit"))) #Find search botton
        _id.click()                     #Click saerch botton
    except TimeoutException:     #Error handling
        print("Box or Button not found in google.com")

def updateDriver(driver,root, name):
    isEnd = True
    try:
        for child in root:
            url = child.xpath("@href")
            if len(url) == 1:
               isEnd = True
               text = child.text.strip()
               if text.strip() == "Next":
                 isEnd = False
                 print(url[0])
                 print(child.text)
                 url = "http://www.opensecrets.org/indivs/"+url[0]
                 try:
                     driver.get(url)
                 except:
                     print '%s Not Found' %name
                     driver.quit()
    except:
         print '%s Not Found' %name
         isEnd = True

    return isEnd

def getXML(driver):
    parser = etree.HTMLParser()
    try:
        html = driver.execute_script("return document.documentElement.outerHTML")
        tree = etree.parse(StringIO(html), parser)
        root = tree.find("//*[@class='pageCtrl']")
    except NoSuchElementException:
        driver.quit()
        print("Name not found")
        sys.exit(0)
    return root

def scrap(driver):
    driver.current_url  #Getting current url
    data = []           #Container for table data
    for tr in driver.find_elements_by_xpath('//table[@id="top"]//tr'): #loop table id top
        tds = tr.find_elements_by_tag_name('td')
        if tds:
            data.append([td.text for td in tds])
    return data

def iter_scrap(driver,name):
    container = []
    endPage = False
    while not endPage:
        root = getXML(driver)
        print( driver.current_url )
        container.append(scrap(driver))
        endPage = updateDriver(driver,root,name)
        print( 'Is end page? %s' %endPage)
    return container

def flatten(xs):
    result = []
    if isinstance(xs, (list, tuple)):
        for x in xs:
            result.extend(flatten(x))
    else:
        result.append(xs)
    return result

def save_file(name,data):
    name = name.replace(' ','_')
    save_root = "./save"
    if not os.path.exists(save_root):
        os.makedirs(save_root)
    name = name+".csv"
    csvfile = "./save/"+name
    with open(csvfile, "w") as output:
        for infos in data:
            for info in infos:
                info[0] = info[0].replace('\n',' ')
            writer = csv.writer(output, lineterminator='\n')
            writer.writerows(infos)
    print( 'Saved as'+name)

def joinName(arg):
    len_ = len(arg)
    name =''
    for i in range(1,len_):
        name = name + arg[i]
        name = name + ' '
    return name

def importCEO(exle, start_row, max_row, limit):
    wb = openpyxl.load_workbook(exle)
    sheet_name = wb.get_sheet_names()
    sheet = wb.get_sheet_by_name(sheet_name[0])
    con = []
    for i in range(start_row,max_row+1):
	    cell = 'A'+str(i)
	    n = sheet[cell].value
	    if n:
		con.append(n)
    con2 = []
    for co in con:
	tmp = co.split()
	tmp2 = ''
	for tm in tmp:
	    if len(tm)>2 or not limit:
		tmp2 = tmp2 + tm
                tmp2 = tmp2 + ' '
	con2.append(tmp2.strip())

    return con2

if __name__ == "__main__":
    parser = arg.ArgumentParser()
    parser.add_argument('--limit', default = False, type = bool, help = 'Ignore len 2 string in excel file')
    parser.add_argument('--max_row', default = 3000, type = int,help = 'Limit reding max row from excel sheet')
    parser.add_argument('--start_row', default = 1, type = int, help = 'Start from which row in excel sheet')
    parser.add_argument('--filename', default = '', type = str, help = 'Name of excel file to read')
    args = parser.parse_args()
    con = importCEO(args.filename,args.start_row,args.max_row,args.limit)
    for co in con:
	driver = init_driver()
	name = co
	lookup(driver, name)
	data = iter_scrap(driver,name)
	driver.quit()
	save_file(name,data)

<|endoftext|>


<commit_before>import logging
from collections import Counter

import discord.utils
import discord.ext.commands as commands

import paths
from .util import checks, config, utils

log = logging.getLogger(__name__)

def setup(bot):
    bot.add_cog(Admin(bot))

class IgnoredConfig(config.ConfigElement):
    def __init__(self, **kwargs):
        self.channels = kwargs.pop('channels', [])
        self.guilds = kwargs.pop('guilds', [])
        self.users = utils.dict_keys_to_int(kwargs.pop('users', {}))

class Admin:
    """Bot management commands and events."""
    def __init__(self, bot):
        self.commands_used = Counter()
        self.ignored = config.Config(paths.IGNORED_CONFIG, encoding='utf-8')
        self.bot = bot

    def __global_check_once(self, ctx):
        """A global check used on every command."""
        author = ctx.author
        guild = ctx.guild
        if author == ctx.bot.owner:
            return True

        if guild is not None:
            # Check if we're ignoring the guild
            if guild.id in self.ignored.guilds:
                return False

            if ctx.channel.id in self.ignored.channels:
                return False

            # Guild owners can't be ignored
            if author.id == guild.owner.id:
                return True

            # Check if the user is banned from using the bot
            if author.id in self.ignored.users.get(guild.id, {}):
                return False

            # Check if the channel is banned, bypass this if the user has the manage guild permission
            channel = ctx.channel
            perms = channel.permissions_for(author)
            if not perms.manage_guild and channel.id in self.ignored.channels:
                return False
        return True

    async def resolve_target(self, ctx, target):
        if target == 'channel':
            return ctx.channel, self.ignored.channels
        elif target == 'guild' or target == 'server':
            return ctx.guild, self.ignored.guilds

        # Try converting to a text channel
        try:
            channel = await commands.TextChannelConverter().convert(ctx, target)
        except commands.BadArgument:
            pass
        else:
            return channel, self.ignored.channels

        # Try converting to a user
        try:
            member = await commands.MemberConverter().convert(ctx, target)
        except commands.BadArgument:
            pass
        else:
            guild_id = ctx.guild.id
            try:
                return member, self.ignored.users[guild_id]
            except KeyError:
                self.ignored.users[guild_id] = []
                return member, self.ignored.users[guild_id]

        # Convert to a guild
        try:
            guild = await utils.GuildConverter().convert(ctx, target)
        except:
            pass
        else:
            return guild, self.ignored.guilds

        # Nope
        raise commands.BadArgument(f'"{target}" not found.')

    def validate_ignore_target(self, ctx, target):
        owner_id = ctx.bot.owner.id
        # Only let the bot owner unignore a guild owner
        if isinstance(target, discord.Member):
            # Do not ignore the bot owner
            if owner_id == target.id:
                raise commands.BadArgument('Cannot ignore/unignore the bot owner.')

            # Only allow the bot owner to unignore the guild owner
            if target.id == ctx.guild.owner.id and ctx.author.id != owner_id:
                raise commands.BadArgument('Only the bot owner can ignore/unignore the owner of a server.')
        elif isinstance(target, discord.Guild):
            # Only allow the bot owner to ignore guilds
            if ctx.author.id != owner_id:
                raise commands.BadArgument('Only the bot owner can ignore/unignore servers.')
        elif isinstance(target, discord.VoiceChannel):
            # Do not ignore voice channels
            raise commands.BadArgument('Cannot ignore/unignore voice channels.')

    @commands.command(aliases=['checkpermissions'])
    async def checkperms(self, ctx):
        name = ctx.bot.user.name
        perms_str = 'Read Messages, Send Messages, Manage Messages, Embed Links, Read Message History, Use External Emojis, Add Reactions'
        perms = discord.Permissions(486464)

        # Check the integration role
        role = discord.utils.get(ctx.guild.roles, name=name)
        if not role or not perms <= role.permissions:
            raise commands.UserInputError(f'Please make sure the integration role `{name}` has all the following permissions and is added to the bot :\n'
                                          f'{perms_str}.\n'
                                          f'Note that you can also kick and re-invite the bot with its default permissions.')

        # Check every channel for overwrites
        failed = []
        for channel in sorted(ctx.guild.text_channels, key=lambda c: c.position):
            if not perms <= channel.permissions_for(ctx.guild.me):
                failed.append(channel)
        if failed:
            raise commands.UserInputError(f'Please make sure the channel permissions overwrites for the following channels do not remove these permissions from the bot :\n'
                                          f'{perms_str}.\n'
                                          f'{" ".join(c.mention for c in failed)}')

        await ctx.send('All good.')

    @commands.command()
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def ignore(self, ctx, *, target):
        """Ignores a channel, a user (server-wide), or a whole server.

        The target can be a name, an ID, the keyword 'channel' or 'server'.
        """
        target, conf = await self.resolve_target(ctx, target)
        self.validate_ignore_target(ctx, target)

        # Save the ignore
        conf.append(target.id)
        self.ignored.save()

        # Leave the server or acknowledge the ignore being successful
        if isinstance(target, discord.Guild):
            await target.leave()
        else:
            await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @commands.command()
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def unignore(self, ctx, *, target):
        """Un-ignores a channel, a user (server-wide), or a whole server."""
        target, conf = await self.resolve_target(ctx, target)
        self.validate_ignore_target(ctx, target)

        try:
            conf.remove(target.id)
        except ValueError:
            await ctx.send('Target not found.')
        else:
            self.ignored.save()
            await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    def get_prune_candidates(self, bot):
        candidates = [g for g in bot.guilds if len([m for m in g.members if m.bot]) >= len(g.members) / 2]
        twitter_destinations = set(chan_id for f in bot.cogs['Twitter'].conf.follows.values() for chan_id in f.discord_channels)
        twitch_destinations = set(chan_id for f in bot.cogs['Twitch'].conf.follows.values() for chan_id in f.channels)

        for guild in candidates:
            guild_channels = set(c.id for c in guild.text_channels)
            if twitter_destinations.intersection(guild_channels) or twitch_destinations.intersection(guild_channels):
                candidates.remove(guild)

        return candidates

    @commands.group(name='prune_guilds', invoke_without_command=True)
    @checks.is_owner()
    async def prune_guilds_group(self, ctx):
        targets = self.get_prune_candidates(ctx.bot)
        if targets:
            log.info(f'Leaving {len(targets)} guilds.')

        for guild in targets:
            await guild.leave()

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @prune_guilds_group.command(name='count')
    @checks.is_owner()
    async def prune_guilds_count(self, ctx):
        await ctx.send(len(self.get_prune_candidates(ctx.bot)))

    @commands.command()
    @checks.is_owner()
    async def restart(self, ctx):
        """Restarts the bot."""
        ctx.bot.restart()

    @commands.command()
    @checks.is_owner()
    async def shutdown(self, ctx):
        """Shuts the bot down."""
        ctx.bot.shutdown()

    @commands.command()
    @checks.is_owner()
    async def status(self, ctx, *, status=None):
        """Changes the bot's status."""
        await ctx.bot.change_presence(game=discord.Game(name=status))
        self.bot.conf.status = status
        self.bot.conf.save()

    async def on_command(self, ctx):
        self.commands_used[ctx.command.qualified_name] += 1
        if ctx.guild is None:
            log.info(f'DM:{ctx.author.name}:{ctx.author.id}:{ctx.message.content}')
        else:
            log.info(f'{ctx.guild.name}:{ctx.guild.id}:{ctx.channel.name}:{ctx.channel.id}:{ctx.author.name}:{ctx.author.id}:{ctx.message.content}')

    async def on_guild_join(self, guild):
        # Log that the bot has been added somewhere
        log.info(f'GUILD_JOIN:{guild.name}:{guild.id}:{guild.owner.name}:{guild.owner.id}:')
        if guild.id in self.ignored.guilds:
            log.info(f'IGNORED GUILD:{guild.name}:{guild.id}:')
            await guild.leave()

    async def on_guild_remove(self, guild):
        # Log that the bot has been removed from somewhere
        log.info(f'GUILD_REMOVE:{guild.name}:{guild.id}:{guild.owner.name}:{guild.owner.id}:')
<commit_msg>Add the command €ignore list.<commit_after>import logging
from collections import Counter

import discord.utils
import discord.ext.commands as commands

import paths
from .util import checks, config, utils

log = logging.getLogger(__name__)

def setup(bot):
    bot.add_cog(Admin(bot))

class IgnoredConfig(config.ConfigElement):
    def __init__(self, **kwargs):
        self.channels = kwargs.pop('channels', [])
        self.guilds = kwargs.pop('guilds', [])
        self.users = utils.dict_keys_to_int(kwargs.pop('users', {}))

class Admin:
    """Bot management commands and events."""
    def __init__(self, bot):
        self.commands_used = Counter()
        self.ignored = config.Config(paths.IGNORED_CONFIG, encoding='utf-8')
        self.bot = bot

    def __global_check_once(self, ctx):
        """A global check used on every command."""
        author = ctx.author
        guild = ctx.guild
        if author == ctx.bot.owner:
            return True

        if guild is not None:
            # Check if we're ignoring the guild
            if guild.id in self.ignored.guilds:
                return False

            if ctx.channel.id in self.ignored.channels:
                return False

            # Guild owners can't be ignored
            if author.id == guild.owner.id:
                return True

            # Check if the user is banned from using the bot
            if author.id in self.ignored.users.get(guild.id, {}):
                return False

            # Check if the channel is banned, bypass this if the user has the manage guild permission
            channel = ctx.channel
            perms = channel.permissions_for(author)
            if not perms.manage_guild and channel.id in self.ignored.channels:
                return False
        return True

    async def resolve_target(self, ctx, target):
        if target == 'channel':
            return ctx.channel, self.ignored.channels
        elif target == 'guild' or target == 'server':
            return ctx.guild, self.ignored.guilds

        # Try converting to a text channel
        try:
            channel = await commands.TextChannelConverter().convert(ctx, target)
        except commands.BadArgument:
            pass
        else:
            return channel, self.ignored.channels

        # Try converting to a user
        try:
            member = await commands.MemberConverter().convert(ctx, target)
        except commands.BadArgument:
            pass
        else:
            guild_id = ctx.guild.id
            try:
                return member, self.ignored.users[guild_id]
            except KeyError:
                self.ignored.users[guild_id] = []
                return member, self.ignored.users[guild_id]

        # Convert to a guild
        try:
            guild = await utils.GuildConverter().convert(ctx, target)
        except:
            pass
        else:
            return guild, self.ignored.guilds

        # Nope
        raise commands.BadArgument(f'"{target}" not found.')

    def validate_ignore_target(self, ctx, target):
        owner_id = ctx.bot.owner.id
        # Only let the bot owner unignore a guild owner
        if isinstance(target, discord.Member):
            # Do not ignore the bot owner
            if owner_id == target.id:
                raise commands.BadArgument('Cannot ignore/unignore the bot owner.')

            # Only allow the bot owner to unignore the guild owner
            if target.id == ctx.guild.owner.id and ctx.author.id != owner_id:
                raise commands.BadArgument('Only the bot owner can ignore/unignore the owner of a server.')
        elif isinstance(target, discord.Guild):
            # Only allow the bot owner to ignore guilds
            if ctx.author.id != owner_id:
                raise commands.BadArgument('Only the bot owner can ignore/unignore servers.')
        elif isinstance(target, discord.VoiceChannel):
            # Do not ignore voice channels
            raise commands.BadArgument('Cannot ignore/unignore voice channels.')

    @commands.command(aliases=['checkpermissions'])
    async def checkperms(self, ctx):
        name = ctx.bot.user.name
        perms_str = 'Read Messages, Send Messages, Manage Messages, Embed Links, Read Message History, Use External Emojis, Add Reactions'
        perms = discord.Permissions(486464)

        # Check the integration role
        role = discord.utils.get(ctx.guild.roles, name=name)
        if not role or not perms <= role.permissions:
            raise commands.UserInputError(f'Please make sure the integration role `{name}` has all the following permissions and is added to the bot :\n'
                                          f'{perms_str}.\n'
                                          f'Note that you can also kick and re-invite the bot with its default permissions.')

        # Check every channel for overwrites
        failed = []
        for channel in sorted(ctx.guild.text_channels, key=lambda c: c.position):
            if not perms <= channel.permissions_for(ctx.guild.me):
                failed.append(channel)
        if failed:
            raise commands.UserInputError(f'Please make sure the channel permissions overwrites for the following channels do not remove these permissions from the bot :\n'
                                          f'{perms_str}.\n'
                                          f'{" ".join(c.mention for c in failed)}')

        await ctx.send('All good.')

    @commands.group(name='ignore', invoke_without_command=True)
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def ignore_group(self, ctx, *, target):
        """Ignores a channel, a user (server-wide), or a whole server.

        The target can be a name, an ID, the keyword 'channel' or 'server'.
        """
        target, conf = await self.resolve_target(ctx, target)
        self.validate_ignore_target(ctx, target)

        # Save the ignore
        conf.append(target.id)
        self.ignored.save()

        # Leave the server or acknowledge the ignore being successful
        if isinstance(target, discord.Guild):
            await target.leave()
        else:
            await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @ignore_group.command(name='list')
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def ignore_list(self, ctx):
        """Lists ignored channels, users and servers related to the command's use."""
        channels = set(discord.utils.get(ctx.guild.text_channels, id=cid) for cid in self.ignored.channels)
        members = set(discord.utils.get(ctx.guild.members, id=uid) for uid in self.ignored.users.get(ctx.guild.id, []))

        embed = discord.Embed(colour=discord.Colour.blurple())
        embed.add_field(name='Ignored channels', value=', '.join(c.mention for c in channels if c is not None) or 'None', inline=False)
        embed.add_field(name='Ignored users', value=', '.join(m.mention for m in members if m is not None) or 'None', inline=False)

        if ctx.author == ctx.bot.owner:
            embed.add_field(name='Ignored guilds', value=', '.join(self.ignored.guilds) or 'None', inline=False)

        await ctx.send(embed=embed)

    @commands.command()
    @commands.guild_only()
    @commands.has_permissions(manage_guild=True)
    async def unignore(self, ctx, *, target):
        """Un-ignores a channel, a user (server-wide), or a whole server."""
        target, conf = await self.resolve_target(ctx, target)
        self.validate_ignore_target(ctx, target)

        try:
            conf.remove(target.id)
        except ValueError:
            await ctx.send('Target not found.')
        else:
            self.ignored.save()
            await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    def get_prune_candidates(self, bot):
        candidates = [g for g in bot.guilds if len([m for m in g.members if m.bot]) >= len(g.members) / 2]
        twitter_destinations = set(chan_id for f in bot.cogs['Twitter'].conf.follows.values() for chan_id in f.discord_channels)
        twitch_destinations = set(chan_id for f in bot.cogs['Twitch'].conf.follows.values() for chan_id in f.channels)

        for guild in candidates:
            guild_channels = set(c.id for c in guild.text_channels)
            if twitter_destinations.intersection(guild_channels) or twitch_destinations.intersection(guild_channels):
                candidates.remove(guild)

        return candidates

    @commands.group(name='prune_guilds', invoke_without_command=True)
    @checks.is_owner()
    async def prune_guilds_group(self, ctx):
        targets = self.get_prune_candidates(ctx.bot)
        if targets:
            log.info(f'Leaving {len(targets)} guilds.')

        for guild in targets:
            await guild.leave()

        await ctx.message.add_reaction('\N{WHITE HEAVY CHECK MARK}')

    @prune_guilds_group.command(name='count')
    @checks.is_owner()
    async def prune_guilds_count(self, ctx):
        await ctx.send(len(self.get_prune_candidates(ctx.bot)))

    @commands.command()
    @checks.is_owner()
    async def restart(self, ctx):
        """Restarts the bot."""
        ctx.bot.restart()

    @commands.command()
    @checks.is_owner()
    async def shutdown(self, ctx):
        """Shuts the bot down."""
        ctx.bot.shutdown()

    @commands.command()
    @checks.is_owner()
    async def status(self, ctx, *, status=None):
        """Changes the bot's status."""
        await ctx.bot.change_presence(game=discord.Game(name=status))
        self.bot.conf.status = status
        self.bot.conf.save()

    async def on_command(self, ctx):
        self.commands_used[ctx.command.qualified_name] += 1
        if ctx.guild is None:
            log.info(f'DM:{ctx.author.name}:{ctx.author.id}:{ctx.message.content}')
        else:
            log.info(f'{ctx.guild.name}:{ctx.guild.id}:{ctx.channel.name}:{ctx.channel.id}:{ctx.author.name}:{ctx.author.id}:{ctx.message.content}')

    async def on_guild_join(self, guild):
        # Log that the bot has been added somewhere
        log.info(f'GUILD_JOIN:{guild.name}:{guild.id}:{guild.owner.name}:{guild.owner.id}:')
        if guild.id in self.ignored.guilds:
            log.info(f'IGNORED GUILD:{guild.name}:{guild.id}:')
            await guild.leave()

    async def on_guild_remove(self, guild):
        # Log that the bot has been removed from somewhere
        log.info(f'GUILD_REMOVE:{guild.name}:{guild.id}:{guild.owner.name}:{guild.owner.id}:')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import collections
import httplib as http

from flask import request

from framework.exceptions import HTTPError
from framework.auth.decorators import must_be_logged_in
from framework.auth.utils import privacy_info_handle
from framework.forms.utils import sanitize
from framework.auth.core import get_current_user

from website import settings
from website.filters import gravatar
from website.models import Guid, Comment
from website.project.decorators import must_be_contributor_or_public
from datetime import datetime
from website.project.model import has_anonymous_link

def resolve_target(node, guid):

    if not guid:
        return node
    target = Guid.load(guid)
    if target is None:
        raise HTTPError(http.BAD_REQUEST)
    return target.referent

def collect_discussion(target, users=None):

    users = users or collections.defaultdict(list)
    for comment in getattr(target, 'commented', []):
        if not comment.is_deleted:
            users[comment.user].append(comment)
        collect_discussion(comment, users=users)
    return users

@must_be_contributor_or_public
def comment_discussion(**kwargs):

    node = kwargs['node'] or kwargs['project']
    auth = kwargs['auth']
    users = collect_discussion(node)
    anonymous = has_anonymous_link(node, auth)
    # Sort users by comment frequency
    # TODO: Allow sorting by recency, combination of frequency and recency
    sorted_users = sorted(
        users.keys(),
        key=lambda item: len(users[item]),
        reverse=True,
    )

    return {
        'discussion': [
            {
                'id': privacy_info_handle(user._id, anonymous),
                'url': privacy_info_handle(user.url, anonymous),
                'fullname': privacy_info_handle(user.fullname, anonymous, name=True),
                'isContributor': node.is_contributor(user),
                'gravatarUrl': privacy_info_handle(
                    gravatar(
                        user, use_ssl=True, size=settings.GRAVATAR_SIZE_DISCUSSION,
                    ),
                    anonymous
                ),

            }
            for user in sorted_users
        ]
    }

def serialize_comment(comment, auth, anonymous=False):
    return {
        'id': comment._id,
        'author': {
            'id': privacy_info_handle(comment.user._id, anonymous),
            'url': privacy_info_handle(comment.user.url, anonymous),
            'name': privacy_info_handle(
                comment.user.fullname, anonymous, name=True
            ),
            'gravatarUrl': privacy_info_handle(
                gravatar(
                    comment.user, use_ssl=True,
                    size=settings.GRAVATAR_SIZE_DISCUSSION
                ),
                anonymous
            ),
        },
        'dateCreated': comment.date_created.isoformat(),
        'dateModified': comment.date_modified.isoformat(),
        'content': comment.content,
        'hasChildren': bool(getattr(comment, 'commented', [])),
        'canEdit': comment.user == auth.user,
        'modified': comment.modified,
        'isDeleted': comment.is_deleted,
        'isAbuse': auth.user and auth.user._id in comment.reports,
    }

def serialize_comments(record, auth, anonymous=False):

    return [
        serialize_comment(comment, auth, anonymous)
        for comment in getattr(record, 'commented', [])
    ]

def kwargs_to_comment(kwargs, owner=False):

    comment = Comment.load(kwargs.get('cid'))
    if comment is None:
        raise HTTPError(http.BAD_REQUEST)

    if owner:
        auth = kwargs['auth']
        if auth.user != comment.user:
            raise HTTPError(http.FORBIDDEN)

    return comment

@must_be_logged_in
@must_be_contributor_or_public
def add_comment(**kwargs):

    auth = kwargs['auth']
    node = kwargs['node'] or kwargs['project']

    if not node.comment_level:
        raise HTTPError(http.BAD_REQUEST)

    if not node.can_comment(auth):
        raise HTTPError(http.FORBIDDEN)

    guid = request.json.get('target')
    target = resolve_target(node, guid)

    content = request.json.get('content').strip()
    content = sanitize(content)
    if not content:
        raise HTTPError(http.BAD_REQUEST)
    if len(content) > settings.COMMENT_MAXLENGTH:
        raise HTTPError(http.BAD_REQUEST)

    comment = Comment.create(
        auth=auth,
        node=node,
        target=target,
        user=auth.user,
        content=content,
    )
    comment.save()

    return {
        'comment': serialize_comment(comment, auth)
    }, http.CREATED

@must_be_contributor_or_public
def list_comments(**kwargs):

    auth = kwargs['auth']
    node = kwargs['node'] or kwargs['project']
    anonymous = has_anonymous_link(node, auth)
    guid = request.args.get('target')
    target = resolve_target(node, guid)
    comments = serialize_comments(target, auth, anonymous)
    n_unread = 0

    if node.is_contributor(auth.user):
        view_timestamp = datetime(1970, 1, 1, 12, 0, 0).isoformat()

        if auth.user.comments_viewed_timestamp is None:
            auth.user.comments_viewed_timestamp = {}
            auth.user.save()

        if auth.user.comments_viewed_timestamp.get(node._id, None):
            view_timestamp = auth.user.comments_viewed_timestamp[node._id]

        n_unread = n_unread_comments(view_timestamp, comments, auth.user)

    return {

        'comments': comments,
        'nUnread': n_unread
    }

def n_unread_comments(view_timestamp, comments, current_user):
    count = 0

    for comment in comments:
        if comment['author']['id'] != current_user._id:
            if comment['dateCreated'] > view_timestamp or comment['dateModified'] > view_timestamp:
                count += 1

    return count

@must_be_logged_in
@must_be_contributor_or_public
def edit_comment(**kwargs):

    auth = kwargs['auth']

    comment = kwargs_to_comment(kwargs, owner=True)

    content = request.json.get('content').strip()
    content = sanitize(content)
    if not content:
        raise HTTPError(http.BAD_REQUEST)
    if len(content) > settings.COMMENT_MAXLENGTH:
        raise HTTPError(http.BAD_REQUEST)

    comment.edit(
        content=content,
        auth=auth,
        save=True
    )

    return serialize_comment(comment, auth)

@must_be_logged_in
@must_be_contributor_or_public
def delete_comment(**kwargs):

    auth = kwargs['auth']
    comment = kwargs_to_comment(kwargs, owner=True)
    comment.delete(auth=auth, save=True)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def undelete_comment(**kwargs):

    auth = kwargs['auth']
    comment = kwargs_to_comment(kwargs, owner=True)
    comment.undelete(auth=auth, save=True)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def view_comments(**kwargs):
    node = kwargs['node'] or kwargs['project']
    user = get_current_user()

    if node.is_contributor(user):
        user.comments_viewed_timestamp[node._id] = datetime.utcnow().isoformat()
        user.save()
        list_comments(**kwargs)
    return {}

@must_be_logged_in
@must_be_contributor_or_public
def report_abuse(**kwargs):

    auth = kwargs['auth']
    user = auth.user

    comment = kwargs_to_comment(kwargs)

    category = request.json.get('category')
    text = request.json.get('text', '')
    if not category:
        raise HTTPError(http.BAD_REQUEST)

    try:
        comment.report_abuse(user, save=True, category=category, text=text)
    except ValueError:
        raise HTTPError(http.BAD_REQUEST)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def unreport_abuse(**kwargs):

    auth = kwargs['auth']
    user = auth.user

    comment = kwargs_to_comment(kwargs)

    try:
        comment.unreport_abuse(user, save=True)
    except ValueError:
        raise HTTPError(http.BAD_REQUEST)

    return {}
<commit_msg>Fix python style error<commit_after># -*- coding: utf-8 -*-
import collections
import httplib as http

from flask import request

from framework.exceptions import HTTPError
from framework.auth.decorators import must_be_logged_in
from framework.auth.utils import privacy_info_handle
from framework.forms.utils import sanitize
from framework.auth.core import get_current_user

from website import settings
from website.filters import gravatar
from website.models import Guid, Comment
from website.project.decorators import must_be_contributor_or_public
from datetime import datetime
from website.project.model import has_anonymous_link

def resolve_target(node, guid):

    if not guid:
        return node
    target = Guid.load(guid)
    if target is None:
        raise HTTPError(http.BAD_REQUEST)
    return target.referent

def collect_discussion(target, users=None):

    users = users or collections.defaultdict(list)
    for comment in getattr(target, 'commented', []):
        if not comment.is_deleted:
            users[comment.user].append(comment)
        collect_discussion(comment, users=users)
    return users

@must_be_contributor_or_public
def comment_discussion(**kwargs):

    node = kwargs['node'] or kwargs['project']
    auth = kwargs['auth']
    users = collect_discussion(node)
    anonymous = has_anonymous_link(node, auth)
    # Sort users by comment frequency
    # TODO: Allow sorting by recency, combination of frequency and recency
    sorted_users = sorted(
        users.keys(),
        key=lambda item: len(users[item]),
        reverse=True,
    )

    return {
        'discussion': [
            {
                'id': privacy_info_handle(user._id, anonymous),
                'url': privacy_info_handle(user.url, anonymous),
                'fullname': privacy_info_handle(user.fullname, anonymous, name=True),
                'isContributor': node.is_contributor(user),
                'gravatarUrl': privacy_info_handle(
                    gravatar(
                        user, use_ssl=True, size=settings.GRAVATAR_SIZE_DISCUSSION,
                    ),
                    anonymous
                ),

            }
            for user in sorted_users
        ]
    }

def serialize_comment(comment, auth, anonymous=False):
    return {
        'id': comment._id,
        'author': {
            'id': privacy_info_handle(comment.user._id, anonymous),
            'url': privacy_info_handle(comment.user.url, anonymous),
            'name': privacy_info_handle(
                comment.user.fullname, anonymous, name=True
            ),
            'gravatarUrl': privacy_info_handle(
                gravatar(
                    comment.user, use_ssl=True,
                    size=settings.GRAVATAR_SIZE_DISCUSSION
                ),
                anonymous
            ),
        },
        'dateCreated': comment.date_created.isoformat(),
        'dateModified': comment.date_modified.isoformat(),
        'content': comment.content,
        'hasChildren': bool(getattr(comment, 'commented', [])),
        'canEdit': comment.user == auth.user,
        'modified': comment.modified,
        'isDeleted': comment.is_deleted,
        'isAbuse': auth.user and auth.user._id in comment.reports,
    }

def serialize_comments(record, auth, anonymous=False):

    return [
        serialize_comment(comment, auth, anonymous)
        for comment in getattr(record, 'commented', [])
    ]

def kwargs_to_comment(kwargs, owner=False):

    comment = Comment.load(kwargs.get('cid'))
    if comment is None:
        raise HTTPError(http.BAD_REQUEST)

    if owner:
        auth = kwargs['auth']
        if auth.user != comment.user:
            raise HTTPError(http.FORBIDDEN)

    return comment

@must_be_logged_in
@must_be_contributor_or_public
def add_comment(**kwargs):

    auth = kwargs['auth']
    node = kwargs['node'] or kwargs['project']

    if not node.comment_level:
        raise HTTPError(http.BAD_REQUEST)

    if not node.can_comment(auth):
        raise HTTPError(http.FORBIDDEN)

    guid = request.json.get('target')
    target = resolve_target(node, guid)

    content = request.json.get('content').strip()
    content = sanitize(content)
    if not content:
        raise HTTPError(http.BAD_REQUEST)
    if len(content) > settings.COMMENT_MAXLENGTH:
        raise HTTPError(http.BAD_REQUEST)

    comment = Comment.create(
        auth=auth,
        node=node,
        target=target,
        user=auth.user,
        content=content,
    )
    comment.save()

    return {
        'comment': serialize_comment(comment, auth)
    }, http.CREATED

@must_be_contributor_or_public
def list_comments(**kwargs):

    auth = kwargs['auth']
    node = kwargs['node'] or kwargs['project']
    anonymous = has_anonymous_link(node, auth)
    guid = request.args.get('target')
    target = resolve_target(node, guid)
    comments = serialize_comments(target, auth, anonymous)
    n_unread = 0

    if node.is_contributor(auth.user):
        view_timestamp = datetime(1970, 1, 1, 12, 0, 0).isoformat()

        if auth.user.comments_viewed_timestamp is None:
            auth.user.comments_viewed_timestamp = {}
            auth.user.save()

        if auth.user.comments_viewed_timestamp.get(node._id, None):
            view_timestamp = auth.user.comments_viewed_timestamp[node._id]

        n_unread = n_unread_comments(view_timestamp, comments, auth.user)

    return {

        'comments': comments,
        'nUnread': n_unread
    }

def n_unread_comments(view_timestamp, comments, current_user):
    count = 0

    for comment in comments:
        if comment['author']['id'] != current_user._id:
            if comment['dateCreated'] > view_timestamp or comment['dateModified'] > view_timestamp:
                count += 1

    return count

@must_be_logged_in
@must_be_contributor_or_public
def edit_comment(**kwargs):

    auth = kwargs['auth']

    comment = kwargs_to_comment(kwargs, owner=True)

    content = request.json.get('content').strip()
    content = sanitize(content)
    if not content:
        raise HTTPError(http.BAD_REQUEST)
    if len(content) > settings.COMMENT_MAXLENGTH:
        raise HTTPError(http.BAD_REQUEST)

    comment.edit(
        content=content,
        auth=auth,
        save=True
    )

    return serialize_comment(comment, auth)

@must_be_logged_in
@must_be_contributor_or_public
def delete_comment(**kwargs):

    auth = kwargs['auth']
    comment = kwargs_to_comment(kwargs, owner=True)
    comment.delete(auth=auth, save=True)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def undelete_comment(**kwargs):

    auth = kwargs['auth']
    comment = kwargs_to_comment(kwargs, owner=True)
    comment.undelete(auth=auth, save=True)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def view_comments(**kwargs):
    node = kwargs['node'] or kwargs['project']
    user = get_current_user()

    if node.is_contributor(user):
        user.comments_viewed_timestamp[node._id] = datetime.utcnow().isoformat()
        user.save()
        list_comments(**kwargs)
    return {}

@must_be_logged_in
@must_be_contributor_or_public
def report_abuse(**kwargs):

    auth = kwargs['auth']
    user = auth.user

    comment = kwargs_to_comment(kwargs)

    category = request.json.get('category')
    text = request.json.get('text', '')
    if not category:
        raise HTTPError(http.BAD_REQUEST)

    try:
        comment.report_abuse(user, save=True, category=category, text=text)
    except ValueError:
        raise HTTPError(http.BAD_REQUEST)

    return {}

@must_be_logged_in
@must_be_contributor_or_public
def unreport_abuse(**kwargs):

    auth = kwargs['auth']
    user = auth.user

    comment = kwargs_to_comment(kwargs)

    try:
        comment.unreport_abuse(user, save=True)
    except ValueError:
        raise HTTPError(http.BAD_REQUEST)

    return {}
<|endoftext|>


<commit_before># Author: Ian Burke
# Module: Emerging Technologies
# Date: September, 2017

# Problem Sheet: https://emerging-technologies.github.io/problems/python-fundamentals.html

import time # import time package

# https://docs.python.org/3/library/time.html
currentTime = time.localtime()
print("Current Time: ", currentTime) # print out time<commit_msg>current time formatted<commit_after># Author: Ian Burke
# Module: Emerging Technologies
# Date: September, 2017

# Problem Sheet: https://emerging-technologies.github.io/problems/python-fundamentals.html

import time # import time package

# https://docs.python.org/3/library/time.html
currentTime = time.asctime()
print("Current Time: ", currentTime) # print out time<|endoftext|>