<commit_before>import ujson

from configman import ConfigurationManager
from configman.dotdict import DotDict
from mock import Mock, patch

from socorro.processor.processor_2015 import (
    Processor2015,
    rule_sets_from_string
)
from socorro.processor.general_transform_rules import (
    CPUInfoRule,
    OSInfoRule
)
from socorro.unittest.testbase import TestCase

thread_over_255_chars = {
    "frames": [
        {
            "file": "nsTerminator.cpp:604367e1fa5e",
            "frame": 0,
            "function": ("mozilla::`anonymous namespace::"
                         "RunWatchdog(void *)" + "a" * 255),
            "function_offset": "0x0",
            "line": 151,
            "module": "xul.dll",
            "module_offset": "0x783f2b",
            "offset": "0x67903f2b",
            "registers": {
                "eax": "0x0000003f",
                "ebp": "0x163ff96c",
                "ebx": "0x0cf44450",
                "ecx": "0x691e3698",
                "edi": "0x76d3f551",
                "edx": "0x01dc1010",
                "efl": "0x00000246",
                "eip": "0x67903f2b",
                "esi": "0x0000003f",
                "esp": "0x163ff968"
            },
            "trust": "context"
        }
    ]
}
stackwalker_output_str = ujson.dumps({
    "crash_info": {
        "address": "0x0",
        "crashing_thread": 0,
        "type": "EXC_BAD_ACCESS / KERN_INVALID_ADDRESS"
    },
    "status": "OK",
    "crashing_thread": thread_over_255_chars,
    "threads": [thread_over_255_chars],
    "system_info": {
        "os": "Windows NT",
        "cpu_arch": "x86"
    },
    "sensitive": {
        "exploitability": "high"
    },
})

rule_set_01 = [
    [
        'transform_rules',
        'socorro.processor.general_transform_rules.CPUInfoRule, '
        'socorro.processor.general_transform_rules.OSInfoRule '
    ]
]
rule_set_01_str = ujson.dumps(rule_set_01)

class TestProcessor2015(TestCase):
    def test_rule_sets_from_string_1(self):
        rule_set_config = rule_sets_from_string(rule_set_01_str)
        rc = rule_set_config.get_required_config()
        assert 'transform_rules' in rc
        expected = (
            'socorro.processor.general_transform_rules.CPUInfoRule, '
            'socorro.processor.general_transform_rules.OSInfoRule '
        )
        assert rc.transform_rules.rules_list.default == expected

    def test_Processor2015_init(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': rule_set_01_str}],
        )
        config = cm.get_config()
        config.logger = Mock()

        p = Processor2015(config)

        assert isinstance(p.rule_system, DotDict)
        assert len(p.rules) == 2
        assert isinstance(p.rules[0], CPUInfoRule)
        assert isinstance(p.rules[1], OSInfoRule)

    def test_process_crash_no_rules(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': '[]'}],
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        p = Processor2015(config)
        raw_crash = DotDict()
        raw_dumps = {}
        with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
            faked_utcnow.return_value = '2015-01-01T00:00:00'
            processed_crash = p.process_crash(
                raw_crash,
                raw_dumps,
                DotDict()
            )

        assert processed_crash.success
        assert processed_crash.started_datetime == '2015-01-01T00:00:00'
        assert processed_crash.startedDateTime == '2015-01-01T00:00:00'
        assert processed_crash.completed_datetime == '2015-01-01T00:00:00'
        assert processed_crash.completeddatetime == '2015-01-01T00:00:00'
        assert processed_crash.processor_notes == 'dwight; Processor2015'

    def test_process_crash_existing_processed_crash(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': '[]'}],
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        p = Processor2015(config)
        raw_crash = DotDict()
        raw_dumps = {}
        processed_crash = DotDict()
        processed_crash.processor_notes = "we've been here before; yep"
        processed_crash.started_datetime = '2014-01-01T00:00:00'
        with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
            faked_utcnow.return_value = '2015-01-01T00:00:00'
            processed_crash = p.process_crash(
                raw_crash,
                raw_dumps,
                processed_crash
            )

        assert processed_crash.success
        assert processed_crash.started_datetime == '2015-01-01T00:00:00'
        assert processed_crash.startedDateTime == '2015-01-01T00:00:00'
        assert processed_crash.completed_datetime == '2015-01-01T00:00:00'
        assert processed_crash.completeddatetime == '2015-01-01T00:00:00'
        expected = (
            "dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; we've been here "
            "before; yep"
        )
        assert processed_crash.processor_notes == expected

    @patch('socorro.processor.breakpad_transform_rules.subprocess')
    def test_process_over_255_chars(self, mocked_subprocess_module):
        cm = ConfigurationManager(
            definition_source=(
                Processor2015.get_required_config(),
            ),
            values_source_list=[]
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        mocked_subprocess_handle = (
            mocked_subprocess_module.Popen.return_value
        )
        mocked_subprocess_handle.wait.return_value = 0
        mocked_subprocess_handle.stdout.read.return_value = (
            stackwalker_output_str
        )
        p = Processor2015(config)
        raw_crash = DotDict({
            "uuid": "00000000-0000-0000-0000-000002140504",
            "CrashTime": "1336519554",
            "SecondsSinceLastCrash": "86985",
            "PluginHang": "1",
            "ProductName": "Firefox",
            "Version": "19",
            "BuildID": "20121031"
        })
        raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"}

        processed_crash = p.process_crash(
            raw_crash,
            raw_dumps,
            DotDict()
        )

        assert processed_crash.success
        expected = (
            'dwight; Processor2015; '
            'SignatureTool: signature truncated due to length; '
            'SignatureTool: signature truncated due to length'
        )
        assert processed_crash.processor_notes == expected
        assert processed_crash.signature.startswith('shutdownhang')
        assert len(processed_crash.signature) == 255
<commit_msg>fixup processor test to reflect unified SigTrunc rule<commit_after>import ujson

from configman import ConfigurationManager
from configman.dotdict import DotDict
from mock import Mock, patch

from socorro.processor.processor_2015 import (
    Processor2015,
    rule_sets_from_string
)
from socorro.processor.general_transform_rules import (
    CPUInfoRule,
    OSInfoRule
)
from socorro.unittest.testbase import TestCase

thread_over_255_chars = {
    "frames": [
        {
            "file": "nsTerminator.cpp:604367e1fa5e",
            "frame": 0,
            "function": ("mozilla::`anonymous namespace::"
                         "RunWatchdog(void *)" + "a" * 255),
            "function_offset": "0x0",
            "line": 151,
            "module": "xul.dll",
            "module_offset": "0x783f2b",
            "offset": "0x67903f2b",
            "registers": {
                "eax": "0x0000003f",
                "ebp": "0x163ff96c",
                "ebx": "0x0cf44450",
                "ecx": "0x691e3698",
                "edi": "0x76d3f551",
                "edx": "0x01dc1010",
                "efl": "0x00000246",
                "eip": "0x67903f2b",
                "esi": "0x0000003f",
                "esp": "0x163ff968"
            },
            "trust": "context"
        }
    ]
}
stackwalker_output_str = ujson.dumps({
    "crash_info": {
        "address": "0x0",
        "crashing_thread": 0,
        "type": "EXC_BAD_ACCESS / KERN_INVALID_ADDRESS"
    },
    "status": "OK",
    "crashing_thread": thread_over_255_chars,
    "threads": [thread_over_255_chars],
    "system_info": {
        "os": "Windows NT",
        "cpu_arch": "x86"
    },
    "sensitive": {
        "exploitability": "high"
    },
})

rule_set_01 = [
    [
        'transform_rules',
        'socorro.processor.general_transform_rules.CPUInfoRule, '
        'socorro.processor.general_transform_rules.OSInfoRule '
    ]
]
rule_set_01_str = ujson.dumps(rule_set_01)

class TestProcessor2015(TestCase):
    def test_rule_sets_from_string_1(self):
        rule_set_config = rule_sets_from_string(rule_set_01_str)
        rc = rule_set_config.get_required_config()
        assert 'transform_rules' in rc
        expected = (
            'socorro.processor.general_transform_rules.CPUInfoRule, '
            'socorro.processor.general_transform_rules.OSInfoRule '
        )
        assert rc.transform_rules.rules_list.default == expected

    def test_Processor2015_init(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': rule_set_01_str}],
        )
        config = cm.get_config()
        config.logger = Mock()

        p = Processor2015(config)

        assert isinstance(p.rule_system, DotDict)
        assert len(p.rules) == 2
        assert isinstance(p.rules[0], CPUInfoRule)
        assert isinstance(p.rules[1], OSInfoRule)

    def test_process_crash_no_rules(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': '[]'}],
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        p = Processor2015(config)
        raw_crash = DotDict()
        raw_dumps = {}
        with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
            faked_utcnow.return_value = '2015-01-01T00:00:00'
            processed_crash = p.process_crash(
                raw_crash,
                raw_dumps,
                DotDict()
            )

        assert processed_crash.success
        assert processed_crash.started_datetime == '2015-01-01T00:00:00'
        assert processed_crash.startedDateTime == '2015-01-01T00:00:00'
        assert processed_crash.completed_datetime == '2015-01-01T00:00:00'
        assert processed_crash.completeddatetime == '2015-01-01T00:00:00'
        assert processed_crash.processor_notes == 'dwight; Processor2015'

    def test_process_crash_existing_processed_crash(self):
        cm = ConfigurationManager(
            definition_source=Processor2015.get_required_config(),
            values_source_list=[{'rule_sets': '[]'}],
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        p = Processor2015(config)
        raw_crash = DotDict()
        raw_dumps = {}
        processed_crash = DotDict()
        processed_crash.processor_notes = "we've been here before; yep"
        processed_crash.started_datetime = '2014-01-01T00:00:00'
        with patch('socorro.processor.processor_2015.utc_now') as faked_utcnow:
            faked_utcnow.return_value = '2015-01-01T00:00:00'
            processed_crash = p.process_crash(
                raw_crash,
                raw_dumps,
                processed_crash
            )

        assert processed_crash.success
        assert processed_crash.started_datetime == '2015-01-01T00:00:00'
        assert processed_crash.startedDateTime == '2015-01-01T00:00:00'
        assert processed_crash.completed_datetime == '2015-01-01T00:00:00'
        assert processed_crash.completeddatetime == '2015-01-01T00:00:00'
        expected = (
            "dwight; Processor2015; earlier processing: 2014-01-01T00:00:00; we've been here "
            "before; yep"
        )
        assert processed_crash.processor_notes == expected

    @patch('socorro.processor.breakpad_transform_rules.subprocess')
    def test_process_over_255_chars(self, mocked_subprocess_module):
        cm = ConfigurationManager(
            definition_source=(
                Processor2015.get_required_config(),
            ),
            values_source_list=[]
        )
        config = cm.get_config()
        config.logger = Mock()
        config.processor_name = 'dwight'

        mocked_subprocess_handle = (
            mocked_subprocess_module.Popen.return_value
        )
        mocked_subprocess_handle.wait.return_value = 0
        mocked_subprocess_handle.stdout.read.return_value = (
            stackwalker_output_str
        )
        p = Processor2015(config)
        raw_crash = DotDict({
            "uuid": "00000000-0000-0000-0000-000002140504",
            "CrashTime": "1336519554",
            "SecondsSinceLastCrash": "86985",
            "PluginHang": "1",
            "ProductName": "Firefox",
            "Version": "19",
            "BuildID": "20121031"
        })
        raw_dumps = {"upload_file_minidump": "a_fake_dump.dump"}

        processed_crash = p.process_crash(
            raw_crash,
            raw_dumps,
            DotDict()
        )

        assert processed_crash.success
        expected = (
            'dwight; Processor2015; SigTrunc: signature truncated due to length'
        )
        assert processed_crash.processor_notes == expected
        assert processed_crash.signature.startswith('shutdownhang')
        assert len(processed_crash.signature) == 255
<|endoftext|>


<commit_before>import numpy as np

def write(X, filename, lengths=None,
          resolution=50000,
          copy=True):
    """
    Write the similarity matrix to a file

    Parameters
    ----------
    X: ndarray,
    """
    if copy:
        X = X.copy()

    output_file = open(filename, 'w')

    tr = X.sum(axis=0) == 0
    if tr.sum() != 0:
        X[tr, :] = np.nan
        X[:, tr] = np.nan

    for i in range(len(X)):
        chr1, loc1 = _get_index(i, lengths)
        if loc1 > lengths[chr1 - 1]:
            raise ValueError("Problem !")
        loc1 *= resolution
        loc1 += 1
        for j in range(len(X)):
            if i > j:
                continue
            chr2, loc2 = _get_index(j, lengths)
            if loc2 > lengths[chr2 - 1]:
                raise ValueError("Problem !")
            loc2 *= resolution
            loc2 += 1

            if chr2 == chr1:
                dis = loc2 - loc1
            else:
                dis = 0

            if not X[i, j] or np.isnan(X[i, j]):
                continue

            output_file.write('%d\t%d\t%d\t%d\t%d\t%f\t0\t0\n' %
                              (chr1, loc1, chr2, loc2, dis, X[i, j]))

    output_file.close()

def _get_index(i, lengths):
    """
    From an index, return the chromosome number and the loci number

    Parameters
    ----------
    i : integer
        index in the full matrix

    lengths : ndarray (n, )

    Returns
    -------
    tuple : (c, l)
        chromosome number, locus
    """
    length_cum = lengths.cumsum()
    c = (length_cum > i).argmax() + 1

    if c > 1:
        length = i - length_cum[c - 2]
    else:
        length = i
    if len(lengths) > 1 and length > lengths[c - 1]:
        raise ValueError("there's a problem")

    if c > len(lengths):
        raise ValueError("There's a problem")
    return c, length
<commit_msg>Now outputs counts as floats<commit_after>import numpy as np

def write(X, filename, lengths=None,
          resolution=50000,
          copy=True):
    """
    Write the similarity matrix to a file

    Parameters
    ----------
    X: ndarray,
    """
    if copy:
        X = X.copy()
    X = X.astype(float)

    output_file = open(filename, 'w')

    tr = X.sum(axis=0) == 0
    if tr.sum() != 0:
        X[tr, :] = np.nan
        X[:, tr] = np.nan

    for i in range(len(X)):
        chr1, loc1 = _get_index(i, lengths)
        if loc1 > lengths[chr1 - 1]:
            raise ValueError("Problem !")
        loc1 *= resolution
        loc1 += 1
        for j in range(len(X)):
            if i > j:
                continue
            chr2, loc2 = _get_index(j, lengths)
            if loc2 > lengths[chr2 - 1]:
                raise ValueError("Problem !")
            loc2 *= resolution
            loc2 += 1

            if chr2 == chr1:
                dis = loc2 - loc1
            else:
                dis = 0

            if not X[i, j] or np.isnan(X[i, j]):
                continue

            output_file.write('%d\t%d\t%d\t%d\t%d\t%f\t0\t0\n' %
                              (chr1, loc1, chr2, loc2, dis, X[i, j]))

    output_file.close()

def _get_index(i, lengths):
    """
    From an index, return the chromosome number and the loci number

    Parameters
    ----------
    i : integer
        index in the full matrix

    lengths : ndarray (n, )

    Returns
    -------
    tuple : (c, l)
        chromosome number, locus
    """
    length_cum = lengths.cumsum()
    c = (length_cum > i).argmax() + 1

    if c > 1:
        length = i - length_cum[c - 2]
    else:
        length = i
    if len(lengths) > 1 and length > lengths[c - 1]:
        raise ValueError("there's a problem")

    if c > len(lengths):
        raise ValueError("There's a problem")
    return c, length
<|endoftext|>


<commit_before># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from oslo.config import cfg

from savanna.openstack.common import log as logging
from savanna.plugins import provisioning as p
from savanna.swift import swift_helper as swift
from savanna.topology import topology_helper as topology
from savanna.utils import types as types
from savanna.utils import xmlutils as x

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

CORE_DEFAULT = x.load_hadoop_xml_defaults(
    'plugins/spark/resources/core-default.xml')

HDFS_DEFAULT = x.load_hadoop_xml_defaults(
    'plugins/spark/resources/hdfs-default.xml')

XML_CONFS = {
    "HDFS": [CORE_DEFAULT, HDFS_DEFAULT]
}

SPARK_CONFS = {
    'SPARK': {
             "BASE": ['SPARK_MASTER_IP', 'SLAVES'],
             "OPTIONS": ['SPARK_MASTER_PORT',
                    'SPARK_MASTER_WEBUI_PORT', 'SPARK_WORKER_CORES',
                   'SPARK_WORKER_MEMORY', 'SPARK_WORKER_PORT',
                   'SPARK_WORKER_WEBUI_PORT', 'SPARK_WORKER_INSTANCES'
                   ]
             }
}

# TODO(aignatov): Environmental configs could be more complex
ENV_CONFS = {
    "HDFS": {
        'Name Node Heap Size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"',
        'Data Node Heap Size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"'
    }
}

ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
                        config_type="bool", priority=1,
                        default_value=True, is_optional=True)

ENABLE_DATA_LOCALITY = p.Config('Enable Data Locality', 'general', 'cluster',
                                config_type="bool", priority=1,
                                default_value=True, is_optional=True)

HIDDEN_CONFS = ['fs.defaultFS', 'dfs.namenode.name.dir',
                'dfs.datanode.data.dir', 'mapred.job.tracker',
                'mapred.system.dir', 'mapred.local.dir',
                'hadoop.proxyuser.hadoop.hosts',
                'hadoop.proxyuser.hadoop.groups']

CLUSTER_WIDE_CONFS = ['dfs.block.size', 'dfs.permissions', 'dfs.replication',
                      'dfs.replication.min', 'dfs.replication.max',
                      'io.file.buffer.size', 'mapreduce.job.counters.max',
                      'mapred.output.compress', 'io.compression.codecs',
                      'mapred.output.compression.codec',
                      'mapred.output.compression.type',
                      'mapred.compress.map.output',
                      'mapred.map.output.compression.codec']

PRIORITY_1_CONFS = ['dfs.datanode.du.reserved',
                    'dfs.datanode.failed.volumes.tolerated',
                    'dfs.datanode.max.xcievers', 'dfs.datanode.handler.count',
                    'dfs.namenode.handler.count', 'mapred.child.java.opts',
                    'mapred.jobtracker.maxtasks.per.job',
                    'mapred.job.tracker.handler.count',
                    'mapred.map.child.java.opts',
                    'mapred.reduce.child.java.opts',
                    'io.sort.mb', 'mapred.tasktracker.map.tasks.maximum',
                    'mapred.tasktracker.reduce.tasks.maximum']

# for now we have not so many cluster-wide configs
# lets consider all of them having high priority
PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS

def _initialise_configs():
    configs = []
    for service, config_lists in XML_CONFS.iteritems():
        for config_list in config_lists:
            for config in config_list:
                if config['name'] not in HIDDEN_CONFS:
                    cfg = p.Config(config['name'], service, "node",
                                   is_optional=True, config_type="string",
                                   default_value=str(config['value']),
                                   description=config['description'])
                    if cfg.default_value in ["true", "false"]:
                        cfg.config_type = "bool"
                        cfg.default_value = (cfg.default_value == 'true')
                    elif types.is_int(cfg.default_value):
                        cfg.config_type = "int"
                        cfg.default_value = int(cfg.default_value)
                    if config['name'] in CLUSTER_WIDE_CONFS:
                        cfg.scope = 'cluster'
                    if config['name'] in PRIORITY_1_CONFS:
                        cfg.priority = 1
                    configs.append(cfg)

    for service, config_items in ENV_CONFS.iteritems():
        for name, param_format_str in config_items.iteritems():
            configs.append(p.Config(name, service, "node",
                                    default_value=1024, priority=1,
                                    config_type="int"))

    for service, config_items in SPARK_CONFS.iteritems():
        for name in config_items['BASE']:
            cfg = p.Config(name, service, "cluster", priority=1)
            configs.append(cfg)
        for name in config_items['OPTIONS']:
            cfg = p.Config(name, service, "cluster",
                        is_optional=True, priority=2)
            configs.append(cfg)

    configs.append(ENABLE_SWIFT)
    if CONF.enable_data_locality:
        configs.append(ENABLE_DATA_LOCALITY)

    return configs

# Initialise plugin Hadoop configurations
PLUGIN_CONFIGS = _initialise_configs()

def get_plugin_configs():
    return PLUGIN_CONFIGS

def get_general_configs(hive_hostname, passwd_hive_mysql):
    config = {
        ENABLE_SWIFT.name: {
            'default_value': ENABLE_SWIFT.default_value,
            'conf': extract_name_values(swift.get_swift_configs())
        }
    }
    if CONF.enable_data_locality:
        config.update({
            ENABLE_DATA_LOCALITY.name: {
                'default_value': ENABLE_DATA_LOCALITY.default_value,
                'conf': extract_name_values(topology.vm_awareness_all_config())
            }
        })
    return config

def get_config_value(service, name, cluster=None):
    if cluster:
        for ng in cluster.node_groups:
            if (ng.configuration.get(service) and
                    ng.configuration[service].get(name)):
                return ng.configuration[service][name]

    for c in PLUGIN_CONFIGS:
        if c.applicable_target == service and c.name == name:
            return c.default_value

    raise RuntimeError("Unable get parameter '%s' from service %s",
                       name, service)

def generate_cfg_from_general(cfg, configs, general_config,
                              rest_excluded=False):
    if 'general' in configs:
        for nm in general_config:
            if nm not in configs['general'] and not rest_excluded:
                configs['general'][nm] = general_config[nm]['default_value']
        for name, value in configs['general'].items():
            if value:
                cfg = _set_config(cfg, general_config, name)
                LOG.info("Applying config: %s" % name)
    else:
        cfg = _set_config(cfg, general_config)
    return cfg

def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port
                         ):
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    """
    dfs.name.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/namenode'),
    'dfs.data.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/datanode'),
    'dfs.name.dir': storage_path + 'hdfs/name',
    'dfs.data.dir': storage_path + 'hdfs/data',

    'dfs.hosts': '/etc/hadoop/dn.incl',
    'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
    """
    if hadoop_port == None:
        hadoop_port = 8020

    cfg = {
        'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
        'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
                                            '/dfs/nn'),
        'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
                                            '/dfs/dn'),
        'dfs.replication': 1,
    }

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT

    if CONF.enable_data_locality:
        cfg.update(topology.TOPOLOGY_CONFIG)
        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, core_all),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    return xml_configs

def write_hadoop_configs_xml_file(xml_configs):
    f_core = open('core-site.xml', 'w')
    f_hdfs = open('hdfs-site.xml', 'w')

    f_core.writelines(xml_configs['core-site'])
    f_hdfs.writelines(xml_configs['hdfs-site'])

    f_hdfs.flush()
    f_core.flush()
    f_hdfs.close()
    f_core.close()

    return True

def generate_spark_env_configs(mastername, masterport, masterwebport=None,
                               workercores=None, workermemory=None,
                               workerport=None, workerwebport=None,
                               workerinstances=None):
    # Load configs template file
    f = open('savanna/plugins/spark/resources/spark-env.sh.template', 'r')
    default_config = f.read()
    # Write new configs
    configs = [default_config]
    configs.append('SPARK_MASTER_IP=' + mastername)
    configs.append('SPARK_MASTER_PORT=' + str(masterport))
    if masterwebport != None:
        configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport))
    # configure for workers
    if workercores != None:
        configs.append('SPARK_WORKER_CORES=' + str(workercores))
    if workermemory != None:
        configs.append('SPARK_WORKER_MEMORY=' + str(workermemory))
    if workerport != None:
        configs.append('SPARK_WORKER_PORT=' + str(workerport))
    if workerwebport != None:
        configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport))
    if workerinstances != None:
        configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances))
    return '\n'.join(configs)

    #workernames need to be a list of woker names
def generate_spark_slaves_configs(workernames):
    #slaves = [str(worker) for worker in workernames]
    return '\n'.join(workernames)

def write_spark_configs(configs, filename):
    f = open(str(filename), 'w')
    f.write(configs)
    f.flush()
    f.close()
    return

def extract_environment_confs(configs):
    """Returns list of Hadoop parameters which should be passed via environment
    """
    lst = []
    for service, srv_confs in configs.items():
        if ENV_CONFS.get(service):
            for param_name, param_value in srv_confs.items():
                for cfg_name, cfg_format_str in ENV_CONFS[service].items():
                    if param_name == cfg_name and param_value is not None:
                        lst.append(cfg_format_str % param_value)
        else:
            LOG.warn("Plugin recieved wrong applicable target '%s' in "
                     "environmental configs" % service)
    return lst

def extract_xml_confs(configs):
    """Returns list of Hadoop parameters which should be passed into general
    configs like core-site.xml
    """
    lst = []
    for service, srv_confs in configs.items():
        if XML_CONFS.get(service):
            for param_name, param_value in srv_confs.items():
                for cfg_list in XML_CONFS[service]:
                    names = [cfg['name'] for cfg in cfg_list]
                    if param_name in names and param_value is not None:
                        lst.append((param_name, param_value))
        else:
            LOG.warn("Plugin recieved wrong applicable target '%s' for "
                     "xml configs" % service)
    return lst

def generate_setup_script(storage_paths, env_configs):
    script_lines = ["#!/bin/bash -x"]
    script_lines.append("echo -n > /tmp/hadoop-env.sh")
    for line in env_configs:
        if 'HADOOP' in line:
            script_lines.append('echo "%s" >> /tmp/hadoop-env.sh' % line)
    script_lines.append("cat /etc/hadoop/hadoop-env.sh >> /tmp/hadoop-env.sh")
    script_lines.append("cp /tmp/hadoop-env.sh /etc/hadoop/hadoop-env.sh")

    hadoop_log = storage_paths[0] + "/log/hadoop/\$USER/"
    script_lines.append('sed -i "s,export HADOOP_LOG_DIR=.*,'
                        'export HADOOP_LOG_DIR=%s," /etc/hadoop/hadoop-env.sh'
                        % hadoop_log)

    hadoop_log = storage_paths[0] + "/log/hadoop/hdfs"
    script_lines.append('sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,'
                        'export HADOOP_SECURE_DN_LOG_DIR=%s," '
                        '/etc/hadoop/hadoop-env.sh' % hadoop_log)

    for path in storage_paths:
        script_lines.append("chown -R hadoop:hadoop %s" % path)
        script_lines.append("chmod -R 755 %s" % path)
    return "\n".join(script_lines)

def extract_name_values(configs):
    return dict((cfg['name'], cfg['value']) for cfg in configs)

def extract_hadoop_path(lst, hadoop_dir):
    return ",".join([p + hadoop_dir for p in lst])

def determine_cluster_config(cluster, service, config_name):
    if service in cluster.cluster_configs:
        service_configs = cluster.cluster_configs.get(service)
        if config_name in service_configs:
            return service_configs.get(config_name)
    all_conf = get_plugin_configs()
    for conf in all_conf:
        if conf.name == config_name:
            return conf.default_value

def _set_config(cfg, gen_cfg, name=None):
    if name in gen_cfg:
        cfg.update(gen_cfg[name]['conf'])
    if name is None:
        for name in gen_cfg:
            cfg.update(gen_cfg[name]['conf'])
    return cfg

def _is_general_option_enabled(cluster, option):
    for ng in cluster.node_groups:
        conf = ng.configuration
        if 'general' in conf and option.name in conf['general']:
                return conf['general'][option.name]
    return option.default_value

def is_data_locality_enabled(cluster):
    if not CONF.enable_data_locality:
        return False
    return _is_general_option_enabled(cluster, ENABLE_DATA_LOCALITY)
<commit_msg>Revert "Fix init config bugs in config_helper"<commit_after># Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from oslo.config import cfg

from savanna.openstack.common import log as logging
from savanna.plugins import provisioning as p
from savanna.swift import swift_helper as swift
from savanna.topology import topology_helper as topology
from savanna.utils import types as types
from savanna.utils import xmlutils as x

LOG = logging.getLogger(__name__)
CONF = cfg.CONF

CORE_DEFAULT = x.load_hadoop_xml_defaults(
    'plugins/spark/resources/core-default.xml')

HDFS_DEFAULT = x.load_hadoop_xml_defaults(
    'plugins/spark/resources/hdfs-default.xml')

XML_CONFS = {
    "HDFS": [CORE_DEFAULT, HDFS_DEFAULT]
}

SPARK_CONFS = {
    "MASTER": {'SPARK_MASTER_IP', 'SPARK_MASTER_PORT',
               'SPARK_MASTER_WEBUI_PORT', 'SPARK_WORKER_CORES',
               'SPARK_WORKER_MEMORY', 'SPARK_WORKER_PORT',
               'SPARK_WORKER_WEBUI_PORT', 'SPARK_WORKER_INSTANCES'
               }
}

# TODO(aignatov): Environmental configs could be more complex
ENV_CONFS = {
    "HDFS": {
        'Name Node Heap Size': 'HADOOP_NAMENODE_OPTS=\\"-Xmx%sm\\"',
        'Data Node Heap Size': 'HADOOP_DATANODE_OPTS=\\"-Xmx%sm\\"'
    }
}

ENABLE_SWIFT = p.Config('Enable Swift', 'general', 'cluster',
                        config_type="bool", priority=1,
                        default_value=True, is_optional=True)

ENABLE_DATA_LOCALITY = p.Config('Enable Data Locality', 'general', 'cluster',
                                config_type="bool", priority=1,
                                default_value=True, is_optional=True)

HIDDEN_CONFS = ['fs.defaultFS', 'dfs.namenode.name.dir',
                'dfs.datanode.data.dir', 'mapred.job.tracker',
                'mapred.system.dir', 'mapred.local.dir',
                'hadoop.proxyuser.hadoop.hosts',
                'hadoop.proxyuser.hadoop.groups']

CLUSTER_WIDE_CONFS = ['dfs.block.size', 'dfs.permissions', 'dfs.replication',
                      'dfs.replication.min', 'dfs.replication.max',
                      'io.file.buffer.size', 'mapreduce.job.counters.max',
                      'mapred.output.compress', 'io.compression.codecs',
                      'mapred.output.compression.codec',
                      'mapred.output.compression.type',
                      'mapred.compress.map.output',
                      'mapred.map.output.compression.codec']

PRIORITY_1_CONFS = ['dfs.datanode.du.reserved',
                    'dfs.datanode.failed.volumes.tolerated',
                    'dfs.datanode.max.xcievers', 'dfs.datanode.handler.count',
                    'dfs.namenode.handler.count', 'mapred.child.java.opts',
                    'mapred.jobtracker.maxtasks.per.job',
                    'mapred.job.tracker.handler.count',
                    'mapred.map.child.java.opts',
                    'mapred.reduce.child.java.opts',
                    'io.sort.mb', 'mapred.tasktracker.map.tasks.maximum',
                    'mapred.tasktracker.reduce.tasks.maximum']

# for now we have not so many cluster-wide configs
# lets consider all of them having high priority
PRIORITY_1_CONFS += CLUSTER_WIDE_CONFS

def _initialise_configs():
    configs = []
    for service, config_lists in XML_CONFS.iteritems():
        for config_list in config_lists:
            for config in config_list:
                if config['name'] not in HIDDEN_CONFS:
                    cfg = p.Config(config['name'], service, "node",
                                   is_optional=True, config_type="string",
                                   default_value=str(config['value']),
                                   description=config['description'])
                    if cfg.default_value in ["true", "false"]:
                        cfg.config_type = "bool"
                        cfg.default_value = (cfg.default_value == 'true')
                    elif types.is_int(cfg.default_value):
                        cfg.config_type = "int"
                        cfg.default_value = int(cfg.default_value)
                    if config['name'] in CLUSTER_WIDE_CONFS:
                        cfg.scope = 'cluster'
                    if config['name'] in PRIORITY_1_CONFS:
                        cfg.priority = 1
                    configs.append(cfg)

    for service, config_items in ENV_CONFS.iteritems():
        for name, param_format_str in config_items.iteritems():
            configs.append(p.Config(name, service, "node",
                                    default_value=1024, priority=1,
                                    config_type="int"))

    for service, config_items in SPARK_CONFS.iteritems():
        for name in config_items.iteritems():
            configs.append(p.Config(name, service, "node"))

    configs.append(ENABLE_SWIFT)
    if CONF.enable_data_locality:
        configs.append(ENABLE_DATA_LOCALITY)

    return configs

# Initialise plugin Hadoop configurations
PLUGIN_CONFIGS = _initialise_configs()

def get_plugin_configs():
    return PLUGIN_CONFIGS

def get_general_configs(hive_hostname, passwd_hive_mysql):
    config = {
        ENABLE_SWIFT.name: {
            'default_value': ENABLE_SWIFT.default_value,
            'conf': extract_name_values(swift.get_swift_configs())
        }
    }
    if CONF.enable_data_locality:
        config.update({
            ENABLE_DATA_LOCALITY.name: {
                'default_value': ENABLE_DATA_LOCALITY.default_value,
                'conf': extract_name_values(topology.vm_awareness_all_config())
            }
        })
    return config

def get_config_value(service, name, cluster=None):
    if cluster:
        for ng in cluster.node_groups:
            if (ng.configuration.get(service) and
                    ng.configuration[service].get(name)):
                return ng.configuration[service][name]

    for c in PLUGIN_CONFIGS:
        if c.applicable_target == service and c.name == name:
            return c.default_value

    raise RuntimeError("Unable get parameter '%s' from service %s",
                       name, service)

def generate_cfg_from_general(cfg, configs, general_config,
                              rest_excluded=False):
    if 'general' in configs:
        for nm in general_config:
            if nm not in configs['general'] and not rest_excluded:
                configs['general'][nm] = general_config[nm]['default_value']
        for name, value in configs['general'].items():
            if value:
                cfg = _set_config(cfg, general_config, name)
                LOG.info("Applying config: %s" % name)
    else:
        cfg = _set_config(cfg, general_config)
    return cfg

def generate_xml_configs(configs, storage_path, nn_hostname, hadoop_port
                         ):
    # inserting common configs depends on provisioned VMs and HDFS placement
    # TODO(aignatov): should be moved to cluster context
    """
    dfs.name.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/namenode'),
    'dfs.data.dir': extract_hadoop_path(storage_path,
                                            '/lib/hadoop/hdfs/datanode'),
    'dfs.name.dir': storage_path + 'hdfs/name',
    'dfs.data.dir': storage_path + 'hdfs/data',

    'dfs.hosts': '/etc/hadoop/dn.incl',
    'dfs.hosts.exclude': '/etc/hadoop/dn.excl',
    """
    if hadoop_port == None:
        hadoop_port = 8020

    cfg = {
        'fs.defaultFS': 'hdfs://%s:%s' % (nn_hostname, str(hadoop_port)),
        'dfs.namenode.name.dir': extract_hadoop_path(storage_path,
                                            '/dfs/nn'),
        'dfs.datanode.data.dir': extract_hadoop_path(storage_path,
                                            '/dfs/dn'),
        'dfs.replication': 1,
    }

    # inserting user-defined configs
    for key, value in extract_xml_confs(configs):
        cfg[key] = value

    # invoking applied configs to appropriate xml files
    core_all = CORE_DEFAULT

    if CONF.enable_data_locality:
        cfg.update(topology.TOPOLOGY_CONFIG)
        # applying vm awareness configs
        core_all += topology.vm_awareness_core_config()

    xml_configs = {
        'core-site': x.create_hadoop_xml(cfg, core_all),
        'hdfs-site': x.create_hadoop_xml(cfg, HDFS_DEFAULT)
    }

    return xml_configs

def write_hadoop_configs_xml_file(xml_configs):
    f_core = open('core-site.xml', 'w')
    f_hdfs = open('hdfs-site.xml', 'w')

    f_core.writelines(xml_configs['core-site'])
    f_hdfs.writelines(xml_configs['hdfs-site'])

    f_hdfs.flush()
    f_core.flush()
    f_hdfs.close()
    f_core.close()

    return True

def generate_spark_env_configs(mastername, masterport, masterwebport=None,
                               workercores=None, workermemory=None,
                               workerport=None, workerwebport=None,
                               workerinstances=None):
    # Load configs template file
    f = open('savanna/plugins/spark/resources/spark-env.sh.template', 'r')
    default_config = f.read()
    # Write new configs
    configs = [default_config]
    configs.append('SPARK_MASTER_IP=' + mastername)
    configs.append('SPARK_MASTER_PORT=' + str(masterport))
    if masterwebport != None:
        configs.append('SPARK_MASTER_WEBUI_PORT=' + str(masterwebport))
    # configure for workers
    if workercores != None:
        configs.append('SPARK_WORKER_CORES=' + str(workercores))
    if workermemory != None:
        configs.append('SPARK_WORKER_MEMORY=' + str(workermemory))
    if workerport != None:
        configs.append('SPARK_WORKER_PORT=' + str(workerport))
    if workerwebport != None:
        configs.append('SPARK_WORKER_WEBUI_PORT=' + str(workerwebport))
    if workerinstances != None:
        configs.append('SPARK_WORKER_INSTANCES=' + str(workerinstances))
    return '\n'.join(configs)

    #workernames need to be a list of woker names
def generate_spark_slaves_configs(workernames):
    #slaves = [str(worker) for worker in workernames]
    return '\n'.join(workernames)

def write_spark_configs(configs, filename):
    f = open(str(filename), 'w')
    f.write(configs)
    f.flush()
    f.close()
    return

def extract_environment_confs(configs):
    """Returns list of Hadoop parameters which should be passed via environment
    """
    lst = []
    for service, srv_confs in configs.items():
        if ENV_CONFS.get(service):
            for param_name, param_value in srv_confs.items():
                for cfg_name, cfg_format_str in ENV_CONFS[service].items():
                    if param_name == cfg_name and param_value is not None:
                        lst.append(cfg_format_str % param_value)
        else:
            LOG.warn("Plugin recieved wrong applicable target '%s' in "
                     "environmental configs" % service)
    return lst

def extract_xml_confs(configs):
    """Returns list of Hadoop parameters which should be passed into general
    configs like core-site.xml
    """
    lst = []
    for service, srv_confs in configs.items():
        if XML_CONFS.get(service):
            for param_name, param_value in srv_confs.items():
                for cfg_list in XML_CONFS[service]:
                    names = [cfg['name'] for cfg in cfg_list]
                    if param_name in names and param_value is not None:
                        lst.append((param_name, param_value))
        else:
            LOG.warn("Plugin recieved wrong applicable target '%s' for "
                     "xml configs" % service)
    return lst

def generate_setup_script(storage_paths, env_configs):
    script_lines = ["#!/bin/bash -x"]
    script_lines.append("echo -n > /tmp/hadoop-env.sh")
    for line in env_configs:
        if 'HADOOP' in line:
            script_lines.append('echo "%s" >> /tmp/hadoop-env.sh' % line)
    script_lines.append("cat /etc/hadoop/hadoop-env.sh >> /tmp/hadoop-env.sh")
    script_lines.append("cp /tmp/hadoop-env.sh /etc/hadoop/hadoop-env.sh")

    hadoop_log = storage_paths[0] + "/log/hadoop/\$USER/"
    script_lines.append('sed -i "s,export HADOOP_LOG_DIR=.*,'
                        'export HADOOP_LOG_DIR=%s," /etc/hadoop/hadoop-env.sh'
                        % hadoop_log)

    hadoop_log = storage_paths[0] + "/log/hadoop/hdfs"
    script_lines.append('sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,'
                        'export HADOOP_SECURE_DN_LOG_DIR=%s," '
                        '/etc/hadoop/hadoop-env.sh' % hadoop_log)

    for path in storage_paths:
        script_lines.append("chown -R hadoop:hadoop %s" % path)
        script_lines.append("chmod -R 755 %s" % path)
    return "\n".join(script_lines)

def extract_name_values(configs):
    return dict((cfg['name'], cfg['value']) for cfg in configs)

def extract_hadoop_path(lst, hadoop_dir):
    return ",".join([p + hadoop_dir for p in lst])

def determine_cluster_config(cluster, service, config_name):
    if service in cluster.cluster_configs:
        service_configs = cluster.cluster_configs.get(service)
        if config_name in service_configs:
            return service_configs.get(config_name)
    all_conf = get_plugin_configs()
    for conf in all_conf:
        if conf.name == config_name:
            return conf.default_value

def _set_config(cfg, gen_cfg, name=None):
    if name in gen_cfg:
        cfg.update(gen_cfg[name]['conf'])
    if name is None:
        for name in gen_cfg:
            cfg.update(gen_cfg[name]['conf'])
    return cfg

def _is_general_option_enabled(cluster, option):
    for ng in cluster.node_groups:
        conf = ng.configuration
        if 'general' in conf and option.name in conf['general']:
                return conf['general'][option.name]
    return option.default_value

def is_data_locality_enabled(cluster):
    if not CONF.enable_data_locality:
        return False
    return _is_general_option_enabled(cluster, ENABLE_DATA_LOCALITY)
<|endoftext|>


<commit_before>from os import remove
from re import compile

from django.conf import settings
from django.contrib.sites.models import Site
from django.template.response import HttpResponse
from django.views.generic import TemplateView

from wkhtmltopdf.utils import template_to_temp_file, wkhtmltopdf

class PDFResponse(HttpResponse):
    def __init__(self, content, filename):
        super(PdfResponse, self).__init__(content, 'application/pdf')
        self.__setitem__('Content-Disposition', 'attachment; filename=%s' % filename)

class PdfResponse(PDFResponse):
    def __init__(self, content, filename):
        warning = '''PdfTemplateView is deprecated in favour of PDFTemplateView. It will be removed in version 1.'''
        raise PendingDeprecationWarning(warning)
        super(PdfResponse, self).__init__(content, filename)

class PDFTemplateView(TemplateView):
    filename = 'rendered_pdf.pdf'
    footer_template = None
    header_template = None
    margin_bottom = 0
    margin_left = 0
    margin_right = 0
    margin_top = 0
    response = PdfResponse

    def get(self, request, context_instance=None, *args, **kwargs):
        if request.GET.get('as', '') == 'html':
            super(PdfTemplateView, self).get(request, *args, **kwargs)

        page_path = template_to_temp_file(self.template_name, self.get_context_data(), context_instance)

        tmp_files = []
        if self.header_template:
            kwargs['header_html'] = template_to_temp_file(self.header_template, self.get_context_data(), context_instance)
            tmp_files.append(kwargs['header_html'])
        if self.footer_template:
            kwargs['footer_html'] = template_to_temp_file(self.footer_template, self.get_context_data(), context_instance)
            tmp_files.append(kwargs['footer_html'])

        map(remove, tmp_files)

        kwargs.update({
            'margin_bottom': self.margin_bottom,
            'margin_left': self.margin_left,
            'margin_right': self.margin_right,
            'margin_top': self.margin_top
        })
        return self.response(wkhtmltopdf(page_path, **kwargs), self.filename)

    def get_context_data(self, **kwargs):
        context = super(PdfTemplateView, self).get_context_data(**kwargs)

        match_full_url = compile(r'^https?://')
        if not match_full_url.match(settings.STATIC_URL):
            context['STATIC_URL'] = 'http://' + Site.objects.get_current().domain + settings.STATIC_URL
        if not match_full_url.match(settings.MEDIA_URL):
            context['MEDIA_URL'] = 'http://' + Site.objects.get_current().domain + settings.MEDIA_URL

        return context

class PdfTemplateView(PDFTemplateView): #TODO: Remove this in v1.0
    def as_view(cls, **initkwargs):
        warning = '''PdfTemplateView is deprecated in favour of PDFTemplateView. It will be removed in version 1.'''
        raise PendingDeprecationWarning(warning)
        return super(PdfTemplateView, cls).as_view(**initkwargs)<commit_msg>Make PDFTemplateView use PDFResponse.<commit_after>from os import remove
from re import compile

from django.conf import settings
from django.contrib.sites.models import Site
from django.template.response import HttpResponse
from django.views.generic import TemplateView

from wkhtmltopdf.utils import template_to_temp_file, wkhtmltopdf

class PDFResponse(HttpResponse):
    def __init__(self, content, filename):
        super(PdfResponse, self).__init__(content, 'application/pdf')
        self.__setitem__('Content-Disposition', 'attachment; filename=%s' % filename)

class PdfResponse(PDFResponse):
    def __init__(self, content, filename):
        warning = '''PdfTemplateView is deprecated in favour of PDFTemplateView. It will be removed in version 1.'''
        raise PendingDeprecationWarning(warning)
        super(PdfResponse, self).__init__(content, filename)

class PDFTemplateView(TemplateView):
    filename = 'rendered_pdf.pdf'
    footer_template = None
    header_template = None
    margin_bottom = 0
    margin_left = 0
    margin_right = 0
    margin_top = 0
    response = PDFResponse

    def get(self, request, context_instance=None, *args, **kwargs):
        if request.GET.get('as', '') == 'html':
            super(PdfTemplateView, self).get(request, *args, **kwargs)

        page_path = template_to_temp_file(self.template_name, self.get_context_data(), context_instance)

        tmp_files = []
        if self.header_template:
            kwargs['header_html'] = template_to_temp_file(self.header_template, self.get_context_data(), context_instance)
            tmp_files.append(kwargs['header_html'])
        if self.footer_template:
            kwargs['footer_html'] = template_to_temp_file(self.footer_template, self.get_context_data(), context_instance)
            tmp_files.append(kwargs['footer_html'])

        map(remove, tmp_files)

        kwargs.update({
            'margin_bottom': self.margin_bottom,
            'margin_left': self.margin_left,
            'margin_right': self.margin_right,
            'margin_top': self.margin_top
        })
        return self.response(wkhtmltopdf(page_path, **kwargs), self.filename)

    def get_context_data(self, **kwargs):
        context = super(PdfTemplateView, self).get_context_data(**kwargs)

        match_full_url = compile(r'^https?://')
        if not match_full_url.match(settings.STATIC_URL):
            context['STATIC_URL'] = 'http://' + Site.objects.get_current().domain + settings.STATIC_URL
        if not match_full_url.match(settings.MEDIA_URL):
            context['MEDIA_URL'] = 'http://' + Site.objects.get_current().domain + settings.MEDIA_URL

        return context

class PdfTemplateView(PDFTemplateView): #TODO: Remove this in v1.0
    def as_view(cls, **initkwargs):
        warning = '''PdfTemplateView is deprecated in favour of PDFTemplateView. It will be removed in version 1.'''
        raise PendingDeprecationWarning(warning)
        return super(PdfTemplateView, cls).as_view(**initkwargs)<|endoftext|>


<commit_before>from django.contrib.contenttypes import generic
from django.db import models
from mezzanine.pages.models import Page
from hs_core.models import AbstractResource, resource_processor, CoreMetaData, AbstractMetaDataElement
from mezzanine.pages.page_processors import processor_for
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist, ValidationError

class BandInformation(AbstractMetaDataElement):
    term = 'BandInformation'
    # required fields
    # has to call the field name rather than bandName, which seems to be enforced by the AbstractMetaDataElement;
    # otherwise, got an error indicating required "name" field does not exist
    name = models.CharField(max_length=50, null=True)
    variableName = models.TextField(null=True)
    variableUnit = models.CharField(max_length=50, null=True)
    # optional fields
    method = models.TextField(null=True, blank=True)
    comment = models.TextField(null=True, blank=True)
    def __unicode__(self):
        self.name
    @classmethod
    def create(cls, **kwargs):
        # Check the required fields and create new BandInformation meta instance
        if 'name' in kwargs:
            # check if the variable metadata already exists
            metadata_obj = kwargs['content_object']
            metadata_type = ContentType.objects.get_for_model(metadata_obj)
            band_info = BandInformation.objects.filter(name__iexact=kwargs['name'], object_id=metadata_obj.id,
                                                       content_type=metadata_type).first()
            if band_info:
                raise ValidationError('BandInformation name:%s already exists' % kwargs['name'])
        else:
            raise ValidationError("name of BandInformation is missing.")

        if not 'variableName' in kwargs:
            raise ValidationError("BandInformation variableName is missing.")

        if not 'variableUnit' in kwargs:
            raise ValidationError("BandInformation variableUnit is missing.")

        band_info = BandInformation.objects.create(name=kwargs['name'], variableName=kwargs['variableName'],
                                                   variableUnit=kwargs['variableUnit'], content_object=metadata_obj)

        # check for the optional fields and save them to the BandInformation metadata
        for key, value in kwargs.iteritems():
            if key in ('method', 'comment'):
                setattr(band_info, key, value)

            band_info.save()

        return band_info

    @classmethod
    def update(cls, element_id, **kwargs):
        band_info = BandInformation.objects.get(id=element_id)
        if band_info:
            if 'name' in kwargs:
                if band_info.name != kwargs['name']:
                    # check to make sure this new name not already exists
                    if BandInformation.objects.filter(name_iexact=kwargs['name'], object_id=band_info.object_id,
                                                      content_type__pk=band_info.content_type.id).count()> 0:
                        raise ValidationError('BandInformation name:%s already exists.' % kwargs['name'])

                band_info.name = kwargs['name']

            for key, value in kwargs.iteritems():
                if key in ('variableName', 'variableUnit', 'method', 'comment'):
                    setattr(band_info, key, value)

            band_info.save()
        else:
            raise ObjectDoesNotExist("No BandInformation element can be found for the provided id:%s" % kwargs['id'])

    @classmethod
    def remove(cls, element_id):
        band_info = BandInformation.objects.get(id=element_id)
        if band_info:
            # make sure we are not deleting all bandInformation of a resource
            if BandInformation.objects.filter(object_id=band_info.object_id, content_type__pk=band_info.content_type.id).count() == 1:
                raise ValidationError("The only BandInformation of the resource can't be deleted.")
            band_info.delete()
        else:
            raise ObjectDoesNotExist("No BandInformation element can be found for id:%d." % element_id)

class CellInformation(AbstractMetaDataElement):
    term = 'CellInformation'
    # required fields
    name = models.CharField(max_length=50, null=True)

    rows = models.IntegerField(null=True)
    columns = models.IntegerField(null=True)
    cellSizeXValue = models.FloatField(null=True)
    cellSizeYValue = models.FloatField(null=True)
    cellSizeUnit = models.CharField(max_length=50, null=True)
    cellDataType = models.CharField(max_length=50, null=True)

    # optional fields
    noDataValue = models.FloatField(null=True)

    def __unicode__(self):
        self.name
    @classmethod
    def create(cls, **kwargs):
        # Check the required fields and create new BandInformation meta instance
        if 'name' in kwargs:
            # check if the variable metadata already exists
            metadata_obj = kwargs['content_object']
            metadata_type = ContentType.objects.get_for_model(metadata_obj)
            cell_info = CellInformation.objects.filter(name__iexact=kwargs['name'], object_id=metadata_obj.id,
                                                       content_type=metadata_type).first()
            if cell_info:
                raise ValidationError('CellInformation name:%s already exists' % kwargs['name'])
        else:
            raise ValidationError("name of CellInformation is missing.")

        if not 'rows' in kwargs:
            raise ValidationError("CellInformation rows is missing.")

        if not 'columns' in kwargs:
            raise ValidationError("CellInformation columns is missing.")

        if not 'cellSizeXValue' in kwargs:
            raise ValidationError("CellInformation cellSizeXValue is missing.")

        if not 'cellSizeYValue' in kwargs:
            raise ValidationError("CellInformation cellSizeYValue is missing.")

        if not 'cellSizeUnit' in kwargs:
            raise ValidationError("CellInformation cellSizeUnit is missing.")

        if not 'cellDataType' in kwargs:
            raise ValidationError("CellInformation cellDataType is missing.")

        cell_info = CellInformation.objects.create(name=kwargs['name'], rows=kwargs['rows'], columns=kwargs['columns'],
                                                   cellSizeXValue=kwargs['cellSizeXValue'], cellSizeYValue=kwargs['cellSizeYValue'],
                                                   cellSizeUnit=kwargs['cellSizeUnit'], cellDataType=kwargs['cellDataType'],
                                                   content_object=metadata_obj)

        # check for the optional fields and save them to the BandInformation metadata
        for key, value in kwargs.iteritems():
            if key in ('noDataValue'):
                setattr(cell_info, key, value)

            cell_info.save()

        return cell_info

    @classmethod
    def update(cls, element_id, **kwargs):
        cell_info = CellInformation.objects.get(id=element_id)
        if cell_info:
            if 'name' in kwargs:
                if cell_info.name != kwargs['name']:
                    # check to make sure this new name not already exists
                    if CellInformation.objects.filter(name_iexact=kwargs['name'], object_id=cell_info.object_id,
                                                      content_type__pk=cell_info.content_type.id).count()> 0:
                        raise ValidationError('CellInformation name:%s already exists.' % kwargs['name'])

                cell_info.name = kwargs['name']

            for key, value in kwargs.iteritems():
                if key in ('rows', 'columns', 'cellSizeXValue', 'cellSizeYValue', 'cellSizeUnit', 'cellDataType', 'noDataValue'):
                    setattr(cell_info, key, value)

            cell_info.save()
        else:
            raise ObjectDoesNotExist("No CellInformation element can be found for the provided id:%s" % kwargs['id'])

    @classmethod
    def remove(cls, element_id):
        cell_info = CellInformation.objects.get(id=element_id)
        if cell_info:
            # make sure we are not deleting all CellInformation of a resource
            if CellInformation.objects.filter(object_id=cell_info.object_id, content_type__pk=cell_info.content_type.id).count() == 1:
                raise ValidationError("The only CellInformation of the resource can't be deleted.")
            cell_info.delete()
        else:
            raise ObjectDoesNotExist("No CellInformation element can be found for id:%d." % element_id)

#
# To create a new resource, use these two super-classes.
#
class RasterResource(Page, AbstractResource):
    class Meta:
        verbose_name = 'Geographic Raster Resource'

    @property
    def metadata(self):
        md = RasterMetaData()
        return self._get_metadata(md)

    def can_add(self, request):
        return AbstractResource.can_add(self, request)

    def can_change(self, request):
        return AbstractResource.can_change(self, request)

    def can_delete(self, request):
        return AbstractResource.can_delete(self, request)

    def can_view(self, request):
        return AbstractResource.can_view(self, request)

class RasterMetaData(CoreMetaData):
    # required non-repeatable cell information metadata elements
    cellInformation = generic.GenericRelation(CellInformation)
    bandInformation = generic.GenericRelation(BandInformation)
    raster_resource = generic.GenericRelation(RasterResource)

    @classmethod
    def get_supported_element_names(cls):
        # get the names of all core metadata elements
        elements = super(RasterMetaData, cls).get_supported_element_names()
        # add the name of any additional element to the list
        elements.append('CellInformation')
        elements.append('BandInformation')
        return elements

    @property
    def resource(self):
        return self.raster_resource.all().first()

    def get_xml(self):
        from lxml import etree
        # get the xml string representation of the core metadata elements
        xml_string = super(RasterMetaData, self).get_xml(pretty_print=False)

        # create an etree xml object
        RDF_ROOT = etree.fromstring(xml_string)

        # get root 'Description' element that contains all other elements
        container = RDF_ROOT.find('rdf:Description', namespaces=self.NAMESPACES)

        # inject raster resource specific metadata elements to container element
        for cell_info in self.cellInformation.all():
            hsterms_cellInfo = etree.SubElement(container, '{%s}rasterCellInformation' % self.NAMESPACES['hsterms'])
            hsterms_cellInfo_rdf_Description = etree.SubElement(hsterms_cellInfo, '{%s}Description' % self.NAMESPACES['rdf'])

            hsterms_name = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}name' % self.NAMESPACES['hsterms'])
            hsterms_name.text = cell_info.name

            hsterms_rows = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}rows' % self.NAMESPACES['hsterms'])
            hsterms_rows.text = str(cell_info.rows)

            hsterms_columns = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}columns' % self.NAMESPACES['hsterms'])
            hsterms_columns.text = str(cell_info.columns)

            hsterms_cellSizeXValue = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeXValue' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeXValue.text = str(cell_info.cellSizeXValue)

            hsterms_cellSizeYValue = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeYValue' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeYValue.text = str(cell_info.cellSizeYValue)

            hsterms_cellSizeUnit = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeUnit' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeUnit.text = cell_info.cellSizeUnit

            hsterms_cellDataType = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellDataType' % self.NAMESPACES['hsterms'])
            hsterms_cellDataType.text = cell_info.cellDataType

        if cell_info.noDataValue:
            hsterms_noDataValue = etree.SubElement(hsterms_cellInfo_rdf_Description,'{%s}noDataValue' % self.NAMESPACES['hsterms'])
            hsterms_noDataValue.text = str(cell_info.noDataValue)

        for band_info in self.bandInformation.all():
            hsterms_bandInfo = etree.SubElement(container, '{%s}rasterBandInformation' % self.NAMESPACES['hsterms'])
            hsterms_bandInfo_rdf_Description = etree.SubElement(hsterms_bandInfo, '{%s}Description' % self.NAMESPACES['rdf'])

            hsterms_name = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}name' % self.NAMESPACES['hsterms'])
            hsterms_name.text = band_info.name

            hsterms_variableName = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}variableName' % self.NAMESPACES['hsterms'])
            hsterms_variableName.text = band_info.variableName

            hsterms_variableUnit = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}variableUnit' % self.NAMESPACES['hsterms'])
            hsterms_variableUnit.text = band_info.variableUnit

            if band_info.method:
                hsterms_method = etree.SubElement(hsterms_bandInfo_rdf_Description,'{%s}method' % self.NAMESPACES['hsterms'])
                hsterms_method.text = band_info.method

            if band_info.comment:
                hsterms_comment = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}comment' % self.NAMESPACES['hsterms'])
                hsterms_comment.text = band_info.comment

        return etree.tostring(RDF_ROOT, pretty_print=True)

processor_for(RasterResource)(resource_processor)
# page processor to populate raster resource specific metadata into my-resources template page
@processor_for(RasterResource)
def main_page(request, page):
    from dublincore import models as dc
    from django import forms
    from collections import OrderedDict
    class DCTerm(forms.ModelForm):
        class Meta:
            model=dc.QualifiedDublinCoreElement
            fields = ['term', 'content']

    content_model = page.get_content_model()

    # create Coverage metadata
    md_dict = OrderedDict()

    md_dict['rows'] = content_model.metadata.cellInformation.all()[0].rows
    md_dict['columns'] = content_model.metadata.cellInformation.all()[0].columns
    md_dict['cellSizeXValue'] = content_model.metadata.cellInformation.all()[0].cellSizeXValue
    md_dict['cellSizeYValue'] = content_model.metadata.cellInformation.all()[0].cellSizeYValue
    md_dict['cellSizeUnit'] = content_model.metadata.cellInformation.all()[0].cellSizeUnit
    md_dict['cellDataType'] = content_model.metadata.cellInformation.all()[0].cellDataType
    md_dict['noDataValue'] = content_model.metadata.cellInformation.all()[0].noDataValue

    band_dict = OrderedDict()
    i = 1
    for band in content_model.metadata.bandInformation.all():
         band_dict['name (band '+str(i)+')'] = band.name
         band_dict['variable (band '+str(i)+')'] = band.variableName
         band_dict['units (band '+str(i)+')'] = band.variableUnit
         band_dict['method (band '+str(i)+')'] = band.method
         band_dict['comment (band '+str(i)+')'] = band.comment
         i = i+1

    return  { 'res_add_metadata': md_dict,
              'band_metadata': band_dict,
              'resource_type' : content_model._meta.verbose_name,
              'dublin_core' : [t for t in content_model.dublin_metadata.all().exclude(term='AB')],
              'dcterm_frm' : DCTerm()
            }
import receivers
<commit_msg>made raster resource landing page work<commit_after>from django.contrib.contenttypes import generic
from django.db import models
from mezzanine.pages.models import Page
from hs_core.models import AbstractResource, resource_processor, CoreMetaData, AbstractMetaDataElement
from mezzanine.pages.page_processors import processor_for
from django.contrib.contenttypes.models import ContentType
from django.core.exceptions import ObjectDoesNotExist, ValidationError
from hs_core import page_processors

class BandInformation(AbstractMetaDataElement):
    term = 'BandInformation'
    # required fields
    # has to call the field name rather than bandName, which seems to be enforced by the AbstractMetaDataElement;
    # otherwise, got an error indicating required "name" field does not exist
    name = models.CharField(max_length=50, null=True)
    variableName = models.TextField(null=True)
    variableUnit = models.CharField(max_length=50, null=True)
    # optional fields
    method = models.TextField(null=True, blank=True)
    comment = models.TextField(null=True, blank=True)
    def __unicode__(self):
        self.name
    @classmethod
    def create(cls, **kwargs):
        # Check the required fields and create new BandInformation meta instance
        if 'name' in kwargs:
            # check if the variable metadata already exists
            metadata_obj = kwargs['content_object']
            metadata_type = ContentType.objects.get_for_model(metadata_obj)
            band_info = BandInformation.objects.filter(name__iexact=kwargs['name'], object_id=metadata_obj.id,
                                                       content_type=metadata_type).first()
            if band_info:
                raise ValidationError('BandInformation name:%s already exists' % kwargs['name'])
        else:
            raise ValidationError("name of BandInformation is missing.")

        if not 'variableName' in kwargs:
            raise ValidationError("BandInformation variableName is missing.")

        if not 'variableUnit' in kwargs:
            raise ValidationError("BandInformation variableUnit is missing.")

        band_info = BandInformation.objects.create(name=kwargs['name'], variableName=kwargs['variableName'],
                                                   variableUnit=kwargs['variableUnit'], content_object=metadata_obj)

        # check for the optional fields and save them to the BandInformation metadata
        for key, value in kwargs.iteritems():
            if key in ('method', 'comment'):
                setattr(band_info, key, value)

            band_info.save()

        return band_info

    @classmethod
    def update(cls, element_id, **kwargs):
        band_info = BandInformation.objects.get(id=element_id)
        if band_info:
            if 'name' in kwargs:
                if band_info.name != kwargs['name']:
                    # check to make sure this new name not already exists
                    if BandInformation.objects.filter(name_iexact=kwargs['name'], object_id=band_info.object_id,
                                                      content_type__pk=band_info.content_type.id).count()> 0:
                        raise ValidationError('BandInformation name:%s already exists.' % kwargs['name'])

                band_info.name = kwargs['name']

            for key, value in kwargs.iteritems():
                if key in ('variableName', 'variableUnit', 'method', 'comment'):
                    setattr(band_info, key, value)

            band_info.save()
        else:
            raise ObjectDoesNotExist("No BandInformation element can be found for the provided id:%s" % kwargs['id'])

    @classmethod
    def remove(cls, element_id):
        band_info = BandInformation.objects.get(id=element_id)
        if band_info:
            # make sure we are not deleting all bandInformation of a resource
            if BandInformation.objects.filter(object_id=band_info.object_id, content_type__pk=band_info.content_type.id).count() == 1:
                raise ValidationError("The only BandInformation of the resource can't be deleted.")
            band_info.delete()
        else:
            raise ObjectDoesNotExist("No BandInformation element can be found for id:%d." % element_id)

class CellInformation(AbstractMetaDataElement):
    term = 'CellInformation'
    # required fields
    name = models.CharField(max_length=50, null=True)

    rows = models.IntegerField(null=True)
    columns = models.IntegerField(null=True)
    cellSizeXValue = models.FloatField(null=True)
    cellSizeYValue = models.FloatField(null=True)
    cellSizeUnit = models.CharField(max_length=50, null=True)
    cellDataType = models.CharField(max_length=50, null=True)

    # optional fields
    noDataValue = models.FloatField(null=True)

    def __unicode__(self):
        self.name
    @classmethod
    def create(cls, **kwargs):
        # Check the required fields and create new BandInformation meta instance
        if 'name' in kwargs:
            # check if the variable metadata already exists
            metadata_obj = kwargs['content_object']
            metadata_type = ContentType.objects.get_for_model(metadata_obj)
            cell_info = CellInformation.objects.filter(name__iexact=kwargs['name'], object_id=metadata_obj.id,
                                                       content_type=metadata_type).first()
            if cell_info:
                raise ValidationError('CellInformation name:%s already exists' % kwargs['name'])
        else:
            raise ValidationError("name of CellInformation is missing.")

        if not 'rows' in kwargs:
            raise ValidationError("CellInformation rows is missing.")

        if not 'columns' in kwargs:
            raise ValidationError("CellInformation columns is missing.")

        if not 'cellSizeXValue' in kwargs:
            raise ValidationError("CellInformation cellSizeXValue is missing.")

        if not 'cellSizeYValue' in kwargs:
            raise ValidationError("CellInformation cellSizeYValue is missing.")

        if not 'cellSizeUnit' in kwargs:
            raise ValidationError("CellInformation cellSizeUnit is missing.")

        if not 'cellDataType' in kwargs:
            raise ValidationError("CellInformation cellDataType is missing.")

        cell_info = CellInformation.objects.create(name=kwargs['name'], rows=kwargs['rows'], columns=kwargs['columns'],
                                                   cellSizeXValue=kwargs['cellSizeXValue'], cellSizeYValue=kwargs['cellSizeYValue'],
                                                   cellSizeUnit=kwargs['cellSizeUnit'], cellDataType=kwargs['cellDataType'],
                                                   content_object=metadata_obj)

        # check for the optional fields and save them to the BandInformation metadata
        for key, value in kwargs.iteritems():
            if key in ('noDataValue'):
                setattr(cell_info, key, value)

            cell_info.save()

        return cell_info

    @classmethod
    def update(cls, element_id, **kwargs):
        cell_info = CellInformation.objects.get(id=element_id)
        if cell_info:
            if 'name' in kwargs:
                if cell_info.name != kwargs['name']:
                    # check to make sure this new name not already exists
                    if CellInformation.objects.filter(name_iexact=kwargs['name'], object_id=cell_info.object_id,
                                                      content_type__pk=cell_info.content_type.id).count()> 0:
                        raise ValidationError('CellInformation name:%s already exists.' % kwargs['name'])

                cell_info.name = kwargs['name']

            for key, value in kwargs.iteritems():
                if key in ('rows', 'columns', 'cellSizeXValue', 'cellSizeYValue', 'cellSizeUnit', 'cellDataType', 'noDataValue'):
                    setattr(cell_info, key, value)

            cell_info.save()
        else:
            raise ObjectDoesNotExist("No CellInformation element can be found for the provided id:%s" % kwargs['id'])

    @classmethod
    def remove(cls, element_id):
        cell_info = CellInformation.objects.get(id=element_id)
        if cell_info:
            # make sure we are not deleting all CellInformation of a resource
            if CellInformation.objects.filter(object_id=cell_info.object_id, content_type__pk=cell_info.content_type.id).count() == 1:
                raise ValidationError("The only CellInformation of the resource can't be deleted.")
            cell_info.delete()
        else:
            raise ObjectDoesNotExist("No CellInformation element can be found for id:%d." % element_id)

#
# To create a new resource, use these two super-classes.
#
class RasterResource(Page, AbstractResource):
    class Meta:
        verbose_name = 'Geographic Raster Resource'

    @property
    def metadata(self):
        md = RasterMetaData()
        return self._get_metadata(md)

    def can_add(self, request):
        return AbstractResource.can_add(self, request)

    def can_change(self, request):
        return AbstractResource.can_change(self, request)

    def can_delete(self, request):
        return AbstractResource.can_delete(self, request)

    def can_view(self, request):
        return AbstractResource.can_view(self, request)

    def get_citation(self):
        citation = ''

        first_author = self.metadata.creators.all().filter(order=1)[0]
        name_parts = first_author.name.split(" ")
        if len(name_parts) > 2:
            citation = "{last_name}, {first_initial}.{middle_initial}.".format(last_name=name_parts[-1],
                                                                              first_initial=name_parts[0][0],
                                                                              middle_initial=name_parts[1][0]) + ", "
        else:
            citation = "{last_name}, {first_initial}.".format(last_name=name_parts[-1],
                                                              first_initial=name_parts[0][0]) + ", "

        other_authors = self.metadata.creators.all().filter(order__gt=1)
        for author in other_authors:
            name_parts = author.name.split(" ")
            if len(name_parts) > 2:
                citation += "{first_initial}.{middle_initial}.{last_name}".format(first_initial=name_parts[0][0],
                                                                                middle_initial=name_parts[1][0],
                                                                                last_name=name_parts[-1]) + ", "
            else:
                citation += "{first_initial}.{last_name}".format(first_initial=name_parts[0][0],
                                                                 last_name=name_parts[-1]) + ", "

        citation = citation[:-2]
        if self.metadata.dates.all().filter(type='published'):
            citation_date = self.metadata.dates.all().filter(type='published')[0]
        else:
            citation_date = self.metadata.dates.all().filter(type='modified')[0]

        citation += " ({year}). ".format(year=citation_date.start_date.year)
        citation += self.title
        citation += ", HydroShare, "

        if self.metadata.identifiers.all().filter(name="doi"):
            hs_identifier = self.metadata.identifiers.all().filter(name="doi")[0]
        else:
            hs_identifier = self.metadata.identifiers.all().filter(name="hydroShareIdentifier")[0]

        citation += "{url}".format(url=hs_identifier.url)

        return citation

class RasterMetaData(CoreMetaData):
    # required non-repeatable cell information metadata elements
    cellInformation = generic.GenericRelation(CellInformation)
    bandInformation = generic.GenericRelation(BandInformation)
    raster_resource = generic.GenericRelation(RasterResource)

    @classmethod
    def get_supported_element_names(cls):
        # get the names of all core metadata elements
        elements = super(RasterMetaData, cls).get_supported_element_names()
        # add the name of any additional element to the list
        elements.append('CellInformation')
        elements.append('BandInformation')
        return elements

    @property
    def resource(self):
        return self.raster_resource.all().first()

    def get_xml(self):
        from lxml import etree
        # get the xml string representation of the core metadata elements
        xml_string = super(RasterMetaData, self).get_xml(pretty_print=False)

        # create an etree xml object
        RDF_ROOT = etree.fromstring(xml_string)

        # get root 'Description' element that contains all other elements
        container = RDF_ROOT.find('rdf:Description', namespaces=self.NAMESPACES)

        # inject raster resource specific metadata elements to container element
        for cell_info in self.cellInformation.all():
            hsterms_cellInfo = etree.SubElement(container, '{%s}rasterCellInformation' % self.NAMESPACES['hsterms'])
            hsterms_cellInfo_rdf_Description = etree.SubElement(hsterms_cellInfo, '{%s}Description' % self.NAMESPACES['rdf'])

            hsterms_name = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}name' % self.NAMESPACES['hsterms'])
            hsterms_name.text = cell_info.name

            hsterms_rows = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}rows' % self.NAMESPACES['hsterms'])
            hsterms_rows.text = str(cell_info.rows)

            hsterms_columns = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}columns' % self.NAMESPACES['hsterms'])
            hsterms_columns.text = str(cell_info.columns)

            hsterms_cellSizeXValue = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeXValue' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeXValue.text = str(cell_info.cellSizeXValue)

            hsterms_cellSizeYValue = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeYValue' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeYValue.text = str(cell_info.cellSizeYValue)

            hsterms_cellSizeUnit = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellSizeUnit' % self.NAMESPACES['hsterms'])
            hsterms_cellSizeUnit.text = cell_info.cellSizeUnit

            hsterms_cellDataType = etree.SubElement(hsterms_cellInfo_rdf_Description, '{%s}cellDataType' % self.NAMESPACES['hsterms'])
            hsterms_cellDataType.text = cell_info.cellDataType

        if cell_info.noDataValue:
            hsterms_noDataValue = etree.SubElement(hsterms_cellInfo_rdf_Description,'{%s}noDataValue' % self.NAMESPACES['hsterms'])
            hsterms_noDataValue.text = str(cell_info.noDataValue)

        for band_info in self.bandInformation.all():
            hsterms_bandInfo = etree.SubElement(container, '{%s}rasterBandInformation' % self.NAMESPACES['hsterms'])
            hsterms_bandInfo_rdf_Description = etree.SubElement(hsterms_bandInfo, '{%s}Description' % self.NAMESPACES['rdf'])

            hsterms_name = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}name' % self.NAMESPACES['hsterms'])
            hsterms_name.text = band_info.name

            hsterms_variableName = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}variableName' % self.NAMESPACES['hsterms'])
            hsterms_variableName.text = band_info.variableName

            hsterms_variableUnit = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}variableUnit' % self.NAMESPACES['hsterms'])
            hsterms_variableUnit.text = band_info.variableUnit

            if band_info.method:
                hsterms_method = etree.SubElement(hsterms_bandInfo_rdf_Description,'{%s}method' % self.NAMESPACES['hsterms'])
                hsterms_method.text = band_info.method

            if band_info.comment:
                hsterms_comment = etree.SubElement(hsterms_bandInfo_rdf_Description, '{%s}comment' % self.NAMESPACES['hsterms'])
                hsterms_comment.text = band_info.comment

        return etree.tostring(RDF_ROOT, pretty_print=True)

processor_for(RasterResource)(resource_processor)
# page processor to populate raster resource specific metadata into my-resources template page
@processor_for(RasterResource)
def main_page(request, page):
    from dublincore import models as dc
    from django import forms
    from collections import OrderedDict

    content_model = page.get_content_model()
    edit_mode = False
    if content_model.creator == request.user or request.user in content_model.owners.all() or request.user in content_model.edit_users.all():
        edit_mode = True

    md_dict = OrderedDict()

    md_dict['rows'] = content_model.metadata.cellInformation.all()[0].rows
    md_dict['columns'] = content_model.metadata.cellInformation.all()[0].columns
    md_dict['cellSizeXValue'] = content_model.metadata.cellInformation.all()[0].cellSizeXValue
    md_dict['cellSizeYValue'] = content_model.metadata.cellInformation.all()[0].cellSizeYValue
    md_dict['cellSizeUnit'] = content_model.metadata.cellInformation.all()[0].cellSizeUnit
    md_dict['cellDataType'] = content_model.metadata.cellInformation.all()[0].cellDataType
    md_dict['noDataValue'] = content_model.metadata.cellInformation.all()[0].noDataValue

    band_dict = OrderedDict()
    i = 1
    for band in content_model.metadata.bandInformation.all():
         band_dict['name (band '+str(i)+')'] = band.name
         band_dict['variable (band '+str(i)+')'] = band.variableName
         band_dict['units (band '+str(i)+')'] = band.variableUnit
         band_dict['method (band '+str(i)+')'] = band.method
         band_dict['comment (band '+str(i)+')'] = band.comment
         i = i+1

    context = page_processors.get_page_context(page, request.user)
    context['res_add_metadata'] = md_dict
    context['band_metadata'] = band_dict
    context['resource_type'] = content_model._meta.verbose_name

    return context

    # return  { 'res_add_metadata': md_dict,
    #           'band_metadata': band_dict,
    #           'resource_type' : content_model._meta.verbose_name
    #         }
import receivers
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Script run by upload.py when a file is received.
"""

import sys, os
import subprocess
import gnupg

from mylogging import debug, info, warning, error, critical

from boto.s3.connection import S3Connection
from boto.s3.key import Key

### SETTINGS ###
PUBLIC_KEY_ID = ''  # public key to use to encrypt files
                    # stored in gpg keyring
AWS_ACCESS_KEY = ''
AWS_SECRET_KEY = ''

BUCKET_NAME = 'haps-dev'
TEMP_DIR = "/tmp"

try:
    from settings import *
except ImportError:
    pass

def whereis(program):
    '''
    Returns the path to program on a user's path, if it exists
    Returns None if it does not
    '''
    for path in os.environ.get('PATH', '').split(':'):
        if os.path.exists(os.path.join(path, program)) and \
           not os.path.isdir(os.path.join(path, program)):
            return os.path.join(path, program)
    return None

def run(program):
    '''
    Run program

    program --  string of shell command, including options
    returns the processes' return code
    '''
    process = subprocess.Popen(
        '%s' % program,
        shell=True
    )
    rc = process.wait() # wait to terminate
    return rc

def encrypt_file(source_file, destination_dir, key):
    '''
    GPG-encrypts source_file with key, saving encrypted file to destination_dir

    f   --  absolute path to the file to encrypt
    destination_dir --  absolute path to directory to save encrypted file in
    key --  keyid of public key to use; must be saved in gpg keyring

    Returns path to the encrypted file
    '''
    # init gpg
    gpg = gnupg.GPG() # use default - current user's $HOME/.gpg
    public_keys = gpg.list_keys()
    assert key in [k['keyid'] for k in public_keys], \
        "Could not find the specified PUBLIC_KEY_ID in keyring"

    # build encrypted filename and path
    e_filename = source_file.split("/")[-1] + ".gpg"
    ef_path = os.path.join(destination_dir, e_filename)

    try:
        fp = open(source_file, 'rb')
        encrypted_data = gpg.encrypt_file(
            fp,             # file object to encrypt
            key,            # public key of recipient
            output=ef_path  # path to encrypted file
        )
        fp.close()
    except IOError as e:
        error(e)

    debug("Encrypted %s -> %s" % (source_file, ef_path))

    return ef_path

def upload_to_s3(local_file, bucket_name, key_name=None, acl='private'):
    '''
    Uploads local_file to bucket_name on Amazon S3

    key_name is the "filename" on Amazon S3. We'll default to the local file's name if
    no alternative key_name is specified.
    '''
    # connect to Amazon S3
    conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
    bucket = conn.create_bucket(bucket_name)
    k = Key(bucket)

    if key_name:
        k.key = key_name
    else:
        k.key = local_file.split("/")[-1]

    k.set_contents_from_filename(local_file)
    k.set_acl(acl)

    debug("Uploaded %s to S3" % local_file)

def shred(f):
    '''
    Uses the *nix command shred to seriously erase f
    '''
    # check to see if we have shred installed; should this be an
    # assertionError, or a warning (and we default to rm)?
    assert whereis("shred") is not None, "Please install shred."
    process = subprocess.Popen(['shred', '-fuz', f], shell=False)
    if process.wait() == 0: # wait for shred to complete, check return code
        debug("Shredded %s" % f)
    else: # some kind of error occurred; handle
        error("shredding the file failed: shred returned %s" % (process.returncode))

def upload_handler(filename):
    """
    Handles file upload from upload.py
    """

    assert os.path.isfile(filename), \
        "filename does not reference a file: %s" % job.body

    # encrypt file
    efile_path = encrypt_file(filename, TEMP_DIR, PUBLIC_KEY_ID)
    # upload to S3
    upload_to_s3(efile_path, BUCKET_NAME)
    # shred original and encrypted files
    shred(filename)
    shred(efile_path)

def main():
    """
    Given filename as a command line argument, processes it

        1. Encrypts the file using the PUBLIC_KEY_ID public key
        2. Copies it to Amazon S3
        3. shreds the original file and the encrypted file
    """
    if len(sys.argv) != 2: # program name and filename
        sys.exit("Must provide a path to the file to process")

    filename = sys.argv[1]

    assert os.path.isfile(filename), "filename does not reference a file: %s" % filename

    info("UPLOAD HANDLER START")
    upload_handler(filename)
    info("UPLOAD HANDLER COMPLETE")

if __name__ == "__main__": main()
<commit_msg>Use Amazon SSE (AES-256).<commit_after>#!/usr/bin/env python
"""
Script run by upload.py when a file is received.
"""

import sys, os
import subprocess
import gnupg

from mylogging import debug, info, warning, error, critical

from boto.s3.connection import S3Connection
from boto.s3.key import Key

### SETTINGS ###
PUBLIC_KEY_ID = ''  # public key to use to encrypt files
                    # stored in gpg keyring
AWS_ACCESS_KEY = ''
AWS_SECRET_KEY = ''

BUCKET_NAME = 'haps-dev'
TEMP_DIR = "/tmp"

try:
    from settings import *
except ImportError:
    pass

def whereis(program):
    '''
    Returns the path to program on a user's path, if it exists
    Returns None if it does not
    '''
    for path in os.environ.get('PATH', '').split(':'):
        if os.path.exists(os.path.join(path, program)) and \
           not os.path.isdir(os.path.join(path, program)):
            return os.path.join(path, program)
    return None

def run(program):
    '''
    Run program

    program --  string of shell command, including options
    returns the processes' return code
    '''
    process = subprocess.Popen(
        '%s' % program,
        shell=True
    )
    rc = process.wait() # wait to terminate
    return rc

def encrypt_file(source_file, destination_dir, key):
    '''
    GPG-encrypts source_file with key, saving encrypted file to destination_dir

    f   --  absolute path to the file to encrypt
    destination_dir --  absolute path to directory to save encrypted file in
    key --  keyid of public key to use; must be saved in gpg keyring

    Returns path to the encrypted file
    '''
    # init gpg
    gpg = gnupg.GPG() # use default - current user's $HOME/.gpg
    public_keys = gpg.list_keys()
    assert key in [k['keyid'] for k in public_keys], \
        "Could not find the specified PUBLIC_KEY_ID in keyring"

    # build encrypted filename and path
    e_filename = source_file.split("/")[-1] + ".gpg"
    ef_path = os.path.join(destination_dir, e_filename)

    try:
        fp = open(source_file, 'rb')
        encrypted_data = gpg.encrypt_file(
            fp,             # file object to encrypt
            key,            # public key of recipient
            output=ef_path  # path to encrypted file
        )
        fp.close()
    except IOError as e:
        error(e)

    debug("Encrypted %s -> %s" % (source_file, ef_path))

    return ef_path

def upload_to_s3(local_file, bucket_name, key_name=None, acl='private'):
    '''
    Uploads local_file to bucket_name on Amazon S3

    key_name is the "filename" on Amazon S3. We'll default to the local file's name if
    no alternative key_name is specified.
    '''
    # connect to Amazon S3
    conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
    bucket = conn.create_bucket(bucket_name)
    k = Key(bucket)

    if key_name:
        k.key = key_name
    else:
        k.key = local_file.split("/")[-1]

    # encrypt_key=True for AES-256 encryption while at rest on Amazon's servers
    k.set_contents_from_filename(local_file, encrypt_key=True)
    k.set_acl(acl)

    debug("Uploaded %s to S3" % local_file)

def shred(f):
    '''
    Uses the *nix command shred to seriously erase f
    '''
    # check to see if we have shred installed; should this be an
    # assertionError, or a warning (and we default to rm)?
    assert whereis("shred") is not None, "Please install shred."
    process = subprocess.Popen(['shred', '-fuz', f], shell=False)
    if process.wait() == 0: # wait for shred to complete, check return code
        debug("Shredded %s" % f)
    else: # some kind of error occurred; handle
        error("shredding the file failed: shred returned %s" % (process.returncode))

def upload_handler(filename):
    """
    Handles file upload from upload.py
    """

    assert os.path.isfile(filename), \
        "filename does not reference a file: %s" % job.body

    # encrypt file
    efile_path = encrypt_file(filename, TEMP_DIR, PUBLIC_KEY_ID)
    # upload to S3
    upload_to_s3(efile_path, BUCKET_NAME)
    # shred original and encrypted files
    shred(filename)
    shred(efile_path)

def main():
    """
    Given filename as a command line argument, processes it

        1. Encrypts the file using the PUBLIC_KEY_ID public key
        2. Copies it to Amazon S3
        3. shreds the original file and the encrypted file
    """
    if len(sys.argv) != 2: # program name and filename
        sys.exit("Must provide a path to the file to process")

    filename = sys.argv[1]

    assert os.path.isfile(filename), "filename does not reference a file: %s" % filename

    upload_handler(filename)
    info("UPLOAD HANDLER COMPLETE")

if __name__ == "__main__": main()
<|endoftext|>