<commit_before># -*- encoding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################
from openerp import models, fields, api, exceptions, _
from datetime import datetime

account_concept_mapping = {
    '01': '4300%00',
    '02': '4100%00',
    '03': '4100%00',
    '04': '4300%00',
    '05': '6800%00',
    '06': '4010%00',
    '07': '5700%00',
    '08': '6800%00',
    '09': '2510%00',
    '10': '5700%00',
    '11': '5700%00',
    '12': '5700%00',
    '13': '5730%00',
    '14': '4300%00',
    '15': '6400%00',
    '16': '6690%00',
    '17': '6690%00',
    '98': '5720%00',
    '99': '5720%00',
}

class AccountBankStatementImport(models.TransientModel):
    _inherit = 'account.bank.statement.import'

    def _process_record_11(self, line):
        """11 - Registro cabecera de cuenta (obligatorio)"""
        st_group = {
            'entidad': line[2:6],
            'oficina': line[6:10],
            'cuenta': line[10:20],
            'fecha_ini': datetime.strptime(line[20:26], '%y%m%d'),
            'fecha_fin': datetime.strptime(line[26:32], '%y%m%d'),
            'divisa': line[47:50],
            'modalidad': line[50:51],  # 1, 2 o 3
            'nombre_propietario': line[51:77],
            'saldo_ini': float("%s.%s" % (line[33:45], line[45:47])),
            'saldo_fin': 0,
            'num_debe': 0,
            'debe': 0,
            'num_haber': 0,
            'haber': 0,
            'lines': [],
        }
        if line[32:33] == '1':
            st_group['saldo_ini'] *= -1
        self.balance_start = st_group['saldo_ini']
        return st_group

    def _process_record_22(self, line):
        """22 - Registro principal de movimiento (obligatorio)"""
        st_line = {
            'of_origen': line[6:10],
            'fecha_oper': datetime.strptime(line[10:16], '%y%m%d'),
            'fecha_valor': datetime.strptime(line[16:22], '%y%m%d'),
            'concepto_c': line[22:24],
            'concepto_p': line[24:27],
            'importe': (float(line[28:40]) + (float(line[40:42]) / 100)),
            'num_documento': line[41:52],
            'referencia1': line[52:64].strip(),
            'referencia2': line[64:].strip(),
            'conceptos': '',
        }
        if line[27:28] == '1':
            st_line['importe'] *= -1
        return st_line

    def _process_record_23(self, st_line, line):
        """23 - Registros complementarios de concepto (opcionales y hasta un
        máximo de 5)"""
        if not st_line.get('conceptos'):
            st_line['conceptos'] = {}
        st_line['conceptos'][line[2:4]] = (line[4:39].strip(),
                                           line[39:].strip())
        return st_line

    def _process_record_24(self, st_line, line):
        """24 - Registro complementario de información de equivalencia del
        importe (opcional y sin valor contable)"""
        st_line['divisa_eq'] = line[4:7]
        st_line['importe_eq'] = float(line[7:19]) + (float(line[19:21]) / 100)
        return st_line

    def _process_record_33(self, st_group, line):
        """33 - Registro final de cuenta"""
        st_group['num_debe'] += int(line[20:25])
        st_group['debe'] += float("%s.%s" % (line[25:37], line[37:39]))
        st_group['num_haber'] += int(line[39:44])
        st_group['haber'] += float("%s.%s" % (line[44:56], line[56:58]))
        st_group['saldo_fin'] += float("%s.%s" % (line[59:71], line[71:73]))
        if line[58:59] == '1':
            st_group['saldo_fin'] *= -1
        self.balance_end = st_group['saldo_fin']
        # Group level checks
        debit_count = 0
        debit = 0.0
        credit_count = 0
        credit = 0.0
        for st_line in st_group['lines']:
            if st_line['importe'] < 0:
                debit_count += 1
                debit -= st_line['importe']
            else:
                credit_count += 1
                credit += st_line['importe']
        if st_group['num_debe'] != debit_count:
            raise exceptions.Warning(
                _("Number of debit records doesn't match with the defined in "
                  "the last record of account."))
        if st_group['num_haber'] != credit_count:
            raise exceptions.Warning(
                _('Error in C43 file'),
                _("Number of credit records doesn't match with the defined "
                  "in the last record of account."))
        if abs(st_group['debe'] - debit) > 0.005:
            raise exceptions.Warning(
                _('Error in C43 file'),
                _("Debit amount doesn't match with the defined in the last "
                  "record of account."))
        if abs(st_group['haber'] - credit) > 0.005:
            raise exceptions.Warning(
                _("Credit amount doesn't match with the defined in the last "
                  "record of account."))
        # Note: Only perform this check if the balance is defined on the file
        # record, as some banks may leave it empty (zero) on some circumstances
        # (like CaixaNova extracts for VISA credit cards).
        if st_group['saldo_fin'] and st_group['saldo_ini']:
            balance = st_group['saldo_ini'] + credit - debit
            if abs(st_group['saldo_fin'] - balance) > 0.005:
                raise exceptions.Warning(
                    _("Final balance amount = (initial balance + credit "
                      "- debit) doesn't match with the defined in the last "
                      "record of account."))
        return st_group

    def _process_record_88(self, st_data, line):
        """88 - Registro de fin de archivo"""
        st_data['num_registros'] = int(line[20:26])
        # File level checks
        # Some banks (like Liderbank) are informing this record number
        # including the record 88, so checking this with the absolute
        # difference allows to bypass the error
        if abs(st_data['num_registros'] - st_data['_num_records']) > 1:
            raise exceptions.Warning(
                _("Number of records doesn't match with the defined in the "
                  "last record."))
        return st_data

    def _parse(self, data_file):
        # st_data will contain data read from the file
        st_data = {
            '_num_records': 0,  # Number of records really counted
            'groups': [],  # Info about each of the groups (account groups)
        }
        for raw_line in data_file.split("\n"):
            if not raw_line:
                continue
            code = raw_line[0:2]
            if code == '11':
                st_group = self._process_record_11(raw_line)
                st_data['groups'].append(st_group)
            elif code == '22':
                st_line = self._process_record_22(raw_line)
                st_group['lines'].append(st_line)
            elif code == '23':
                self._process_record_23(st_line, raw_line)
            elif code == '24':
                self._process_record_24(st_line, raw_line)
            elif code == '33':
                self._process_record_33(st_group, raw_line)
            elif code == '88':
                self._process_record_88(st_data, raw_line)
            elif ord(raw_line[0]) == 26:
                # CTRL-Z (^Z), is often used as an end-of-file marker in DOS
                continue
            else:
                raise exceptions.ValidationError(
                    _('Record type %s is not valid.') % raw_line[0:2])
            # Update the record counter
            st_data['_num_records'] += 1
        return st_data['groups']

    def _check_n43(self, data_file):
        data_file = data_file.decode('iso-8859-1')
        try:
            n43 = self._parse(data_file)
        except exceptions.ValidationError:
            return False
        return n43

    def _get_ref(self, line):
        try:
            ref1 = int(line['referencia1'])
        except:
            ref1 = line['referencia1']
        try:
            ref2 = int(line['referencia2'])
        except:
            ref2 = line['referencia2']
        if not ref1:
            return line['referencia2'] or '/'
        elif not ref2:
            return line['referencia1'] or '/'
        else:
            return "%s / %s" % (line['referencia1'], line['referencia2'])

    def _get_partner_from_caixabank(self, conceptos):
        partner_obj = self.env['res.partner']
        partners = []
        # Try to match from VAT included in concept complementary record #02
        if conceptos.get('02'):
            vat = conceptos['02'][0][:2] + conceptos['02'][0][7:]
            if vat:
                partners = partner_obj.search([('vat', '=', vat)])
        if not partners:
            # Try to match from partner name
            if conceptos.get('01'):
                name = conceptos['01'][0][4:] + conceptos['01'][1]
                if name:
                    partners = partner_obj.search([('name', 'ilike', name)])
        return partners and partners[0].id or False

    def _get_partner_from_santander(self, conceptos):
        partner_obj = self.env['res.partner']
        partners = []
        # Try to match from VAT included in concept complementary record #01
        if conceptos.get('01'):
            if conceptos['01'][1]:
                vat = conceptos['01'][1]
                if vat:
                    partners = partner_obj.search([('vat', 'ilike', vat)])
        if not partners:
            # Try to match from partner name
            if conceptos.get('01'):
                name = conceptos['01'][0]
                if name:
                    partners = partner_obj.search([('name', 'ilike', name)])
        return partners and partners[0].id or False

    def _get_partner(self, line):
        if not line.get('conceptos'):
            return False
        partner_id = self._get_partner_from_caixabank(line['conceptos'])
        if partner_id:
            return partner_id
        partner_id = self._get_partner_from_santander(line['conceptos'])
        return partner_id

    def _get_account(self, line):
        accounts = []
        if line['concepto_c']:
            accounts = self.env['account.account'].search(
                [('code', 'like',
                  account_concept_mapping[line['concepto_c']])])
        return accounts and accounts[0].id or False

    @api.model
    def _parse_file(self, data_file):
        n43 = self._check_n43(data_file)
        if not n43:
            return super(AccountBankStatementImport, self)._parse_file(
                data_file)
        transactions = []
        for group in n43:
            for line in group['lines']:
                conceptos = []
                for concept_line in line['conceptos']:
                    conceptos.extend(x.strip()
                                     for x in line['conceptos'][concept_line]
                                     if x.strip())
                vals_line = {
                    'date': fields.Date.to_string(line['fecha_valor']),
                    'name': ' '.join(conceptos),
                    'ref': self._get_ref(line),
                    'amount': line['importe'],
                    'partner_id': self._get_partner(line),
                }
                if not vals_line['name']:
                    vals_line['name'] = vals_line['ref']
                transactions.append(vals_line)
        vals_bank_statement = {
            'transactions': transactions,
            'balance_start': n43 and n43[0]['saldo_ini'] or 0.0,
            'balance_end_real': n43 and n43[-1]['saldo_fin'] or 0.0,
        }
        return 'EUR', False, [vals_bank_statement]

    @api.model
    def _get_hide_journal_field(self):
        # Show the journal_id field if not coming from a context where is set
        return bool(self.env.context.get('journal_id'))
<commit_msg>[IMP] l10n_es_account_bank_statement_import_n43. Importar extractos con monedas distintas de euros segun issue #217 y basado en comentario de @fgarciag2001<commit_after># -*- encoding: utf-8 -*-
##############################################################################
# For copyright and license notices, see __openerp__.py file in root directory
##############################################################################
from openerp import models, fields, api, exceptions, _
from datetime import datetime

account_concept_mapping = {
    '01': '4300%00',
    '02': '4100%00',
    '03': '4100%00',
    '04': '4300%00',
    '05': '6800%00',
    '06': '4010%00',
    '07': '5700%00',
    '08': '6800%00',
    '09': '2510%00',
    '10': '5700%00',
    '11': '5700%00',
    '12': '5700%00',
    '13': '5730%00',
    '14': '4300%00',
    '15': '6400%00',
    '16': '6690%00',
    '17': '6690%00',
    '98': '5720%00',
    '99': '5720%00',
}

class AccountBankStatementImport(models.TransientModel):
    _inherit = 'account.bank.statement.import'

    def _process_record_11(self, line):
        """11 - Registro cabecera de cuenta (obligatorio)"""
        st_group = {
            'entidad': line[2:6],
            'oficina': line[6:10],
            'cuenta': line[10:20],
            'fecha_ini': datetime.strptime(line[20:26], '%y%m%d'),
            'fecha_fin': datetime.strptime(line[26:32], '%y%m%d'),
            'divisa': line[47:50],
            'modalidad': line[50:51],  # 1, 2 o 3
            'nombre_propietario': line[51:77],
            'saldo_ini': float("%s.%s" % (line[33:45], line[45:47])),
            'saldo_fin': 0,
            'num_debe': 0,
            'debe': 0,
            'num_haber': 0,
            'haber': 0,
            'lines': [],
        }
        if line[32:33] == '1':
            st_group['saldo_ini'] *= -1
        self.balance_start = st_group['saldo_ini']
        return st_group

    def _process_record_22(self, line):
        """22 - Registro principal de movimiento (obligatorio)"""
        st_line = {
            'of_origen': line[6:10],
            'fecha_oper': datetime.strptime(line[10:16], '%y%m%d'),
            'fecha_valor': datetime.strptime(line[16:22], '%y%m%d'),
            'concepto_c': line[22:24],
            'concepto_p': line[24:27],
            'importe': (float(line[28:40]) + (float(line[40:42]) / 100)),
            'num_documento': line[41:52],
            'referencia1': line[52:64].strip(),
            'referencia2': line[64:].strip(),
            'conceptos': '',
        }
        if line[27:28] == '1':
            st_line['importe'] *= -1
        return st_line

    def _process_record_23(self, st_line, line):
        """23 - Registros complementarios de concepto (opcionales y hasta un
        máximo de 5)"""
        if not st_line.get('conceptos'):
            st_line['conceptos'] = {}
        st_line['conceptos'][line[2:4]] = (line[4:39].strip(),
                                           line[39:].strip())
        return st_line

    def _process_record_24(self, st_line, line):
        """24 - Registro complementario de información de equivalencia del
        importe (opcional y sin valor contable)"""
        st_line['divisa_eq'] = line[4:7]
        st_line['importe_eq'] = float(line[7:19]) + (float(line[19:21]) / 100)
        return st_line

    def _process_record_33(self, st_group, line):
        """33 - Registro final de cuenta"""
        st_group['num_debe'] += int(line[20:25])
        st_group['debe'] += float("%s.%s" % (line[25:37], line[37:39]))
        st_group['num_haber'] += int(line[39:44])
        st_group['haber'] += float("%s.%s" % (line[44:56], line[56:58]))
        st_group['saldo_fin'] += float("%s.%s" % (line[59:71], line[71:73]))
        if line[58:59] == '1':
            st_group['saldo_fin'] *= -1
        self.balance_end = st_group['saldo_fin']
        # Group level checks
        debit_count = 0
        debit = 0.0
        credit_count = 0
        credit = 0.0
        for st_line in st_group['lines']:
            if st_line['importe'] < 0:
                debit_count += 1
                debit -= st_line['importe']
            else:
                credit_count += 1
                credit += st_line['importe']
        if st_group['num_debe'] != debit_count:
            raise exceptions.Warning(
                _("Number of debit records doesn't match with the defined in "
                  "the last record of account."))
        if st_group['num_haber'] != credit_count:
            raise exceptions.Warning(
                _('Error in C43 file'),
                _("Number of credit records doesn't match with the defined "
                  "in the last record of account."))
        if abs(st_group['debe'] - debit) > 0.005:
            raise exceptions.Warning(
                _('Error in C43 file'),
                _("Debit amount doesn't match with the defined in the last "
                  "record of account."))
        if abs(st_group['haber'] - credit) > 0.005:
            raise exceptions.Warning(
                _("Credit amount doesn't match with the defined in the last "
                  "record of account."))
        # Note: Only perform this check if the balance is defined on the file
        # record, as some banks may leave it empty (zero) on some circumstances
        # (like CaixaNova extracts for VISA credit cards).
        if st_group['saldo_fin'] and st_group['saldo_ini']:
            balance = st_group['saldo_ini'] + credit - debit
            if abs(st_group['saldo_fin'] - balance) > 0.005:
                raise exceptions.Warning(
                    _("Final balance amount = (initial balance + credit "
                      "- debit) doesn't match with the defined in the last "
                      "record of account."))
        return st_group

    def _process_record_88(self, st_data, line):
        """88 - Registro de fin de archivo"""
        st_data['num_registros'] = int(line[20:26])
        # File level checks
        # Some banks (like Liderbank) are informing this record number
        # including the record 88, so checking this with the absolute
        # difference allows to bypass the error
        if abs(st_data['num_registros'] - st_data['_num_records']) > 1:
            raise exceptions.Warning(
                _("Number of records doesn't match with the defined in the "
                  "last record."))
        return st_data

    def _parse(self, data_file):
        # st_data will contain data read from the file
        st_data = {
            '_num_records': 0,  # Number of records really counted
            'groups': [],  # Info about each of the groups (account groups)
        }
        for raw_line in data_file.split("\n"):
            if not raw_line:
                continue
            code = raw_line[0:2]
            if code == '11':
                st_group = self._process_record_11(raw_line)
                st_data['groups'].append(st_group)
            elif code == '22':
                st_line = self._process_record_22(raw_line)
                st_group['lines'].append(st_line)
            elif code == '23':
                self._process_record_23(st_line, raw_line)
            elif code == '24':
                self._process_record_24(st_line, raw_line)
            elif code == '33':
                self._process_record_33(st_group, raw_line)
            elif code == '88':
                self._process_record_88(st_data, raw_line)
            elif ord(raw_line[0]) == 26:
                # CTRL-Z (^Z), is often used as an end-of-file marker in DOS
                continue
            else:
                raise exceptions.ValidationError(
                    _('Record type %s is not valid.') % raw_line[0:2])
            # Update the record counter
            st_data['_num_records'] += 1
        return st_data['groups']

    def _check_n43(self, data_file):
        data_file = data_file.decode('iso-8859-1')
        try:
            n43 = self._parse(data_file)
        except exceptions.ValidationError:
            return False
        return n43

    def _get_ref(self, line):
        try:
            ref1 = int(line['referencia1'])
        except:
            ref1 = line['referencia1']
        try:
            ref2 = int(line['referencia2'])
        except:
            ref2 = line['referencia2']
        if not ref1:
            return line['referencia2'] or '/'
        elif not ref2:
            return line['referencia1'] or '/'
        else:
            return "%s / %s" % (line['referencia1'], line['referencia2'])

    def _get_partner_from_caixabank(self, conceptos):
        partner_obj = self.env['res.partner']
        partners = []
        # Try to match from VAT included in concept complementary record #02
        if conceptos.get('02'):
            vat = conceptos['02'][0][:2] + conceptos['02'][0][7:]
            if vat:
                partners = partner_obj.search([('vat', '=', vat)])
        if not partners:
            # Try to match from partner name
            if conceptos.get('01'):
                name = conceptos['01'][0][4:] + conceptos['01'][1]
                if name:
                    partners = partner_obj.search([('name', 'ilike', name)])
        return partners and partners[0].id or False

    def _get_partner_from_santander(self, conceptos):
        partner_obj = self.env['res.partner']
        partners = []
        # Try to match from VAT included in concept complementary record #01
        if conceptos.get('01'):
            if conceptos['01'][1]:
                vat = conceptos['01'][1]
                if vat:
                    partners = partner_obj.search([('vat', 'ilike', vat)])
        if not partners:
            # Try to match from partner name
            if conceptos.get('01'):
                name = conceptos['01'][0]
                if name:
                    partners = partner_obj.search([('name', 'ilike', name)])
        return partners and partners[0].id or False

    def _get_partner(self, line):
        if not line.get('conceptos'):
            return False
        partner_id = self._get_partner_from_caixabank(line['conceptos'])
        if partner_id:
            return partner_id
        partner_id = self._get_partner_from_santander(line['conceptos'])
        return partner_id

    def _get_account(self, line):
        accounts = []
        if line['concepto_c']:
            accounts = self.env['account.account'].search(
                [('code', 'like',
                  account_concept_mapping[line['concepto_c']])])
        return accounts and accounts[0].id or False

    @api.model
    def _parse_file(self, data_file):
        n43 = self._check_n43(data_file)
        if not n43:
            return super(AccountBankStatementImport, self)._parse_file(
                data_file)
        transactions = []
        for group in n43:
            for line in group['lines']:
                conceptos = []
                for concept_line in line['conceptos']:
                    conceptos.extend(x.strip()
                                     for x in line['conceptos'][concept_line]
                                     if x.strip())
                vals_line = {
                    'date': fields.Date.to_string(line['fecha_valor']),
                    'name': ' '.join(conceptos),
                    'ref': self._get_ref(line),
                    'amount': line['importe'],
                    'partner_id': self._get_partner(line),
                }
                if not vals_line['name']:
                    vals_line['name'] = vals_line['ref']
                transactions.append(vals_line)
        vals_bank_statement = {
            'transactions': transactions,
            'balance_start': n43 and n43[0]['saldo_ini'] or 0.0,
            'balance_end_real': n43 and n43[-1]['saldo_fin'] or 0.0,
        }
        str_currency = self.journal_id.currency and  \
            self.journal_id.currency.name or \
            self.journal_id.company_id.currency_id.name
        return str_currency, False, [vals_bank_statement]

    @api.model
    def _get_hide_journal_field(self):
        # Show the journal_id field if not coming from a context where is set
        return bool(self.env.context.get('journal_id'))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Benchmark scipy.special.erfinv."""

import timeit

name = "erfinv"
repeats = 3
iterations = 1000000

def print_version():
    """Print the TAP version."""

    print("TAP version 13")

def print_summary(total, passing):
    """Print the benchmark summary.

    # Arguments

    * `total`: total number of tests
    * `passing`: number of passing tests

    """

    print("#")
    print("1.." + str(total))  # TAP plan
    print("# total " + str(total))
    print("# pass  " + str(passing))
    print("#")
    print("# ok")

def print_results(elapsed):
    """Print benchmark results.

    # Arguments

    * `elapsed`: elapsed time (in seconds)

    # Examples

    ``` python
    python> print_results(0.131009101868)
    ```
    """

    rate = iterations / elapsed

    print("  ---")
    print("  iterations: " + str(iterations))
    print("  elapsed: " + str(elapsed))
    print("  rate: " + str(rate))
    print("  ...")

def benchmark():
    """Run the benchmark and print benchmark results."""

    setup = "from scipy.special import erfinv; from random import random;"
    stmt = "y = erfinv(2.0*random() - 1.0)"

    t = timeit.Timer(stmt, setup=setup)

    print_version()

    for i in xrange(repeats):
        print("# python::" + name)
        elapsed = t.timeit(number=iterations)
        print_results(elapsed)
        print("ok " + str(i+1) + " benchmark finished")

    print_summary(repeats, repeats)

def main():
    """Run the benchmark."""
    benchmark()

if __name__ == "__main__":
    main()
<commit_msg>Fix lint errors<commit_after>#!/usr/bin/env python
"""Benchmark scipy.special.erfinv."""

from __future__ import print_function
import timeit

NAME = "erfinv"
REPEATS = 3
ITERATIONS = 1000000

def print_version():
    """Print the TAP version."""
    print("TAP version 13")

def print_summary(total, passing):
    """Print the benchmark summary.

    # Arguments

    * `total`: total number of tests
    * `passing`: number of passing tests

    """
    print("#")
    print("1.." + str(total))  # TAP plan
    print("# total " + str(total))
    print("# pass  " + str(passing))
    print("#")
    print("# ok")

def print_results(elapsed):
    """Print benchmark results.

    # Arguments

    * `elapsed`: elapsed time (in seconds)

    # Examples

    ``` python
    python> print_results(0.131009101868)
    ```
    """
    rate = ITERATIONS / elapsed

    print("  ---")
    print("  iterations: " + str(ITERATIONS))
    print("  elapsed: " + str(elapsed))
    print("  rate: " + str(rate))
    print("  ...")

def benchmark():
    """Run the benchmark and print benchmark results."""
    setup = "from scipy.special import erfinv; from random import random;"
    stmt = "y = erfinv(2.0*random() - 1.0)"

    t = timeit.Timer(stmt, setup=setup)

    print_version()

    for i in xrange(REPEATS):
        print("# python::" + NAME)
        elapsed = t.timeit(number=ITERATIONS)
        print_results(elapsed)
        print("ok " + str(i+1) + " benchmark finished")

    print_summary(REPEATS, REPEATS)

def main():
    """Run the benchmark."""
    benchmark()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import base64
import hashlib
import os
import configparser
import shutil

from boto.s3.connection import S3Connection
from google.cloud import storage as gc

from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.system import System
from dvc.config import ConfigError
from dvc.settings import Settings

class DataCloudError(DvcException):
    def __init__(self, msg):
        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))

def sizeof_fmt(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

def percent_cb(complete, total):
    Logger.debug('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total)))

def file_md5(fname):
    """ get the (md5 hexdigest, md5 digest) of a file """
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(1024*1000), b""):
            hash_md5.update(chunk)
    return (hash_md5.hexdigest(), hash_md5.digest())

class DataCloudBase(object):
    """ Base class for DataCloud """
    def __init__(self, settings, config, cloud_config):
        self._settings = settings
        self._config = config
        self._cloud_config = cloud_config

    @property
    def storage_path(self):
        """ get storage path

        Precedence: Storage, then cloud specific
        """

        path = self._config['Global'].get('StoragePath', None)
        if path:
            return path

        path = self._cloud_config.get('StoragePath', None)
        if path is None:
            raise ConfigError('invalid StoragePath: not set for Data or cloud specific')

        return path

    def _storage_path_parts(self):
        return self.storage_path.strip('/').split('/', 1)

    @property
    def storage_bucket(self):
        """ Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """
        return self._storage_path_parts()[0]

    @property
    def storage_prefix(self):
        parts = self._storage_path_parts()
        if len(parts) > 1:
            return parts[1]
        return ''

    def cache_file_key(self, file):
        return '{}/{}'.format(self.storage_prefix, file).strip('/')

    def sanity_check(self):
        pass

    def sync_to_cloud(self, item):
        pass

    def sync_from_cloud(self, item):
        pass

    def sync(self, fname):
        item = self._settings.path_factory.data_item(fname)
        if os.path.isfile(item.resolved_cache.dvc):
            self.sync_to_cloud(item)
        else:
            self.sync_from_cloud(item)

    def remove_from_cloud(self, item):
        pass

class DataCloudLOCAL(DataCloudBase):
    def sync_to_cloud(self, item):
        Logger.debug('sync to cloud ' + item.resolved_cache.dvc + " " + self.storage_path)
        shutil.copy(item.resolved_cache.dvc, self.storage_path)

    def sync_from_cloud(self, item):
        Logger.debug('sync from cloud ' + self.storage_path + " " + item.resolved_cache.dvc)
        shutil.copy(self.storage_path, item.resolved_cache.dvc)

    def remove_from_cloud(self, item):
        Logger.debug('rm from cloud ' + item.resolved_cache.dvc)
        os.remove(item.resolved_cache.dvc)

class DataCloudAWS(DataCloudBase):
    """ DataCloud class for Amazon Web Services """
    @property
    def aws_access_key_id(self):
        if not self._aws_creds:
            self._aws_creds = self.get_aws_credentials()
        if not self._aws_creds:
            return None
        return self._aws_creds[0]

    @property
    def aws_secret_access_key(self):
        if not self._aws_creds:
            self._aws_creds = self.get_aws_credentials()
        if not self._aws_creds:
            return None
        return self._aws_creds[1]

    @property
    def aws_region_host(self):
        """ get the region host needed for s3 access

        See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
        """

        region = self._cloud_config['Region']
        if region is None or region == '':
            return 's3.amazonaws.com'
        if region == 'us-east-1':
            return 's3.amazonaws.com'
        return 's3.%s.amazonaws.com' % region

    def get_aws_credentials(self):
        """ gets aws credentials, looking in various places

        Params:

        Searches:
        1 any override in dvc.conf [AWS] CredentialPath;
        2 ~/.aws/credentials

        Returns:
            if successfully found, (access_key_id, secret)
            None otherwise
        """
        default = os.path.expanduser('~/.aws/credentials')

        paths = []
        credpath = self._cloud_config.get('CredentialPath', None)
        if credpath is not None and len(credpath) > 0:
            credpath = os.path.expanduser(credpath)
            if os.path.isfile(credpath):
                paths.append(credpath)
            else:
                Logger.warn('AWS CredentialPath "%s" not found; falling back to default "%s"' % (credpath, default))
                paths.append(default)
        else:
            paths.append(default)

        for path in paths:
            cc = configparser.SafeConfigParser()
            threw = False
            try:
                # use readfp(open( ... to aid mocking.
                cc.readfp(open(path, 'r'))
            except Exception as e:
                threw = True
            if not threw and 'default' in cc.keys():
                access_key = cc['default'].get('aws_access_key_id', None)
                secret = cc['default'].get('aws_secret_access_key', None)

                if access_key is not None and secret is not None:
                    return (access_key, secret)

        return None

    def sanity_check(self):
        creds = self.get_aws_credentials()
        if creds is None:
            raise ConfigError('can\'t find aws credentials.')
        self._aws_creds = creds

    def _get_bucket_aws(self):
        """ get a bucket object, aws """
        conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key, host=self.aws_region_host)
        bucket_name = self.storage_bucket
        bucket = conn.lookup(bucket_name)
        if bucket is None:
            raise DataCloudError('Storage path is not setup correctly')
        return bucket

    def sync_from_cloud(self, item):
        """ sync from cloud, aws version """

        bucket = self._get_bucket_aws()

        key_name = self.cache_file_key(item.resolved_cache.dvc)
        key = bucket.get_key(key_name)
        if not key:
            raise DataCloudError('File "{}" does not exist in the cloud'.format(key_name))

        Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name))
        key.get_contents_to_filename(item.resolved_cache.relative, cb=percent_cb)
        Logger.info('Downloading completed')

    def sync_to_cloud(self, data_item):
        """ sync_to_cloud, aws version """

        aws_key = self.cache_file_key(data_item.resolved_cache.dvc)
        bucket = self._get_bucket_aws()
        key = bucket.get_key(aws_key)
        if key:
            Logger.debug('File already uploaded to the cloud. Checksum validation...')

            md5_cloud = key.etag[1:-1]
            md5_local = file_md5(data_item.resolved_cache.dvc)[0]
            if md5_cloud == md5_local:
                Logger.debug('File checksum matches. No uploading is needed.')
                return

            Logger.debug('Checksum miss-match. Re-uploading is required.')

        Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.resolved_cache.relative, aws_key))
        key = bucket.new_key(aws_key)
        key.set_contents_from_filename(data_item.resolved_cache.relative, cb=percent_cb)
        Logger.info('Uploading completed')

    def remove_from_cloud(self, data_item):
        aws_file_name = self.cache_file_key(data_item.cache.dvc)

        Logger.debug(u'[Cmd-Remove] Remove from cloud {}.'.format(aws_file_name))

        if not self.aws_access_key_id or not self.aws_secret_access_key:
            Logger.debug('[Cmd-Remove] Unable to check cache file in the cloud')
            return
        conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key)
        bucket_name = self.storage_bucket
        bucket = conn.lookup(bucket_name)
        if bucket:
            key = bucket.get_key(aws_file_name)
            if not key:
                Logger.warn('[Cmd-Remove] S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name))
            else:
                key.delete()
                Logger.info('[Cmd-Remove] File "{}" was removed from S3'.format(aws_file_name))
        pass

class DataCloudGCP(DataCloudBase):
    """ DataCloud class for Google Cloud Platform """
    @property
    def gc_project_name(self):
        return self._cloud_config.get('ProjectName', None)

    def sanity_check(self):
        project = self.gc_project_name
        if project is None or len(project) < 1:
            raise ConfigError('can\'t read google cloud project name. Please set ProjectName in section GC.')

    def _get_bucket_gc(self):
        """ get a bucket object, gc """
        client = gc.Client(project=self.gc_project_name)
        bucket = client.bucket(self.storage_bucket)
        if not bucket.exists():
            raise DataCloudError('sync up: google cloud bucket {} doesn\'t exist'.format(self.storage_bucket))
        return bucket

    def sync_from_cloud(self, item):
        """ sync from cloud, gcp version """

        bucket = self._get_bucket_gc()
        key = self.cache_file_key(item.resolved_cache.dvc)

        blob = bucket.get_blob(key)
        if not blob:
            raise DataCloudError('File "{}" does not exist in the cloud'.format(key))

        Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))

        blob.download_to_filename(item.resolved_cache.dvc)
        Logger.info('Downloading completed')

    def sync_to_cloud(self, data_item):
        """ sync_to_cloud, gcp version """

        bucket = self._get_bucket_gc()
        blob_name = self.cache_file_key(data_item.resolved_cache.dvc)

        blob = bucket.get_blob(blob_name)
        if blob is not None and blob.exists():
            b64_encoded_md5 = base64.b64encode(file_md5(data_item.resolved_cache.dvc)[1])

            if blob.md5_hash == b64_encoded_md5:
                Logger.debug('checksum %s matches.  Skipping upload' % data_item.cache.relative)
                return
            Logger.debug('checksum %s mismatch.  re-uploading' % data_item.cache.relative)

        Logger.info('uploading cache file "{} to gc "{}"'.format(data_item.cache.relative, blob_name))

        blob = bucket.blob(blob_name)
        blob.upload_from_filename(data_item.resolved_cache.relative)
        Logger.info('uploading %s completed' % data_item.resolved_cache.relative)

    def remove_from_cloud(self, item):
        raise Exception('NOT IMPLEMENTED YET')

class DataCloud(object):
    """ Generic class to do initial config parsing and redirect to proper DataCloud methods """

    CLOUD_MAP = {
        'AWS': DataCloudAWS,
        'GCP': DataCloudGCP,
        'LOCAL': DataCloudLOCAL,
    }

    def __init__(self, settings):
        assert isinstance(settings, Settings)

        #To handle ConfigI case
        if not hasattr(settings.config, '_config'):
            self._cloud = DataCloudBase(None, None, None)
            return

        self._settings = settings
        self._config = self._settings.config._config

        self.typ = self._config['Global'].get('Cloud', '').strip().upper()
        if self.typ not in self.CLOUD_MAP.keys():
            raise ConfigError('Wrong cloud type %s specified' % self.typ)

        self._cloud = self.CLOUD_MAP[self.typ](self._settings, self._config, self._config[self.typ])

        self.sanity_check()

    def sanity_check(self):
        """ sanity check a config

        check that we have a cloud and storagePath
        if aws, check can read credentials
        if google, check ProjectName

        Returns:
            (T,) if good
            (F, issues) if bad
        """
        for key in ['Cloud']:
            if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
                raise ConfigError('Please set %s in section Global in config file %s' % (key, self.file))

        # now that a cloud is chosen, can check StoragePath
        sp = self._cloud.storage_path
        if sp is None or len(sp) == 0:
            raise ConfigError('Please set StoragePath = bucket/{optional path} in conf file "%s" '
                           'either in Global or a cloud specific section' % self.CONFIG)

        self._cloud.sanity_check()

    def sync(self, fname):
        return self._cloud.sync(fname)

    def remove_from_cloud(self, item):
        return self._cloud.remove_from_cloud(item)
<commit_msg>data_cloud: check if cloud section exists<commit_after>import base64
import hashlib
import os
import configparser
import shutil

from boto.s3.connection import S3Connection
from google.cloud import storage as gc

from dvc.command.base import CmdBase, DvcLock
from dvc.logger import Logger
from dvc.exceptions import DvcException
from dvc.runtime import Runtime
from dvc.system import System
from dvc.config import ConfigError
from dvc.settings import Settings

class DataCloudError(DvcException):
    def __init__(self, msg):
        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))

def sizeof_fmt(num, suffix='B'):
    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

def percent_cb(complete, total):
    Logger.debug('{} transferred out of {}'.format(sizeof_fmt(complete), sizeof_fmt(total)))

def file_md5(fname):
    """ get the (md5 hexdigest, md5 digest) of a file """
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(1024*1000), b""):
            hash_md5.update(chunk)
    return (hash_md5.hexdigest(), hash_md5.digest())

class DataCloudBase(object):
    """ Base class for DataCloud """
    def __init__(self, settings, config, cloud_config):
        self._settings = settings
        self._config = config
        self._cloud_config = cloud_config

    @property
    def storage_path(self):
        """ get storage path

        Precedence: Storage, then cloud specific
        """

        path = self._config['Global'].get('StoragePath', None)
        if path:
            return path

        path = self._cloud_config.get('StoragePath', None)
        if path is None:
            raise ConfigError('invalid StoragePath: not set for Data or cloud specific')

        return path

    def _storage_path_parts(self):
        return self.storage_path.strip('/').split('/', 1)

    @property
    def storage_bucket(self):
        """ Data -> StoragePath takes precedence; if doesn't exist, use cloud-specific """
        return self._storage_path_parts()[0]

    @property
    def storage_prefix(self):
        parts = self._storage_path_parts()
        if len(parts) > 1:
            return parts[1]
        return ''

    def cache_file_key(self, file):
        return '{}/{}'.format(self.storage_prefix, file).strip('/')

    def sanity_check(self):
        pass

    def sync_to_cloud(self, item):
        pass

    def sync_from_cloud(self, item):
        pass

    def sync(self, fname):
        item = self._settings.path_factory.data_item(fname)
        if os.path.isfile(item.resolved_cache.dvc):
            self.sync_to_cloud(item)
        else:
            self.sync_from_cloud(item)

    def remove_from_cloud(self, item):
        pass

class DataCloudLOCAL(DataCloudBase):
    def sync_to_cloud(self, item):
        Logger.debug('sync to cloud ' + item.resolved_cache.dvc + " " + self.storage_path)
        shutil.copy(item.resolved_cache.dvc, self.storage_path)

    def sync_from_cloud(self, item):
        Logger.debug('sync from cloud ' + self.storage_path + " " + item.resolved_cache.dvc)
        shutil.copy(self.storage_path, item.resolved_cache.dvc)

    def remove_from_cloud(self, item):
        Logger.debug('rm from cloud ' + item.resolved_cache.dvc)
        os.remove(item.resolved_cache.dvc)

class DataCloudAWS(DataCloudBase):
    """ DataCloud class for Amazon Web Services """
    @property
    def aws_access_key_id(self):
        if not self._aws_creds:
            self._aws_creds = self.get_aws_credentials()
        if not self._aws_creds:
            return None
        return self._aws_creds[0]

    @property
    def aws_secret_access_key(self):
        if not self._aws_creds:
            self._aws_creds = self.get_aws_credentials()
        if not self._aws_creds:
            return None
        return self._aws_creds[1]

    @property
    def aws_region_host(self):
        """ get the region host needed for s3 access

        See notes http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
        """

        region = self._cloud_config['Region']
        if region is None or region == '':
            return 's3.amazonaws.com'
        if region == 'us-east-1':
            return 's3.amazonaws.com'
        return 's3.%s.amazonaws.com' % region

    def get_aws_credentials(self):
        """ gets aws credentials, looking in various places

        Params:

        Searches:
        1 any override in dvc.conf [AWS] CredentialPath;
        2 ~/.aws/credentials

        Returns:
            if successfully found, (access_key_id, secret)
            None otherwise
        """
        default = os.path.expanduser('~/.aws/credentials')

        paths = []
        credpath = self._cloud_config.get('CredentialPath', None)
        if credpath is not None and len(credpath) > 0:
            credpath = os.path.expanduser(credpath)
            if os.path.isfile(credpath):
                paths.append(credpath)
            else:
                Logger.warn('AWS CredentialPath "%s" not found; falling back to default "%s"' % (credpath, default))
                paths.append(default)
        else:
            paths.append(default)

        for path in paths:
            cc = configparser.SafeConfigParser()
            threw = False
            try:
                # use readfp(open( ... to aid mocking.
                cc.readfp(open(path, 'r'))
            except Exception as e:
                threw = True
            if not threw and 'default' in cc.keys():
                access_key = cc['default'].get('aws_access_key_id', None)
                secret = cc['default'].get('aws_secret_access_key', None)

                if access_key is not None and secret is not None:
                    return (access_key, secret)

        return None

    def sanity_check(self):
        creds = self.get_aws_credentials()
        if creds is None:
            raise ConfigError('can\'t find aws credentials.')
        self._aws_creds = creds

    def _get_bucket_aws(self):
        """ get a bucket object, aws """
        conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key, host=self.aws_region_host)
        bucket_name = self.storage_bucket
        bucket = conn.lookup(bucket_name)
        if bucket is None:
            raise DataCloudError('Storage path is not setup correctly')
        return bucket

    def sync_from_cloud(self, item):
        """ sync from cloud, aws version """

        bucket = self._get_bucket_aws()

        key_name = self.cache_file_key(item.resolved_cache.dvc)
        key = bucket.get_key(key_name)
        if not key:
            raise DataCloudError('File "{}" does not exist in the cloud'.format(key_name))

        Logger.info('Downloading cache file from S3 "{}/{}"'.format(bucket.name, key_name))
        key.get_contents_to_filename(item.resolved_cache.relative, cb=percent_cb)
        Logger.info('Downloading completed')

    def sync_to_cloud(self, data_item):
        """ sync_to_cloud, aws version """

        aws_key = self.cache_file_key(data_item.resolved_cache.dvc)
        bucket = self._get_bucket_aws()
        key = bucket.get_key(aws_key)
        if key:
            Logger.debug('File already uploaded to the cloud. Checksum validation...')

            md5_cloud = key.etag[1:-1]
            md5_local = file_md5(data_item.resolved_cache.dvc)[0]
            if md5_cloud == md5_local:
                Logger.debug('File checksum matches. No uploading is needed.')
                return

            Logger.debug('Checksum miss-match. Re-uploading is required.')

        Logger.info('Uploading cache file "{}" to S3 "{}"'.format(data_item.resolved_cache.relative, aws_key))
        key = bucket.new_key(aws_key)
        key.set_contents_from_filename(data_item.resolved_cache.relative, cb=percent_cb)
        Logger.info('Uploading completed')

    def remove_from_cloud(self, data_item):
        aws_file_name = self.cache_file_key(data_item.cache.dvc)

        Logger.debug(u'[Cmd-Remove] Remove from cloud {}.'.format(aws_file_name))

        if not self.aws_access_key_id or not self.aws_secret_access_key:
            Logger.debug('[Cmd-Remove] Unable to check cache file in the cloud')
            return
        conn = S3Connection(self.aws_access_key_id, self.aws_secret_access_key)
        bucket_name = self.storage_bucket
        bucket = conn.lookup(bucket_name)
        if bucket:
            key = bucket.get_key(aws_file_name)
            if not key:
                Logger.warn('[Cmd-Remove] S3 remove warning: file "{}" does not exist in S3'.format(aws_file_name))
            else:
                key.delete()
                Logger.info('[Cmd-Remove] File "{}" was removed from S3'.format(aws_file_name))
        pass

class DataCloudGCP(DataCloudBase):
    """ DataCloud class for Google Cloud Platform """
    @property
    def gc_project_name(self):
        return self._cloud_config.get('ProjectName', None)

    def sanity_check(self):
        project = self.gc_project_name
        if project is None or len(project) < 1:
            raise ConfigError('can\'t read google cloud project name. Please set ProjectName in section GC.')

    def _get_bucket_gc(self):
        """ get a bucket object, gc """
        client = gc.Client(project=self.gc_project_name)
        bucket = client.bucket(self.storage_bucket)
        if not bucket.exists():
            raise DataCloudError('sync up: google cloud bucket {} doesn\'t exist'.format(self.storage_bucket))
        return bucket

    def sync_from_cloud(self, item):
        """ sync from cloud, gcp version """

        bucket = self._get_bucket_gc()
        key = self.cache_file_key(item.resolved_cache.dvc)

        blob = bucket.get_blob(key)
        if not blob:
            raise DataCloudError('File "{}" does not exist in the cloud'.format(key))

        Logger.info('Downloading cache file from gc "{}/{}"'.format(bucket.name, key))

        blob.download_to_filename(item.resolved_cache.dvc)
        Logger.info('Downloading completed')

    def sync_to_cloud(self, data_item):
        """ sync_to_cloud, gcp version """

        bucket = self._get_bucket_gc()
        blob_name = self.cache_file_key(data_item.resolved_cache.dvc)

        blob = bucket.get_blob(blob_name)
        if blob is not None and blob.exists():
            b64_encoded_md5 = base64.b64encode(file_md5(data_item.resolved_cache.dvc)[1])

            if blob.md5_hash == b64_encoded_md5:
                Logger.debug('checksum %s matches.  Skipping upload' % data_item.cache.relative)
                return
            Logger.debug('checksum %s mismatch.  re-uploading' % data_item.cache.relative)

        Logger.info('uploading cache file "{} to gc "{}"'.format(data_item.cache.relative, blob_name))

        blob = bucket.blob(blob_name)
        blob.upload_from_filename(data_item.resolved_cache.relative)
        Logger.info('uploading %s completed' % data_item.resolved_cache.relative)

    def remove_from_cloud(self, item):
        raise Exception('NOT IMPLEMENTED YET')

class DataCloud(object):
    """ Generic class to do initial config parsing and redirect to proper DataCloud methods """

    CLOUD_MAP = {
        'AWS': DataCloudAWS,
        'GCP': DataCloudGCP,
        'LOCAL': DataCloudLOCAL,
    }

    def __init__(self, settings):
        assert isinstance(settings, Settings)

        #To handle ConfigI case
        if not hasattr(settings.config, '_config'):
            self._cloud = DataCloudBase(None, None, None)
            return

        self._settings = settings
        self._config = self._settings.config._config

        self.typ = self._config['Global'].get('Cloud', '').strip().upper()
        if self.typ not in self.CLOUD_MAP.keys():
            raise ConfigError('Wrong cloud type %s specified' % self.typ)

        if self.typ not in self._config.keys():
            raise ConfigError('Can\'t find cloud section \'[%s]\' in config' % self.typ)

        self._cloud = self.CLOUD_MAP[self.typ](self._settings, self._config, self._config[self.typ])

        self.sanity_check()

    def sanity_check(self):
        """ sanity check a config

        check that we have a cloud and storagePath
        if aws, check can read credentials
        if google, check ProjectName

        Returns:
            (T,) if good
            (F, issues) if bad
        """
        for key in ['Cloud']:
            if key.lower() not in self._config['Global'].keys() or len(self._config['Global'][key]) < 1:
                raise ConfigError('Please set %s in section Global in config file %s' % (key, self.file))

        # now that a cloud is chosen, can check StoragePath
        sp = self._cloud.storage_path
        if sp is None or len(sp) == 0:
            raise ConfigError('Please set StoragePath = bucket/{optional path} in conf file "%s" '
                           'either in Global or a cloud specific section' % self.CONFIG)

        self._cloud.sanity_check()

    def sync(self, fname):
        return self._cloud.sync(fname)

    def remove_from_cloud(self, item):
        return self._cloud.remove_from_cloud(item)
<|endoftext|>


<commit_before># Copyright 2015 OpenStack Foundation.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron_lib.utils import host
from oslo_config import cfg

from neutron._i18n import _

DEDUCE_MODE = 'deduce'
USER_MODE = 'user'
GROUP_MODE = 'group'
ALL_MODE = 'all'
SOCKET_MODES = (DEDUCE_MODE, USER_MODE, GROUP_MODE, ALL_MODE)

SHARED_OPTS = [
    cfg.StrOpt('metadata_proxy_socket',
               default='$state_path/metadata_proxy',
               help=_('Location for Metadata Proxy UNIX domain socket.')),
    cfg.StrOpt('metadata_proxy_user',
               default='',
               help=_("User (uid or name) running metadata proxy after "
                      "its initialization (if empty: agent effective "
                      "user).")),
    cfg.StrOpt('metadata_proxy_group',
               default='',
               help=_("Group (gid or name) running metadata proxy after "
                      "its initialization (if empty: agent effective "
                      "group)."))
]

METADATA_PROXY_HANDLER_OPTS = [
    cfg.StrOpt('auth_ca_cert',
               help=_("Certificate Authority public key (CA cert) "
                      "file for ssl")),
    cfg.HostAddressOpt('nova_metadata_host',
                       default='127.0.0.1',
                       deprecated_name='nova_metadata_ip',
                       help=_("IP address or DNS name of Nova metadata "
                              "server.")),
    cfg.PortOpt('nova_metadata_port',
                default=8775,
                help=_("TCP Port used by Nova metadata server.")),
    cfg.StrOpt('metadata_proxy_shared_secret',
               default='',
               help=_('When proxying metadata requests, Neutron signs the '
                      'Instance-ID header with a shared secret to prevent '
                      'spoofing. You may select any string for a secret, '
                      'but it must match here and in the configuration used '
                      'by the Nova Metadata Server. NOTE: Nova uses the same '
                      'config key, but in [neutron] section.'),
               secret=True),
    cfg.StrOpt('nova_metadata_protocol',
               default='http',
               choices=['http', 'https'],
               help=_("Protocol to access nova metadata, http or https")),
    cfg.BoolOpt('nova_metadata_insecure', default=False,
                help=_("Allow to perform insecure SSL (https) requests to "
                       "nova metadata")),
    cfg.StrOpt('nova_client_cert',
               default='',
               help=_("Client certificate for nova metadata api server.")),
    cfg.StrOpt('nova_client_priv_key',
               default='',
               help=_("Private key of client certificate."))
]

UNIX_DOMAIN_METADATA_PROXY_OPTS = [
    cfg.StrOpt('metadata_proxy_socket_mode',
               default=DEDUCE_MODE,
               choices=SOCKET_MODES,
               help=_("Metadata Proxy UNIX domain socket mode, 4 values "
                      "allowed: "
                      "'deduce': deduce mode from metadata_proxy_user/group "
                      "values, "
                      "'user': set metadata proxy socket mode to 0o644, to "
                      "use when metadata_proxy_user is agent effective user "
                      "or root, "
                      "'group': set metadata proxy socket mode to 0o664, to "
                      "use when metadata_proxy_group is agent effective "
                      "group or root, "
                      "'all': set metadata proxy socket mode to 0o666, to use "
                      "otherwise.")),
    cfg.IntOpt('metadata_workers',
               default=host.cpu_count() // 2,
               help=_('Number of separate worker processes for metadata '
                      'server (defaults to half of the number of CPUs)')),
    cfg.IntOpt('metadata_backlog',
               default=4096,
               help=_('Number of backlog requests to configure the '
                      'metadata server socket with'))
]

def register_meta_conf_opts(opts, cfg=cfg.CONF):
    cfg.register_opts(opts)
<commit_msg>Remove deprecated nova_metadata_ip<commit_after># Copyright 2015 OpenStack Foundation.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from neutron_lib.utils import host
from oslo_config import cfg

from neutron._i18n import _

DEDUCE_MODE = 'deduce'
USER_MODE = 'user'
GROUP_MODE = 'group'
ALL_MODE = 'all'
SOCKET_MODES = (DEDUCE_MODE, USER_MODE, GROUP_MODE, ALL_MODE)

SHARED_OPTS = [
    cfg.StrOpt('metadata_proxy_socket',
               default='$state_path/metadata_proxy',
               help=_('Location for Metadata Proxy UNIX domain socket.')),
    cfg.StrOpt('metadata_proxy_user',
               default='',
               help=_("User (uid or name) running metadata proxy after "
                      "its initialization (if empty: agent effective "
                      "user).")),
    cfg.StrOpt('metadata_proxy_group',
               default='',
               help=_("Group (gid or name) running metadata proxy after "
                      "its initialization (if empty: agent effective "
                      "group)."))
]

METADATA_PROXY_HANDLER_OPTS = [
    cfg.StrOpt('auth_ca_cert',
               help=_("Certificate Authority public key (CA cert) "
                      "file for ssl")),
    cfg.HostAddressOpt('nova_metadata_host',
                       default='127.0.0.1',
                       help=_("IP address or DNS name of Nova metadata "
                              "server.")),
    cfg.PortOpt('nova_metadata_port',
                default=8775,
                help=_("TCP Port used by Nova metadata server.")),
    cfg.StrOpt('metadata_proxy_shared_secret',
               default='',
               help=_('When proxying metadata requests, Neutron signs the '
                      'Instance-ID header with a shared secret to prevent '
                      'spoofing. You may select any string for a secret, '
                      'but it must match here and in the configuration used '
                      'by the Nova Metadata Server. NOTE: Nova uses the same '
                      'config key, but in [neutron] section.'),
               secret=True),
    cfg.StrOpt('nova_metadata_protocol',
               default='http',
               choices=['http', 'https'],
               help=_("Protocol to access nova metadata, http or https")),
    cfg.BoolOpt('nova_metadata_insecure', default=False,
                help=_("Allow to perform insecure SSL (https) requests to "
                       "nova metadata")),
    cfg.StrOpt('nova_client_cert',
               default='',
               help=_("Client certificate for nova metadata api server.")),
    cfg.StrOpt('nova_client_priv_key',
               default='',
               help=_("Private key of client certificate."))
]

UNIX_DOMAIN_METADATA_PROXY_OPTS = [
    cfg.StrOpt('metadata_proxy_socket_mode',
               default=DEDUCE_MODE,
               choices=SOCKET_MODES,
               help=_("Metadata Proxy UNIX domain socket mode, 4 values "
                      "allowed: "
                      "'deduce': deduce mode from metadata_proxy_user/group "
                      "values, "
                      "'user': set metadata proxy socket mode to 0o644, to "
                      "use when metadata_proxy_user is agent effective user "
                      "or root, "
                      "'group': set metadata proxy socket mode to 0o664, to "
                      "use when metadata_proxy_group is agent effective "
                      "group or root, "
                      "'all': set metadata proxy socket mode to 0o666, to use "
                      "otherwise.")),
    cfg.IntOpt('metadata_workers',
               default=host.cpu_count() // 2,
               help=_('Number of separate worker processes for metadata '
                      'server (defaults to half of the number of CPUs)')),
    cfg.IntOpt('metadata_backlog',
               default=4096,
               help=_('Number of backlog requests to configure the '
                      'metadata server socket with'))
]

def register_meta_conf_opts(opts, cfg=cfg.CONF):
    cfg.register_opts(opts)
<|endoftext|>


<commit_before>import random
from weakref import WeakKeyDictionary

import numpy as np

from .core import CallbackProperty

__all__ = ['ChoiceSeparator', 'SelectionCallbackProperty']

class ChoiceSeparator(str):
    pass

class SelectionCallbackProperty(CallbackProperty):

    def __init__(self, default_index=0, choices=None, display_func=None, **kwargs):
        if choices is not None and 'default' not in kwargs:
            kwargs['default'] = choices[default_index]
        super(SelectionCallbackProperty, self).__init__(**kwargs)
        self.default_index = default_index
        self.default_choices = choices or []
        self._default_display_func = display_func
        self._choices = WeakKeyDictionary()
        self._display = WeakKeyDictionary()
        self._force_next_sync = WeakKeyDictionary()

    def __set__(self, instance, value):
        if value is not None:
            choices = self.get_choices(instance)
            # For built-in scalar types we use ==, and for other types we use
            # is, otherwise e.g. ComponentID returns something that evaluates
            # to true when using ==.
            if ((np.isscalar(value) and not any(value == x for x in choices)) or
                    (not np.isscalar(value) and not any(value is x for x in choices))):
                raise ValueError('value {0} is not in valid choices: {1}'.format(value, choices))
        super(SelectionCallbackProperty, self).__set__(instance, value)

    def force_next_sync(self, instance):
        self._force_next_sync[instance] = True

    def _get_full_info(self, instance):
        if self._force_next_sync.get(instance, False):
            try:
                return self.__get__(instance), random.random()
            finally:
                self._force_next_sync[instance] = False
        else:
            return self.__get__(instance), self.get_choices(instance), self.get_choice_labels(instance)

    def get_display_func(self, instance):
        return self._display.get(instance, self._default_display_func)

    def set_display_func(self, instance, display):
        self._display[instance] = display
        # selection = self.__get__(instance)
        # self.notify(instance, selection, selection)

    def get_choices(self, instance):
        return self._choices.get(instance, self.default_choices)

    def get_choice_labels(self, instance):
        display = self._display.get(instance, str)
        labels = []
        for choice in self.get_choices(instance):
            if isinstance(choice, ChoiceSeparator):
                labels.append(str(choice))
            else:
                labels.append(display(choice))
        return labels

    def set_choices(self, instance, choices):
        self._choices[instance] = choices
        self._choices_updated(instance, choices)
        selection = self.__get__(instance)
        self.notify(instance, selection, selection)

    def _choices_updated(self, instance, choices):

        if not choices:
            self.__set__(instance, None)
            return

        selection = self.__get__(instance)

        # We do the following because 'selection in choice' actually compares
        # equality not identity (and we really just care about identity here)
        for choice in choices:
            if selection is choice:
                return

        choices_without_separators = [choice for choice in choices
                                      if not isinstance(choice, ChoiceSeparator)]

        if choices_without_separators:
            try:
                selection = choices_without_separators[self.default_index]
            except IndexError:
                if self.default_index > 0:
                    selection = choices_without_separators[-1]
                else:
                    selection = choices_without_separators[0]
        else:
            selection = None

        self.__set__(instance, selection)
<commit_msg>Fix bug in updating of choices<commit_after>import random
from weakref import WeakKeyDictionary

import numpy as np

from .core import CallbackProperty

__all__ = ['ChoiceSeparator', 'SelectionCallbackProperty']

class ChoiceSeparator(str):
    pass

class SelectionCallbackProperty(CallbackProperty):

    def __init__(self, default_index=0, choices=None, display_func=None, **kwargs):
        if choices is not None and 'default' not in kwargs:
            kwargs['default'] = choices[default_index]
        super(SelectionCallbackProperty, self).__init__(**kwargs)
        self.default_index = default_index
        self.default_choices = choices or []
        self._default_display_func = display_func
        self._choices = WeakKeyDictionary()
        self._display = WeakKeyDictionary()
        self._force_next_sync = WeakKeyDictionary()

    def __set__(self, instance, value):
        if value is not None:
            choices = self.get_choices(instance)
            # For built-in scalar types we use ==, and for other types we use
            # is, otherwise e.g. ComponentID returns something that evaluates
            # to true when using ==.
            if ((np.isscalar(value) and not any(value == x for x in choices)) or
                    (not np.isscalar(value) and not any(value is x for x in choices))):
                raise ValueError('value {0} is not in valid choices: {1}'.format(value, choices))
        super(SelectionCallbackProperty, self).__set__(instance, value)

    def force_next_sync(self, instance):
        self._force_next_sync[instance] = True

    def _get_full_info(self, instance):
        if self._force_next_sync.get(instance, False):
            try:
                return self.__get__(instance), random.random()
            finally:
                self._force_next_sync[instance] = False
        else:
            return self.__get__(instance), self.get_choices(instance), self.get_choice_labels(instance)

    def get_display_func(self, instance):
        return self._display.get(instance, self._default_display_func)

    def set_display_func(self, instance, display):
        self._display[instance] = display
        # selection = self.__get__(instance)
        # self.notify(instance, selection, selection)

    def get_choices(self, instance):
        return self._choices.get(instance, self.default_choices)

    def get_choice_labels(self, instance):
        display = self._display.get(instance, str)
        labels = []
        for choice in self.get_choices(instance):
            if isinstance(choice, ChoiceSeparator):
                labels.append(str(choice))
            else:
                labels.append(display(choice))
        return labels

    def set_choices(self, instance, choices):
        self._choices[instance] = choices
        self._choices_updated(instance, choices)
        selection = self.__get__(instance)
        self.notify(instance, selection, selection)

    def _choices_updated(self, instance, choices):

        if not choices:
            self.__set__(instance, None)
            return

        selection = self.__get__(instance)

        # We do the following because 'selection in choice' actually compares
        # equality not identity (and we really just care about identity here)
        # However, for simple Python types, we also need to check ==.
        for choice in choices:
            if selection is choice or ((np.isreal(choice) or isinstance(choice, str)) and selection == choice):
                return

        choices_without_separators = [choice for choice in choices
                                      if not isinstance(choice, ChoiceSeparator)]

        if choices_without_separators:
            try:
                selection = choices_without_separators[self.default_index]
            except IndexError:
                if self.default_index > 0:
                    selection = choices_without_separators[-1]
                else:
                    selection = choices_without_separators[0]
        else:
            selection = None

        self.__set__(instance, selection)
<|endoftext|>


<commit_before>import ctypes as ct

def is_sequence(arg):
    return (not hasattr(arg, "strip") and
            hasattr(arg, "__getitem__") or
            hasattr(arg, "__iter__"))

def cast_ptr(obj, ptrType):
    return ct.cast(obj, ct.POINTER(ptrType))

def to_c_str(text, hackRef=[], extList=False):
    ''' Convert python strings to null terminated c strings. '''
    # We only want to keep the cString data around for 1 call
    # This is a pretty terrible hack but oh well :(
    if not extList and len(hackRef) != 0:
        hackRef.remove(hackRef[0])
    cStr = ct.create_string_buffer(text.encode(encoding='UTF-8'))
    hackRef.append(cStr)
    return ct.cast(ct.pointer(cStr), ct.POINTER(ct.c_char))

def conv_list(listIn, cType):
    ''' Convert a python list into a ctypes array '''
    return (cType * len(listIn))(*listIn)

def conv_list_2d(listIn, cType):
    ''' Convert a python 2d list into a ctypes 2d array '''
    xlength = len(listIn)
    ylength = len(listIn[0])

    arrayOut = (cType * ylength * xlength)()

    for x in range(xlength):
        for y in range(ylength):
            arrayOut[x][y] = listIn[x][y]

    return arrayOut<commit_msg>Add 2d to 1d list conversion function.<commit_after>import ctypes as ct

def is_sequence(arg):
    return (not hasattr(arg, "strip") and
            hasattr(arg, "__getitem__") or
            hasattr(arg, "__iter__"))

def cast_ptr(obj, ptrType):
    return ct.cast(obj, ct.POINTER(ptrType))

def to_c_str(text, hackRef=[], extList=False):
    ''' Convert python strings to null terminated c strings. '''
    # We only want to keep the cString data around for 1 call
    # This is a pretty terrible hack but oh well :(
    if not extList and len(hackRef) != 0:
        hackRef.remove(hackRef[0])
    cStr = ct.create_string_buffer(text.encode(encoding='UTF-8'))
    hackRef.append(cStr)
    return ct.cast(ct.pointer(cStr), ct.POINTER(ct.c_char))

def conv_list(listIn, cType):
    ''' Convert a python list into a ctypes array '''
    return (cType * len(listIn))(*listIn)

def conv_list_2d(listIn, cType):
    ''' Convert a python 2d list into a ctypes 2d array '''
    xlength = len(listIn)
    ylength = len(listIn[0])

    arrayOut = (cType * ylength * xlength)()

    for x in range(xlength):
        for y in range(ylength):
            arrayOut[x][y] = listIn[x][y]

    return arrayOut

def list_2d_to_1d(inlist):
    ''' Convert a python 2D list to python 1D list. '''
    sizeX = len(inlist[0])
    sizeY = len(inlist)

    rtnList = [None for x in range(sizeY*sizeX)]

    for x in range(sizeY):
        for y in range(sizeX):
            print((y + x * sizeX), x, y)
            rtnList[y + x * sizeX] = inlist[x][y]

    return rtnList
<|endoftext|>


<commit_before>"""
Django base settings for nodeconductor project.
"""
from __future__ import absolute_import

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import warnings

from datetime import timedelta
from celery.schedules import crontab

from nodeconductor.core import NodeConductorExtension
from nodeconductor.server.admin.settings import *

BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), '..'))

DEBUG = False

TEMPLATE_DEBUG = False

MEDIA_ROOT = '/tmp/'

MEDIA_URL = '/media/'

ALLOWED_HOSTS = []

# Application definition
INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.humanize',
    'django.contrib.staticfiles',

    'nodeconductor.landing',
    'nodeconductor.logging',
    'nodeconductor.core',
    'nodeconductor.backup',
    'nodeconductor.template',
    'nodeconductor.monitoring',
    'nodeconductor.quotas',
    'nodeconductor.structure',
    'nodeconductor.cost_tracking',
    'nodeconductor.billing',
    'nodeconductor.openstack',
    # 'nodeconductor.oracle',
    'nodeconductor.iaas',
    'nodeconductor.support',
    'nodeconductor.ldapsync',
    # TODO: remove
    'nodeconductor_zabbix',

    'nodeconductor.testdata',

    'rest_framework',
    'rest_framework.authtoken',

    'permission',
    'django_fsm',
    'reversion',
)

INSTALLED_APPS += ADMIN_APPS

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'nodeconductor.logging.middleware.CaptureEventContextMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

REST_FRAMEWORK = {
    'TEST_REQUEST_DEFAULT_FORMAT': 'json',
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'nodeconductor.core.authentication.TokenAuthentication',
        'nodeconductor.core.authentication.SessionAuthentication',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
    ),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.JSONRenderer',
        'nodeconductor.core.renderers.BrowsableAPIRenderer',
    ),
    'DEFAULT_PAGINATION_CLASS': 'nodeconductor.core.pagination.LinkHeaderPagination',
    'PAGE_SIZE': 10,
    'EXCEPTION_HANDLER': 'nodeconductor.core.views.exception_handler',

    # Return native `Date` and `Time` objects in `serializer.data`
    'DATETIME_FORMAT': None,
    'DATE_FORMAT': None,
    'TIME_FORMAT': None,
}

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'permission.backends.PermissionBackend',
)

ANONYMOUS_USER_ID = None

TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, 'nodeconductor', 'templates'),
    os.path.join(BASE_DIR, 'nodeconductor', 'billing', 'templates'),
    os.path.join(BASE_DIR, 'nodeconductor', 'landing', 'templates'),
)

ROOT_URLCONF = 'nodeconductor.server.urls'

AUTH_USER_MODEL = 'core.User'

WSGI_APPLICATION = 'nodeconductor.server.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

BROKER_URL = 'redis://localhost'
CELERY_RESULT_BACKEND = 'redis://localhost'

CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_RESULT_SERIALIZER = 'json'

CELERY_QUEUES = {
    'tasks': {'exchange': 'tasks'},
    'heavy': {'exchange': 'heavy'},
}
CELERY_DEFAULT_QUEUE = 'tasks'
CELERY_ROUTES = ('nodeconductor.server.celery.PriorityRouter',)

# Regular tasks
CELERYBEAT_SCHEDULE = {
    'update-instance-monthly-slas': {
        'task': 'nodeconductor.monitoring.tasks.update_instance_sla',
        'schedule': timedelta(minutes=5),
        'args': ('monthly',),
    },
    'update-instance-yearly-slas': {
        'task': 'nodeconductor.monitoring.tasks.update_instance_sla',
        'schedule': timedelta(minutes=10),
        'args': ('yearly',),
    },

    'debit-customers': {
        'task': 'nodeconductor.billing.debit_customers',
        'schedule': crontab(hour=0, minute=30),
        'args': (),
    },

    'sync-services': {
        'task': 'nodeconductor.iaas.sync_services',
        'schedule': timedelta(minutes=60),
        'args': (),
    },

    'sync-service-settings': {
        'task': 'nodeconductor.structure.sync_service_settings',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'recover-erred-services': {
        'task': 'nodeconductor.structure.recover_erred_services',
        'schedule': timedelta(minutes=45),
        'args': (),
    },

    'pull-service-statistics': {
        'task': 'nodeconductor.iaas.tasks.iaas.pull_service_statistics',
        'schedule': timedelta(minutes=15),
        'args': (),
    },
    'pull-cloud-project-memberships': {
        'task': 'nodeconductor.iaas.tasks.iaas.pull_cloud_memberships',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'check-cloud-project-memberships-quotas': {
        'task': 'nodeconductor.iaas.tasks.iaas.check_cloud_memberships_quotas',
        'schedule': timedelta(minutes=1440),
        'args': (),
    },

    'sync-instances-with-zabbix': {
        'task': 'nodeconductor.iaas.tasks.iaas.sync_instances_with_zabbix',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'create-zabbix-hosts-and-services': {
        'task': 'nodeconductor.iaas.tasks.zabbix.zabbix_create_host_and_service_for_all_instances',
        'schedule': timedelta(hours=1),
        'args': (),
    },

    'execute-backup-schedules': {
        'task': 'nodeconductor.backup.tasks.execute_schedules',
        'schedule': timedelta(minutes=10),
        'args': (),
    },

    'delete-expired-backups': {
        'task': 'nodeconductor.backup.tasks.delete_expired_backups',
        'schedule': timedelta(minutes=10),
        'args': (),
    },

    'pull-instances-installation-state': {
        'task': 'nodeconductor.iaas.tasks.zabbix.pull_instances_installation_state',
        'schedule': timedelta(minutes=1),
        'args': (),
    },

    'update-current-month-cost-projections': {
        'task': 'nodeconductor.cost_tracking.update_projected_estimate',
        'schedule': timedelta(hours=24),
        'args': (),
    },

    'update-today-usage': {
        'task': 'nodeconductor.cost_tracking.update_today_usage',
        'schedule': crontab(minute=10),
        'args': (),
    },

    'update-openstack-service-project-links-quotas': {
        'task': 'nodeconductor.structure.sync_service_project_links',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'close-alerts-without-scope': {
        'task': 'nodeconductor.logging.close_alerts_without_scope',
        'schedule': timedelta(minutes=30),
        'args': (),
    }
}

CELERY_TASK_THROTTLING = {
    'nodeconductor.iaas.tasks.openstack.openstack_provision_instance': {
        'concurrency': 1,
        'retry_delay': 30,
    },
}

NODECONDUCTOR = {
    'EXTENSIONS_AUTOREGISTER': True,
    'ENABLE_ORDER_PROCESSING': False,
    'DEFAULT_SECURITY_GROUPS': (
        {
            'name': 'ssh',
            'description': 'Security group for secure shell access and ping',
            'rules': (
                {
                    'protocol': 'tcp',
                    'cidr': '0.0.0.0/0',
                    'from_port': 22,
                    'to_port': 22,
                },
                {
                    'protocol': 'icmp',
                    'cidr': '0.0.0.0/0',
                    'icmp_type': -1,
                    'icmp_code': -1,
                },
            ),
        },
    ),
    'JIRA_SUPPORT': {'dummy': True},
    'ELASTICSEARCH_DUMMY': True,
    'SUSPEND_UNPAID_CUSTOMERS': False,
    'TOKEN_KEY': 'x-auth-token',
}

for ext in NodeConductorExtension.get_extensions():
    INSTALLED_APPS += (ext.django_app(),)

    for name, task in ext.celery_tasks().items():
        if name in CELERYBEAT_SCHEDULE:
            warnings.warn(
                "Celery beat task %s from NodeConductor extension %s "
                "is overlapping with primary tasks definition" % (name, ext.django_app()))
        else:
            CELERYBEAT_SCHEDULE[name] = task

    for key, val in ext.Settings.__dict__.items():
        if not key.startswith('_'):
            globals()[key] = val
<commit_msg>Remove zabbix from installed apps<commit_after>"""
Django base settings for nodeconductor project.
"""
from __future__ import absolute_import

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
import warnings

from datetime import timedelta
from celery.schedules import crontab

from nodeconductor.core import NodeConductorExtension
from nodeconductor.server.admin.settings import *

BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(os.path.dirname(__file__)), '..'))

DEBUG = False

TEMPLATE_DEBUG = False

MEDIA_ROOT = '/tmp/'

MEDIA_URL = '/media/'

ALLOWED_HOSTS = []

# Application definition
INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.humanize',
    'django.contrib.staticfiles',

    'nodeconductor.landing',
    'nodeconductor.logging',
    'nodeconductor.core',
    'nodeconductor.backup',
    'nodeconductor.template',
    'nodeconductor.monitoring',
    'nodeconductor.quotas',
    'nodeconductor.structure',
    'nodeconductor.cost_tracking',
    'nodeconductor.billing',
    'nodeconductor.openstack',
    # 'nodeconductor.oracle',
    'nodeconductor.iaas',
    'nodeconductor.support',
    'nodeconductor.ldapsync',

    'nodeconductor.testdata',

    'rest_framework',
    'rest_framework.authtoken',

    'permission',
    'django_fsm',
    'reversion',
)

INSTALLED_APPS += ADMIN_APPS

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'nodeconductor.logging.middleware.CaptureEventContextMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

REST_FRAMEWORK = {
    'TEST_REQUEST_DEFAULT_FORMAT': 'json',
    'DEFAULT_AUTHENTICATION_CLASSES': (
        'nodeconductor.core.authentication.TokenAuthentication',
        'nodeconductor.core.authentication.SessionAuthentication',
    ),
    'DEFAULT_PERMISSION_CLASSES': (
        'rest_framework.permissions.IsAuthenticated',
    ),
    'DEFAULT_FILTER_BACKENDS': ('rest_framework.filters.DjangoFilterBackend',),
    'DEFAULT_RENDERER_CLASSES': (
        'rest_framework.renderers.JSONRenderer',
        'nodeconductor.core.renderers.BrowsableAPIRenderer',
    ),
    'DEFAULT_PAGINATION_CLASS': 'nodeconductor.core.pagination.LinkHeaderPagination',
    'PAGE_SIZE': 10,
    'EXCEPTION_HANDLER': 'nodeconductor.core.views.exception_handler',

    # Return native `Date` and `Time` objects in `serializer.data`
    'DATETIME_FORMAT': None,
    'DATE_FORMAT': None,
    'TIME_FORMAT': None,
}

AUTHENTICATION_BACKENDS = (
    'django.contrib.auth.backends.ModelBackend',
    'permission.backends.PermissionBackend',
)

ANONYMOUS_USER_ID = None

TEMPLATE_DIRS = (
    os.path.join(BASE_DIR, 'nodeconductor', 'templates'),
    os.path.join(BASE_DIR, 'nodeconductor', 'billing', 'templates'),
    os.path.join(BASE_DIR, 'nodeconductor', 'landing', 'templates'),
)

ROOT_URLCONF = 'nodeconductor.server.urls'

AUTH_USER_MODEL = 'core.User'

WSGI_APPLICATION = 'nodeconductor.server.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'

BROKER_URL = 'redis://localhost'
CELERY_RESULT_BACKEND = 'redis://localhost'

CELERY_TASK_SERIALIZER = 'json'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_RESULT_SERIALIZER = 'json'

CELERY_QUEUES = {
    'tasks': {'exchange': 'tasks'},
    'heavy': {'exchange': 'heavy'},
}
CELERY_DEFAULT_QUEUE = 'tasks'
CELERY_ROUTES = ('nodeconductor.server.celery.PriorityRouter',)

# Regular tasks
CELERYBEAT_SCHEDULE = {
    'update-instance-monthly-slas': {
        'task': 'nodeconductor.monitoring.tasks.update_instance_sla',
        'schedule': timedelta(minutes=5),
        'args': ('monthly',),
    },
    'update-instance-yearly-slas': {
        'task': 'nodeconductor.monitoring.tasks.update_instance_sla',
        'schedule': timedelta(minutes=10),
        'args': ('yearly',),
    },

    'debit-customers': {
        'task': 'nodeconductor.billing.debit_customers',
        'schedule': crontab(hour=0, minute=30),
        'args': (),
    },

    'sync-services': {
        'task': 'nodeconductor.iaas.sync_services',
        'schedule': timedelta(minutes=60),
        'args': (),
    },

    'sync-service-settings': {
        'task': 'nodeconductor.structure.sync_service_settings',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'recover-erred-services': {
        'task': 'nodeconductor.structure.recover_erred_services',
        'schedule': timedelta(minutes=45),
        'args': (),
    },

    'pull-service-statistics': {
        'task': 'nodeconductor.iaas.tasks.iaas.pull_service_statistics',
        'schedule': timedelta(minutes=15),
        'args': (),
    },
    'pull-cloud-project-memberships': {
        'task': 'nodeconductor.iaas.tasks.iaas.pull_cloud_memberships',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'check-cloud-project-memberships-quotas': {
        'task': 'nodeconductor.iaas.tasks.iaas.check_cloud_memberships_quotas',
        'schedule': timedelta(minutes=1440),
        'args': (),
    },

    'sync-instances-with-zabbix': {
        'task': 'nodeconductor.iaas.tasks.iaas.sync_instances_with_zabbix',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'create-zabbix-hosts-and-services': {
        'task': 'nodeconductor.iaas.tasks.zabbix.zabbix_create_host_and_service_for_all_instances',
        'schedule': timedelta(hours=1),
        'args': (),
    },

    'execute-backup-schedules': {
        'task': 'nodeconductor.backup.tasks.execute_schedules',
        'schedule': timedelta(minutes=10),
        'args': (),
    },

    'delete-expired-backups': {
        'task': 'nodeconductor.backup.tasks.delete_expired_backups',
        'schedule': timedelta(minutes=10),
        'args': (),
    },

    'pull-instances-installation-state': {
        'task': 'nodeconductor.iaas.tasks.zabbix.pull_instances_installation_state',
        'schedule': timedelta(minutes=1),
        'args': (),
    },

    'update-current-month-cost-projections': {
        'task': 'nodeconductor.cost_tracking.update_projected_estimate',
        'schedule': timedelta(hours=24),
        'args': (),
    },

    'update-today-usage': {
        'task': 'nodeconductor.cost_tracking.update_today_usage',
        'schedule': crontab(minute=10),
        'args': (),
    },

    'update-openstack-service-project-links-quotas': {
        'task': 'nodeconductor.structure.sync_service_project_links',
        'schedule': timedelta(minutes=30),
        'args': (),
    },

    'close-alerts-without-scope': {
        'task': 'nodeconductor.logging.close_alerts_without_scope',
        'schedule': timedelta(minutes=30),
        'args': (),
    }
}

CELERY_TASK_THROTTLING = {
    'nodeconductor.iaas.tasks.openstack.openstack_provision_instance': {
        'concurrency': 1,
        'retry_delay': 30,
    },
}

NODECONDUCTOR = {
    'EXTENSIONS_AUTOREGISTER': True,
    'ENABLE_ORDER_PROCESSING': False,
    'DEFAULT_SECURITY_GROUPS': (
        {
            'name': 'ssh',
            'description': 'Security group for secure shell access and ping',
            'rules': (
                {
                    'protocol': 'tcp',
                    'cidr': '0.0.0.0/0',
                    'from_port': 22,
                    'to_port': 22,
                },
                {
                    'protocol': 'icmp',
                    'cidr': '0.0.0.0/0',
                    'icmp_type': -1,
                    'icmp_code': -1,
                },
            ),
        },
    ),
    'JIRA_SUPPORT': {'dummy': True},
    'ELASTICSEARCH_DUMMY': True,
    'SUSPEND_UNPAID_CUSTOMERS': False,
    'TOKEN_KEY': 'x-auth-token',
}

for ext in NodeConductorExtension.get_extensions():
    INSTALLED_APPS += (ext.django_app(),)

    for name, task in ext.celery_tasks().items():
        if name in CELERYBEAT_SCHEDULE:
            warnings.warn(
                "Celery beat task %s from NodeConductor extension %s "
                "is overlapping with primary tasks definition" % (name, ext.django_app()))
        else:
            CELERYBEAT_SCHEDULE[name] = task

    for key, val in ext.Settings.__dict__.items():
        if not key.startswith('_'):
            globals()[key] = val
<|endoftext|>