<commit_before>from itertools import groupby

from django.utils.translation import pgettext
from satchless import cart
from satchless.item import ItemSet, Partitioner

from delivery import DummyShipping, DigitalDelivery
from product.models import StockedProduct, DigitalShip

class BaseGroup(list, ItemSet):
    '''
    Base Saleor cart group. It is used in cart and checkout to group products
    depends on delivery methods.
    '''
    delivery_method = None

    def __init__(self, items, delivery_method=None):
        super(BaseGroup, self).__init__(items)
        if delivery_method and delivery_method not in self.delivery_method():
            raise AttributeError('Bad delivery method')
        self.delivery_method = delivery_method

    def get_delivery_methods(self, **kwargs):
        '''
        Method should returns iterable object with delivery groups.
        '''
        raise NotImplementedError()

    def get_delivery_total(self, **kwargs):
        '''
        Method returns price from the self.delivery_method or lowest price
        form delivery methods.
        '''
        if self.delivery_method:
            return self.delivery_method.get_price_per_item(**kwargs)
        methods = self.get_delivery_methods()
        return min(method.get_price_per_item(**kwargs) for method in methods)

class ShippedGroup(BaseGroup):
    '''
    Group for shippable products.
    '''
    def get_delivery_methods(self):
        '''
        Returns shippable delivery methods.
        '''
        yield DummyShipping(self)

class DigitalGroup(BaseGroup):
    '''
    Group for digital products.
    '''
    def get_delivery_methods(self, **kwargs):
        '''
        Returns digital delivery methods.
        '''
        yield DigitalDelivery(self)

class CartPartitioner(Partitioner):
    '''
    Dividing cart into groups.
    '''
    def __iter__(self):
        '''
        Change this method to provide custom delivery groups.
        '''
        for classifier, items in groupby(self.subject, self.classify):
            delivery_class = self.get_delivery_class(classifier)
            delivery = delivery_class(items)
            yield delivery

    def classify(self, item):
        return item.product.__class__

    def get_delivery_class(self, classifier):
        if issubclass(classifier, DigitalShip):
            return DigitalGroup
        return ShippedGroup

    def get_delivery_subtotal(self, partion, **kwargs):
        return partion.get_delivery_total(**kwargs)

    def get_delivery_total(self, **kwargs):
        items = [self.get_delivery_subtotal(partion, **kwargs)
                 for partion in self]
        if not items:
            raise AttributeError(
                'Calling get_delivery_total() for an empty cart')
        return sum(items[1:], items[0])

    def get_total(self):
        total = super(CartPartitioner, self).get_total()
        return total + self.get_delivery_total()

    def __repr__(self):
        return 'CartPartitioner(%r)' % (list(self),)

class InsufficientStockException(Exception):
    '''An error while validating product stock.'''
    def __init__(self, product):
        super(InsufficientStockException, self).__init__(
            'Insufficient stock for %r' % (product,))
        self.product = product

class Cart(cart.Cart):
    '''
    Contains cart items. Serialized instance of cart is saved into django
    session.
    '''
    SESSION_KEY = 'cart'
    timestamp = None
    billing_address = None

    def __unicode__(self):
        return pgettext(u'Shopping cart', u'Your cart (%(cart_count)s)') % {
            'cart_count': self.count()}

    def check_quantity(self, product, quantity, data=None):
        '''
        Raises exception when product has stock and user insert too big
        quantity.
        '''
        super(Cart, self).check_quantity(product, quantity, data)
        if isinstance(product, StockedProduct) and quantity > product.stock:
            raise InsufficientStockException(product)

def remove_cart_from_request(request):
    '''
    Method removes cart instance from django session.
    '''
    del request.session[Cart.SESSION_KEY]
<commit_msg>Sort items before grouping<commit_after>from itertools import groupby

from django.utils.translation import pgettext
from satchless import cart
from satchless.item import ItemSet, Partitioner

from delivery import DummyShipping, DigitalDelivery
from product.models import StockedProduct, DigitalShip

class BaseGroup(list, ItemSet):
    '''
    Base Saleor cart group. It is used in cart and checkout to group products
    depends on delivery methods.
    '''
    delivery_method = None

    def __init__(self, items, delivery_method=None):
        super(BaseGroup, self).__init__(items)
        if delivery_method and delivery_method not in self.delivery_method():
            raise AttributeError('Bad delivery method')
        self.delivery_method = delivery_method

    def get_delivery_methods(self, **kwargs):
        '''
        Method should returns iterable object with delivery groups.
        '''
        raise NotImplementedError()

    def get_delivery_total(self, **kwargs):
        '''
        Method returns price from the self.delivery_method or lowest price
        form delivery methods.
        '''
        if self.delivery_method:
            return self.delivery_method.get_price_per_item(**kwargs)
        methods = self.get_delivery_methods()
        return min(method.get_price_per_item(**kwargs) for method in methods)

class ShippedGroup(BaseGroup):
    '''
    Group for shippable products.
    '''
    def get_delivery_methods(self):
        '''
        Returns shippable delivery methods.
        '''
        yield DummyShipping(self)

class DigitalGroup(BaseGroup):
    '''
    Group for digital products.
    '''
    def get_delivery_methods(self, **kwargs):
        '''
        Returns digital delivery methods.
        '''
        yield DigitalDelivery(self)

class CartPartitioner(Partitioner):
    '''
    Dividing cart into groups.
    '''
    def __iter__(self):
        '''
        Change this method to provide custom delivery groups.
        '''
        self.subject = sorted(self.subject, key=self.classify)
        for classifier, items in groupby(self.subject, self.classify):
            delivery_class = self.get_delivery_class(classifier)
            delivery = delivery_class(items)
            yield delivery

    def classify(self, item):
        return item.product.__class__

    def get_delivery_class(self, classifier):
        if issubclass(classifier, DigitalShip):
            return DigitalGroup
        return ShippedGroup

    def get_delivery_subtotal(self, partion, **kwargs):
        return partion.get_delivery_total(**kwargs)

    def get_delivery_total(self, **kwargs):
        items = [self.get_delivery_subtotal(partion, **kwargs)
                 for partion in self]
        if not items:
            raise AttributeError(
                'Calling get_delivery_total() for an empty cart')
        return sum(items[1:], items[0])

    def get_total(self):
        total = super(CartPartitioner, self).get_total()
        return total + self.get_delivery_total()

    def __repr__(self):
        return 'CartPartitioner(%r)' % (list(self),)

class InsufficientStockException(Exception):
    '''An error while validating product stock.'''
    def __init__(self, product):
        super(InsufficientStockException, self).__init__(
            'Insufficient stock for %r' % (product,))
        self.product = product

class Cart(cart.Cart):
    '''
    Contains cart items. Serialized instance of cart is saved into django
    session.
    '''
    SESSION_KEY = 'cart'
    timestamp = None
    billing_address = None

    def __unicode__(self):
        return pgettext(u'Shopping cart', u'Your cart (%(cart_count)s)') % {
            'cart_count': self.count()}

    def check_quantity(self, product, quantity, data=None):
        '''
        Raises exception when product has stock and user insert too big
        quantity.
        '''
        super(Cart, self).check_quantity(product, quantity, data)
        if isinstance(product, StockedProduct) and quantity > product.stock:
            raise InsufficientStockException(product)

def remove_cart_from_request(request):
    '''
    Method removes cart instance from django session.
    '''
    del request.session[Cart.SESSION_KEY]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

import csv
import os
import sys
from difflib import SequenceMatcher

import numpy as np
import pandas
import tensorflow as tf

from niftynet.io.misc_io import touch_folder
from niftynet.utilities.filename_matching import KeywordsMatching

def match_first_degree(name_list1, name_list2):
    """
    First immediate matching between two possible name lists (exact equality
    between one item of list1 and of list2
    :param name_list1: First list of names to match
    :param name_list2: Second list of names where to find a match
    :return init_match1:
    :return init_match2:
    :return ind_match1: Indices of second list that correspond to each given
    item of list 1 if exists (-1 otherwise)
    :return ind_match2: Indices of first list that correspond to each given
    item of list 2 if exists (-1) otherwise
    """
    if name_list1 is None or name_list2 is None:
        return None, None, None, None
    init_match1 = [''] * len(name_list1)
    init_match2 = [''] * len(name_list2)
    ind_match1 = [-1] * len(name_list1)
    ind_match2 = [-1] * len(name_list2)
    flatten_list1 = [item for sublist in name_list1 for item in sublist]
    flatten_list2 = [item for sublist in name_list2 for item in sublist]
    indflat_1 = [i for i in range(0, len(init_match1)) for item in
                 name_list1[i] if init_match1[i] == '']
    indflat_2 = [i for i in range(0, len(init_match2)) for item in
                 name_list2[i] if init_match2[i] == '']
    for i in range(0, len(name_list1)):
        for name in name_list1[i]:
            if name in flatten_list2:
                init_match1[i] = name
                ind_match1[i] = indflat_2[flatten_list2.index(name)]
                break
    for i in range(0, len(name_list2)):
        for name in name_list2[i]:
            if name in flatten_list1:
                init_match2[i] = name
                ind_match2[i] = indflat_1[flatten_list1.index(name)]
                break
    return init_match1, init_match2, ind_match1, ind_match2

def __find_max_overlap_in_list(name, list_names):
    """
    Given a name and list of names to match it to, find the maximum overlap
    existing

    :param name: string to match to any of list_names
    :param list_names: list of candidate strings
    :return match_seq: matched substring
    :return index: index of element in list_names to which the match is
    associated. Returns -1 if there is no found match
    """
    match_max = 0
    match_seq = ''
    match_orig = ''
    match_ratio = 0
    if not list_names:
        return '', -1
    for test in list_names:
        if test:
            match = SequenceMatcher(None, name, test).find_longest_match(
                0, len(name), 0, len(test))
            if match.size >= match_max and match.size / len(test) >= \
                    match_ratio:
                match_max = match.size
                match_seq = test[match.b:(match.b + match.size)]
                match_ratio = match.size / len(test)
                match_orig = test
    if match_max == 0:
        return '', -1
    other_list = [name for name in list_names if match_seq in name and
                  match_max / len(name) == match_ratio]
    if len(other_list) > 1:
        return '', -1
    return match_seq, list_names.index(match_orig)

def match_second_degree(name_list1, name_list2):
    """
    Perform the double matching between two lists of
    possible names.
    First find the direct matches, remove them from
    the ones still to match and
    match the remaining ones using the maximum overlap.
    Returns the name
    match for each list, and the index correspondences.

    More subtle matching with first direct matching and then secondary
    overlap matching between list of list of potential names
    :param name_list1:
    :param name_list2:
    :return init_match1:
    :return ind_match1: Index of corresponding match in name_list2
    :return init_match2: Matching string in list2
    :return ind_match2: Index of corresponding match in name_list1
    """
    if name_list1 is None or name_list2 is None:
        return None, None, None, None
    init_match1, init_match2, ind_match1, ind_match2 = match_first_degree(
        name_list1, name_list2)
    reduced_list1 = [names for names in name_list1
                     if init_match1[name_list1.index(names)] == '']
    reduced_list2 = [names for names in name_list2
                     if init_match2[name_list2.index(names)] == '']
    redflat_1 = [item for sublist in reduced_list1 for item in sublist]
    indflat_1 = [i for i in range(0, len(init_match1)) for item in
                 name_list1[i] if init_match1[i] == '']
    redflat_2 = [item for sublist in reduced_list2 for item in sublist]
    indflat_2 = [i for i in range(0, len(init_match2)) for item in
                 name_list2[i] if init_match2[i] == '']
    for i in range(0, len(name_list1)):
        if init_match1[i] == '':
            for n in name_list1[i]:
                init_match1[i], index = __find_max_overlap_in_list(n, redflat_2)
                if index >= 0:
                    ind_match1[i] = indflat_2[index]
    for i in range(0, len(name_list2)):
        if init_match2[i] == '':
            for n in name_list2[i]:
                init_match2[i], index = __find_max_overlap_in_list(n, redflat_1)
                if index >= 0:
                    ind_match2[i] = indflat_1[index]
    return init_match1, ind_match1, init_match2, ind_match2

# From a list of list of names and a list of list of files that are
# associated, find the name correspondance and therefore the files associations
def join_subject_id_and_filename_list(name_list, list_files):
    """
    From the list of list of names and the list of list of files
    corresponding to each constraint find the association between a single
    name id and the different file lists
    :param name_list: list of list of names
    :param list_files: list of list of files (one list per constraint)
    :return list_combined: List per subject of name and list of files given
    by the constraints
    """
    ind_max = np.argmax([len(names) for names in name_list])
    name_max = name_list[ind_max]
    name_tot = []
    ind_tot = []
    name_max_to_use = []
    for c in range(0, len(list_files)):
        name_match, ind_match, _, _ = match_second_degree(name_max,
                                                          name_list[c])
        if c == ind_max:
            name_max_to_use = name_match
        name_tot.append(name_match)
        ind_tot.append(ind_match)

    list_combined = []
    for (i, name) in enumerate(name_max_to_use):
        list_temp = [name]
        'To do : Taking care of the case when the list of a constraint is ' \
        'completely empty'
        for c in range(0, len(list_files)):
            output = list_files[c][ind_tot[c][i]] if ind_tot[c][i] > -1 else ''
            list_temp.append(output)
        list_combined.append(list_temp)
    return list_combined

def remove_duplicated_names(name_list):
    """
    From a list of list of names remove the items that are duplicated
    :param name_list: list of list of names to investigate
    :return duplicates_removed: list of list of names freed of duplicates
    """
    flattened_list = [item for sublist in name_list for item in sublist]
    list_duplicated = [item for item in flattened_list
                       if flattened_list.count(item) > 1]
    duplicates_removed = []
    for names in name_list:
        duplicates_removed.append([name for name in names
                                   if name not in list_duplicated])
    return duplicates_removed

def match_and_write_filenames_to_csv(list_constraints, csv_file):
    """
    Combine all elements of file searching until finally writing the names
    :param list_constraints: list of constraints (defined by list of paths to
    search, list of elements the filename should contain and of those that
    are forbidden
    :param csv_file: file on which to write the final list of files.
    :return:
    """
    name_tot = []
    list_tot = []
    if list_constraints is None or len(list_constraints) == 0:
        return
    for c in list_constraints:
        list_files, name_list = \
            KeywordsMatching.matching_subjects_and_filenames(c)
        name_list = remove_duplicated_names(name_list)
        name_tot.append(name_list)
        list_tot.append(list_files)
    list_combined = join_subject_id_and_filename_list(name_tot, list_tot)
    touch_folder(os.path.dirname(csv_file))

    # csv writer has different behaviour in python 2/3
    if sys.version_info[0] >= 3:
        with open(csv_file, 'w', newline='', encoding='utf8') as csvfile:
            file_writer = csv.writer(csvfile)
            for list_temp in list_combined:
                file_writer.writerow(list_temp)
    else:
        with open(csv_file, 'wb') as csvfile:
            file_writer = csv.writer(csvfile, delimiter=',')
            for list_temp in list_combined:
                file_writer.writerow(list_temp)
    return

def load_and_merge_csv_files(data_param):
    """
    Converts a list of csv_files in data_param
    in to a joint list of file names (by matching the first column)
    This function returns a <pandas.core.frame.DataFrame> of the
    joint list
    """
    if not data_param:
        tf.logging.fatal('nothing to load, please check reader.names')
        raise ValueError
    _file_list = None
    for modality_name in data_param:
        try:
            csv_file = data_param[modality_name].csv_file
        except AttributeError:
            tf.logging.fatal('unrecognised parameter format')
            raise
        if hasattr(data_param[modality_name], 'path_to_search'):
            tf.logging.info(
                '[%s] search file folders, writing csv file %s',
                modality_name, csv_file)
            section_tuple = data_param[modality_name].__dict__.items()
            matcher = KeywordsMatching.from_tuple(section_tuple)
            match_and_write_filenames_to_csv([matcher], csv_file)
        else:
            tf.logging.info(
                '[%s] using existing csv file %s, skipped folder search',
                modality_name, csv_file)
        if not os.path.isfile(csv_file):
            tf.logging.fatal(
                "[%s] csv file %s not found.", modality_name, csv_file)
            raise IOError
        csv_list = pandas.read_csv(
            csv_file, header=None, names=['subject_id', modality_name])
        if _file_list is None:
            _file_list = csv_list
            continue

        # merge _file_list based on subject_ids (first column of each csv)
        n_rows = _file_list.shape[0]
        _file_list = pandas.merge(_file_list, csv_list, on='subject_id')
        if _file_list.shape[0] != n_rows:
            tf.logging.warning("rows not matched in %s", csv_file)

    if _file_list is None or _file_list.size == 0:
        tf.logging.fatal(
            "empty filename lists, please check the csv "
            "files. (remove csv files to automatically"
            "search folders and generate new csv files again)")
        raise IOError
    return _file_list
<commit_msg>Fix bug when default path_to_search is blank and csv_file is specified<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

import csv
import os
import sys
from difflib import SequenceMatcher

import numpy as np
import pandas
import tensorflow as tf

from niftynet.io.misc_io import touch_folder
from niftynet.utilities.filename_matching import KeywordsMatching

def match_first_degree(name_list1, name_list2):
    """
    First immediate matching between two possible name lists (exact equality
    between one item of list1 and of list2
    :param name_list1: First list of names to match
    :param name_list2: Second list of names where to find a match
    :return init_match1:
    :return init_match2:
    :return ind_match1: Indices of second list that correspond to each given
    item of list 1 if exists (-1 otherwise)
    :return ind_match2: Indices of first list that correspond to each given
    item of list 2 if exists (-1) otherwise
    """
    if name_list1 is None or name_list2 is None:
        return None, None, None, None
    init_match1 = [''] * len(name_list1)
    init_match2 = [''] * len(name_list2)
    ind_match1 = [-1] * len(name_list1)
    ind_match2 = [-1] * len(name_list2)
    flatten_list1 = [item for sublist in name_list1 for item in sublist]
    flatten_list2 = [item for sublist in name_list2 for item in sublist]
    indflat_1 = [i for i in range(0, len(init_match1)) for item in
                 name_list1[i] if init_match1[i] == '']
    indflat_2 = [i for i in range(0, len(init_match2)) for item in
                 name_list2[i] if init_match2[i] == '']
    for i in range(0, len(name_list1)):
        for name in name_list1[i]:
            if name in flatten_list2:
                init_match1[i] = name
                ind_match1[i] = indflat_2[flatten_list2.index(name)]
                break
    for i in range(0, len(name_list2)):
        for name in name_list2[i]:
            if name in flatten_list1:
                init_match2[i] = name
                ind_match2[i] = indflat_1[flatten_list1.index(name)]
                break
    return init_match1, init_match2, ind_match1, ind_match2

def __find_max_overlap_in_list(name, list_names):
    """
    Given a name and list of names to match it to, find the maximum overlap
    existing

    :param name: string to match to any of list_names
    :param list_names: list of candidate strings
    :return match_seq: matched substring
    :return index: index of element in list_names to which the match is
    associated. Returns -1 if there is no found match
    """
    match_max = 0
    match_seq = ''
    match_orig = ''
    match_ratio = 0
    if not list_names:
        return '', -1
    for test in list_names:
        if test:
            match = SequenceMatcher(None, name, test).find_longest_match(
                0, len(name), 0, len(test))
            if match.size >= match_max and match.size / len(test) >= \
                    match_ratio:
                match_max = match.size
                match_seq = test[match.b:(match.b + match.size)]
                match_ratio = match.size / len(test)
                match_orig = test
    if match_max == 0:
        return '', -1
    other_list = [name for name in list_names if match_seq in name and
                  match_max / len(name) == match_ratio]
    if len(other_list) > 1:
        return '', -1
    return match_seq, list_names.index(match_orig)

def match_second_degree(name_list1, name_list2):
    """
    Perform the double matching between two lists of
    possible names.
    First find the direct matches, remove them from
    the ones still to match and
    match the remaining ones using the maximum overlap.
    Returns the name
    match for each list, and the index correspondences.

    More subtle matching with first direct matching and then secondary
    overlap matching between list of list of potential names
    :param name_list1:
    :param name_list2:
    :return init_match1:
    :return ind_match1: Index of corresponding match in name_list2
    :return init_match2: Matching string in list2
    :return ind_match2: Index of corresponding match in name_list1
    """
    if name_list1 is None or name_list2 is None:
        return None, None, None, None
    init_match1, init_match2, ind_match1, ind_match2 = match_first_degree(
        name_list1, name_list2)
    reduced_list1 = [names for names in name_list1
                     if init_match1[name_list1.index(names)] == '']
    reduced_list2 = [names for names in name_list2
                     if init_match2[name_list2.index(names)] == '']
    redflat_1 = [item for sublist in reduced_list1 for item in sublist]
    indflat_1 = [i for i in range(0, len(init_match1)) for item in
                 name_list1[i] if init_match1[i] == '']
    redflat_2 = [item for sublist in reduced_list2 for item in sublist]
    indflat_2 = [i for i in range(0, len(init_match2)) for item in
                 name_list2[i] if init_match2[i] == '']
    for i in range(0, len(name_list1)):
        if init_match1[i] == '':
            for n in name_list1[i]:
                init_match1[i], index = __find_max_overlap_in_list(n, redflat_2)
                if index >= 0:
                    ind_match1[i] = indflat_2[index]
    for i in range(0, len(name_list2)):
        if init_match2[i] == '':
            for n in name_list2[i]:
                init_match2[i], index = __find_max_overlap_in_list(n, redflat_1)
                if index >= 0:
                    ind_match2[i] = indflat_1[index]
    return init_match1, ind_match1, init_match2, ind_match2

# From a list of list of names and a list of list of files that are
# associated, find the name correspondance and therefore the files associations
def join_subject_id_and_filename_list(name_list, list_files):
    """
    From the list of list of names and the list of list of files
    corresponding to each constraint find the association between a single
    name id and the different file lists
    :param name_list: list of list of names
    :param list_files: list of list of files (one list per constraint)
    :return list_combined: List per subject of name and list of files given
    by the constraints
    """
    ind_max = np.argmax([len(names) for names in name_list])
    name_max = name_list[ind_max]
    name_tot = []
    ind_tot = []
    name_max_to_use = []
    for c in range(0, len(list_files)):
        name_match, ind_match, _, _ = match_second_degree(name_max,
                                                          name_list[c])
        if c == ind_max:
            name_max_to_use = name_match
        name_tot.append(name_match)
        ind_tot.append(ind_match)

    list_combined = []
    for (i, name) in enumerate(name_max_to_use):
        list_temp = [name]
        'To do : Taking care of the case when the list of a constraint is ' \
        'completely empty'
        for c in range(0, len(list_files)):
            output = list_files[c][ind_tot[c][i]] if ind_tot[c][i] > -1 else ''
            list_temp.append(output)
        list_combined.append(list_temp)
    return list_combined

def remove_duplicated_names(name_list):
    """
    From a list of list of names remove the items that are duplicated
    :param name_list: list of list of names to investigate
    :return duplicates_removed: list of list of names freed of duplicates
    """
    flattened_list = [item for sublist in name_list for item in sublist]
    list_duplicated = [item for item in flattened_list
                       if flattened_list.count(item) > 1]
    duplicates_removed = []
    for names in name_list:
        duplicates_removed.append([name for name in names
                                   if name not in list_duplicated])
    return duplicates_removed

def match_and_write_filenames_to_csv(list_constraints, csv_file):
    """
    Combine all elements of file searching until finally writing the names
    :param list_constraints: list of constraints (defined by list of paths to
    search, list of elements the filename should contain and of those that
    are forbidden
    :param csv_file: file on which to write the final list of files.
    :return:
    """
    name_tot = []
    list_tot = []
    if list_constraints is None or len(list_constraints) == 0:
        return
    for c in list_constraints:
        list_files, name_list = \
            KeywordsMatching.matching_subjects_and_filenames(c)
        name_list = remove_duplicated_names(name_list)
        name_tot.append(name_list)
        list_tot.append(list_files)
    list_combined = join_subject_id_and_filename_list(name_tot, list_tot)
    touch_folder(os.path.dirname(csv_file))

    # csv writer has different behaviour in python 2/3
    if sys.version_info[0] >= 3:
        with open(csv_file, 'w', newline='', encoding='utf8') as csvfile:
            file_writer = csv.writer(csvfile)
            for list_temp in list_combined:
                file_writer.writerow(list_temp)
    else:
        with open(csv_file, 'wb') as csvfile:
            file_writer = csv.writer(csvfile, delimiter=',')
            for list_temp in list_combined:
                file_writer.writerow(list_temp)
    return

def load_and_merge_csv_files(data_param):
    """
    Converts a list of csv_files in data_param
    in to a joint list of file names (by matching the first column)
    This function returns a <pandas.core.frame.DataFrame> of the
    joint list
    """
    if not data_param:
        tf.logging.fatal('nothing to load, please check reader.names')
        raise ValueError
    _file_list = None
    for modality_name in data_param:
        try:
            csv_file = data_param[modality_name].csv_file
        except AttributeError:
            tf.logging.fatal('unrecognised parameter format')
            raise
        if hasattr(data_param[modality_name], 'path_to_search') and \
                len(data_param[modality_name].path_to_search):
            tf.logging.info(
                '[%s] search file folders, writing csv file %s',
                modality_name, csv_file)
            section_tuple = data_param[modality_name].__dict__.items()
            matcher = KeywordsMatching.from_tuple(section_tuple)
            match_and_write_filenames_to_csv([matcher], csv_file)
        else:
            tf.logging.info(
                '[%s] using existing csv file %s, skipped folder search',
                modality_name, csv_file)
        if not os.path.isfile(csv_file):
            tf.logging.fatal(
                "[%s] csv file %s not found.", modality_name, csv_file)
            raise IOError
        csv_list = pandas.read_csv(
            csv_file, header=None, names=['subject_id', modality_name])
        if _file_list is None:
            _file_list = csv_list
            continue

        # merge _file_list based on subject_ids (first column of each csv)
        n_rows = _file_list.shape[0]
        _file_list = pandas.merge(_file_list, csv_list, on='subject_id')
        if _file_list.shape[0] != n_rows:
            tf.logging.warning("rows not matched in %s", csv_file)

    if _file_list is None or _file_list.size == 0:
        tf.logging.fatal(
            "empty filename lists, please check the csv "
            "files. (remove csv files to automatically"
            "search folders and generate new csv files again)")
        raise IOError
    return _file_list
<|endoftext|>


<commit_before># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for all nameserver related activity."""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

#if __name__ == '__main__':
#  sys.path.append('..')

import re
import socket
import sys
import time

# external dependencies (from nb_third_party)
import dns.exception
import dns.message
import dns.name
import dns.query
import dns.rcode
import dns.rdataclass
import dns.rdatatype
import dns.resolver
import dns.reversename
import dns.version

import health_checks
import util

# Look for buggy system versions of namebench
if dns.version.hexversion < 17301744:
  raise ValueError('dnspython 1.8.0+ required, while only %s was found. The '
                   'namebench source bundles 1.8.0, so use it.' % dns.version.version)

# Pick the most accurate timer for a platform. Stolen from timeit.py:
if sys.platform[:3] == 'win':
  DEFAULT_TIMER = time.clock
else:
  DEFAULT_TIMER = time.time

# How many failures before we disable system nameservers
MAX_NORMAL_FAILURES = 2
MAX_SYSTEM_FAILURES = 7
MAX_PREFERRED_FAILURES = 5
MAX_WARNINGS = 7

FAILURE_PRONE_RATE = 10

def ResponseToAscii(response):
  if not response:
    return None
  if response.answer:
    answers = [', '.join(map(str, x.items)) for x in response.answer]
    return ' -> '.join(answers).rstrip('"').lstrip('"')
  else:
    return dns.rcode.to_text(response.rcode())

class NameServer(health_checks.NameServerHealthChecks):
  """Hold information about a particular nameserver."""

  def __init__(self, ip, name=None, internal=False, preferred=False):
    self.name = name
    # We use _ for IPV6 representation in our configuration due to ConfigParser issues.
    self.ip = ip.replace('_', ':')
    self.is_system = internal
    self.is_regional = False
    self.is_global = False
    self.is_custom = False
    self.system_position = None
    self.is_preferred = preferred
    self.timeout = 5
    self.health_timeout = 5
    self.ping_timeout = 1
    self.ResetTestStatus()
    self.port_behavior = None
    self._version = None
    self._node_ids = set()
    self._hostname = None
    self.timer = DEFAULT_TIMER

    if ':' in self.ip:
      self.is_ipv6 = True
    else:
      self.is_ipv6 = False

  @property
  def check_average(self):
    # If we only have a ping result, sort by it. Otherwise, use all non-ping results.
    if len(self.checks) == 1:
      return self.checks[0][3]
    else:
      return util.CalculateListAverage([x[3] for x in self.checks[1:]])

  @property
  def fastest_check_duration(self):
    if self.checks:
      return min([x[3] for x in self.checks])
    else:
      return 0.0

  @property
  def slowest_check_duration(self):
    if self.checks:
      return max([x[3] for x in self.checks])
    else:
      return None

  @property
  def check_duration(self):
    return sum([x[3] for x in self.checks])

  @property
  def warnings_string(self):
    if self.disabled:
      return '(excluded: %s)' % self.disabled
    else:
      return ', '.join(map(str, self.warnings))

  @property
  def warnings_comment(self):
    if self.warnings or self.disabled:
      return '# ' + self.warnings_string
    else:
      return ''

  @property
  def errors(self):
    return ['%s (%s requests)' % (_[0], _[1]) for _ in self.error_map.items() if _[0] != 'Timeout']

  @property
  def error_count(self):
    return sum([_[1] for _ in self.error_map.items() if _[0] != 'Timeout'])

  @property
  def timeout_count(self):
    return self.error_map.get('Timeout', 0)

  @property
  def notes(self):
    """Return a list of notes about this nameserver object."""
    my_notes = []
    if self.system_position == 0:
      my_notes.append('The current preferred DNS server.')
    elif self.system_position:
      my_notes.append('A backup DNS server for this system.')
    if self.is_failure_prone:
      my_notes.append('%0.0f queries to this host failed' % self.failure_rate)
    if self.port_behavior and 'POOR' in self.port_behavior:
      my_notes.append('Vulnerable to poisoning attacks (poor port diversity)')
    if self.disabled:
      my_notes.append(self.disabled)
    else:
      my_notes.extend(self.warnings)
    if self.errors:
      my_notes.extend(self.errors)
    return my_notes

  @property
  def hostname(self):
    if self._hostname is None:
      self._hostname = self.RequestReverseIP(self.ip)
    return self._hostname

  @property
  def version(self):
    if self._version is None:
      self.RequestVersion()
    return self._version

  @property
  def node_ids(self):
    """Return a set of node_ids seen on this system."""
    # We use a slightly different pattern here because we want to
    # append to our results each time this is called.
    self.RequestNodeId()
    # Only return non-blank entries
    return [x for x in self._node_ids if x]

  @property
  def partial_node_ids(self):
    partials = []
    for node_id in self._node_ids:
      node_bits = node_id.split('.')
      if len(node_bits) >= 3:
        partials.append('.'.join(node_bits[0:-2]))
      else:
        partials.append('.'.join(node_bits))
    return partials

  @property
  def name_and_node(self):
    if self.node_ids:
      return '%s [%s]' % (self.name, ', '.join(self.partial_node_ids))
    else:
      return self.name

  @property
  def is_failure_prone(self):
    if self.failure_rate >= FAILURE_PRONE_RATE:
      return True
    else:
      return False

  @property
  def failure_rate(self):
    if not self.failure_count or not self.request_count:
      return 0
    else:
      return (float(self.failure_count) / float(self.request_count)) * 100

  def __str__(self):
    return '%s [%s]' % (self.name, self.ip)

  def __repr__(self):
    return self.__str__()

  def ResetTestStatus(self):
    """Reset testing status of this host."""
    self.warnings = set()
    self.shared_with = set()
    self.disabled = False
    self.checks = []
    self.failed_test_count = 0
    self.share_check_count = 0
    self.cache_checks = []
    self.is_slower_replica = False
    self.ResetErrorCounts()

  def ResetErrorCounts(self):
    """NOTE: This gets called by benchmark.Run()!"""

    self.request_count = 0
    self.failure_count = 0
    self.error_map = {}

  def AddFailure(self, message, fatal=False):
    """Add a failure for this nameserver. This will effectively disable it's use."""
    if self.is_system:
      max_count = MAX_SYSTEM_FAILURES
    elif self.is_preferred:
      max_count = MAX_PREFERRED_FAILURES
    else:
      max_count = MAX_NORMAL_FAILURES

    self.failed_test_count += 1

    if self.is_system or self.is_preferred:
      # If the preferred host is IPv6 and we have no previous checks, fail quietly.
      if self.is_ipv6 and len(self.checks) <= 1:
        self.disabled = message
      else:
        print "\n* %s failed test #%s/%s: %s" % (self, self.failed_test_count, max_count, message)

    # Fatal doesn't count for system & preferred nameservers.
    if fatal and not (self.is_system or self.is_preferred):
      self.disabled = message
    elif self.failed_test_count >= max_count:
      self.disabled = "Failed %s tests, last: %s" % (self.failed_test_count, message)

  def AddWarning(self, message):
    self.warnings.add(message)
    if len(self.warnings) >= MAX_WARNINGS:
      self.AddFailure('Too many warnings (%s), probably broken.' % len(self.warnings), fatal=True)

  def CreateRequest(self, record, request_type, return_type):
    """Function to work around any dnspython make_query quirks."""
    return dns.message.make_query(record, request_type, return_type)

  def Query(self, request, timeout):
    return dns.query.udp(request, self.ip, timeout, 53)

  def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None):
    """Make a DNS request, returning the reply and duration it took.

    Args:
      type_string: DNS record type to query (string)
      record_string: DNS record name to query (string)
      timeout: optional timeout (float)
      rdataclass: optional result class (defaults to rdataclass.IN)

    Returns:
      A tuple of (response, duration in ms [float], error_msg)

    In the case of a DNS response timeout, the response object will be None.
    """
    if not rdataclass:
      rdataclass = dns.rdataclass.IN
    else:
      rdataclass = dns.rdataclass.from_text(rdataclass)

    request_type = dns.rdatatype.from_text(type_string)
    record = dns.name.from_text(record_string, None)
    request = None
    self.request_count += 1

    # Sometimes it takes great effort just to craft a UDP packet.
    try:
      request = self.CreateRequest(record, request_type, rdataclass)
    except ValueError, exc:
      if not request:
        return (None, 0, util.GetLastExceptionString())

    if not timeout:
      timeout = self.timeout

    error_msg = None
    exc = None
    duration = None
    try:
      start_time = self.timer()
      response = self.Query(request, timeout)
      duration = self.timer() - start_time
    except (dns.exception.Timeout), exc:
      response = None
    except (dns.query.BadResponse, dns.message.TrailingJunk,
            dns.query.UnexpectedSource), exc:
      error_msg = util.GetLastExceptionString()
      response = None
    # This is pretty normal if someone runs namebench offline.
    except socket.error:
      response = None
      if ':' in self.ip:
        error_msg = 'socket error: IPv6 may not be available.'
      else:
        error_msg = util.GetLastExceptionString()
    # Pass these exceptions up the food chain
    except (KeyboardInterrupt, SystemExit, SystemError), exc:
      raise exc
    except:
      error_msg = util.GetLastExceptionString()
      print "* Unusual error with %s:%s on %s: %s" % (type_string, record_string, self, error_msg)
      response = None

    if not response:
      self.failure_count += 1

    if not duration:
      duration = self.timer() - start_time

    if exc and not error_msg:
      error_msg = '%s: %s' % (record_string, util.GetLastExceptionString())

    if error_msg:
      key = util.GetLastExceptionString()
      self.error_map[key] = self.error_map.setdefault(key, 0) + 1

    return (response, util.SecondsToMilliseconds(duration), error_msg)

  def RequestVersion(self):
    version = ''
    (response, duration, error_msg) = self.TimedRequest('TXT', 'version.bind.', rdataclass='CHAOS',
                                                        timeout=self.health_timeout*2)
    if response and response.answer:
      response_string = ResponseToAscii(response)
      if (re.search('\d', response_string) or
          (re.search('recursive|ns|server|bind|unbound', response_string, re.I)
           and 'ontact' not in response_string and '...' not in response_string)):
        version = response_string
    self._version = version
    return (version, duration, error_msg)

  def RequestReverseIP(self, ip):
    """Request a hostname for a given IP address."""
    try:
      answer = dns.resolver.query(dns.reversename.from_address(ip), 'PTR')
    except:
      answer = None
    if answer:
      return answer[0].to_text().rstrip('.')
    else:
      return ip

  def RequestNodeId(self):
    """Try to determine the node id for this nameserver (tries many methods)."""
    node = ''
    rdataclass = None
    reverse_lookup = False

    if self.hostname.endswith('ultradns.net') or self.ip.startswith('156.154.7'):
      query_type, record_name = ('A', 'whoareyou.ultradns.net.')
      reverse_lookup = True
    elif self.ip.startswith('8.8'):
      query_type, record_name = ('TXT', 'o-o.myaddr.google.com.')
      reverse_lookup = True
    elif self.hostname.endswith('opendns.com') or self.ip.startswith('208.67.22'):
      query_type, record_name = ('TXT', 'which.opendns.com.')
    else:
      query_type, record_name, rdataclass = ('TXT', 'hostname.bind.', 'CHAOS')

    (response, duration, error_msg) = self.TimedRequest(query_type, record_name, rdataclass=rdataclass,
                                                        timeout=self.health_timeout*2)
    if not response or not response.answer:
      query_type, record_name, rdataclass = ('TXT', 'id.server.', 'CHAOS')
      (response, duration, error_msg) = self.TimedRequest(query_type, record_name, rdataclass=rdataclass,
                                                          timeout=self.health_timeout*2)

    if response and response.answer:
      node = ResponseToAscii(response)
      if reverse_lookup:
        node = self.RequestReverseIP(node)

    # This is what the .node* properties use.
    self._node_ids.add(node)
    return (node, duration, error_msg)

if __name__ == '__main__':
  ns = NameServer(sys.argv[1])
  print "-" * 64
  print "IP:      %s" % ns.ip
  print "Host:    %s" % ns.hostname
  print "Version: %s" % ns.version
  print "Node:    %s" % ns.node_ids
<commit_msg>Prevent non-string warnings from being added. Update GDNS node id detection<commit_after># Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module for all nameserver related activity."""

__author__ = 'tstromberg@google.com (Thomas Stromberg)'

#if __name__ == '__main__':
#  sys.path.append('..')

import re
import socket
import sys
import time

# external dependencies (from nb_third_party)
import dns.exception
import dns.message
import dns.name
import dns.query
import dns.rcode
import dns.rdataclass
import dns.rdatatype
import dns.resolver
import dns.reversename
import dns.version

import health_checks
import util

# Look for buggy system versions of namebench
if dns.version.hexversion < 17301744:
  raise ValueError('dnspython 1.8.0+ required, while only %s was found. The '
                   'namebench source bundles 1.8.0, so use it.' % dns.version.version)

# Pick the most accurate timer for a platform. Stolen from timeit.py:
if sys.platform[:3] == 'win':
  DEFAULT_TIMER = time.clock
else:
  DEFAULT_TIMER = time.time

# How many failures before we disable system nameservers
MAX_NORMAL_FAILURES = 2
MAX_SYSTEM_FAILURES = 7
MAX_PREFERRED_FAILURES = 5
MAX_WARNINGS = 7

FAILURE_PRONE_RATE = 10

def ResponseToAscii(response):
  if not response:
    return None
  if response.answer:
    answers = [', '.join(map(str, x.items)) for x in response.answer]
    return ' -> '.join(answers).rstrip('"').lstrip('"')
  else:
    return dns.rcode.to_text(response.rcode())

class NameServer(health_checks.NameServerHealthChecks):
  """Hold information about a particular nameserver."""

  def __init__(self, ip, name=None, internal=False, preferred=False):
    self.name = name
    # We use _ for IPV6 representation in our configuration due to ConfigParser issues.
    self.ip = ip.replace('_', ':')
    self.is_system = internal
    self.is_regional = False
    self.is_global = False
    self.is_custom = False
    self.system_position = None
    self.is_preferred = preferred
    self.timeout = 5
    self.health_timeout = 5
    self.ping_timeout = 1
    self.ResetTestStatus()
    self.port_behavior = None
    self._version = None
    self._node_ids = set()
    self._hostname = None
    self.timer = DEFAULT_TIMER

    if ':' in self.ip:
      self.is_ipv6 = True
    else:
      self.is_ipv6 = False

  @property
  def check_average(self):
    # If we only have a ping result, sort by it. Otherwise, use all non-ping results.
    if len(self.checks) == 1:
      return self.checks[0][3]
    else:
      return util.CalculateListAverage([x[3] for x in self.checks[1:]])

  @property
  def fastest_check_duration(self):
    if self.checks:
      return min([x[3] for x in self.checks])
    else:
      return 0.0

  @property
  def slowest_check_duration(self):
    if self.checks:
      return max([x[3] for x in self.checks])
    else:
      return None

  @property
  def check_duration(self):
    return sum([x[3] for x in self.checks])

  @property
  def warnings_string(self):
    if self.disabled:
      return '(excluded: %s)' % self.disabled
    else:
      return ', '.join(map(str, self.warnings))

  @property
  def warnings_comment(self):
    if self.warnings or self.disabled:
      return '# ' + self.warnings_string
    else:
      return ''

  @property
  def errors(self):
    return ['%s (%s requests)' % (_[0], _[1]) for _ in self.error_map.items() if _[0] != 'Timeout']

  @property
  def error_count(self):
    return sum([_[1] for _ in self.error_map.items() if _[0] != 'Timeout'])

  @property
  def timeout_count(self):
    return self.error_map.get('Timeout', 0)

  @property
  def notes(self):
    """Return a list of notes about this nameserver object."""
    my_notes = []
    if self.system_position == 0:
      my_notes.append('The current preferred DNS server.')
    elif self.system_position:
      my_notes.append('A backup DNS server for this system.')
    if self.is_failure_prone:
      my_notes.append('%0.0f queries to this host failed' % self.failure_rate)
    if self.port_behavior and 'POOR' in self.port_behavior:
      my_notes.append('Vulnerable to poisoning attacks (poor port diversity)')
    if self.disabled:
      my_notes.append(self.disabled)
    else:
      my_notes.extend(self.warnings)
    if self.errors:
      my_notes.extend(self.errors)
    return my_notes

  @property
  def hostname(self):
    if self._hostname is None:
      self._hostname = self.RequestReverseIP(self.ip)
    return self._hostname

  @property
  def version(self):
    if self._version is None:
      self.RequestVersion()
    return self._version

  @property
  def node_ids(self):
    """Return a set of node_ids seen on this system."""
    # We use a slightly different pattern here because we want to
    # append to our results each time this is called.
    self.RequestNodeId()
    # Only return non-blank entries
    return [x for x in self._node_ids if x]

  @property
  def partial_node_ids(self):
    partials = []
    for node_id in self._node_ids:
      node_bits = node_id.split('.')
      if len(node_bits) >= 3:
        partials.append('.'.join(node_bits[0:-2]))
      else:
        partials.append('.'.join(node_bits))
    return partials

  @property
  def name_and_node(self):
    if self.node_ids:
      return '%s [%s]' % (self.name, ', '.join(self.partial_node_ids))
    else:
      return self.name

  @property
  def is_failure_prone(self):
    if self.failure_rate >= FAILURE_PRONE_RATE:
      return True
    else:
      return False

  @property
  def failure_rate(self):
    if not self.failure_count or not self.request_count:
      return 0
    else:
      return (float(self.failure_count) / float(self.request_count)) * 100

  def __str__(self):
    return '%s [%s]' % (self.name, self.ip)

  def __repr__(self):
    return self.__str__()

  def ResetTestStatus(self):
    """Reset testing status of this host."""
    self.warnings = set()
    self.shared_with = set()
    self.disabled = False
    self.checks = []
    self.failed_test_count = 0
    self.share_check_count = 0
    self.cache_checks = []
    self.is_slower_replica = False
    self.ResetErrorCounts()

  def ResetErrorCounts(self):
    """NOTE: This gets called by benchmark.Run()!"""

    self.request_count = 0
    self.failure_count = 0
    self.error_map = {}

  def AddFailure(self, message, fatal=False):
    """Add a failure for this nameserver. This will effectively disable it's use."""
    if self.is_system:
      max_count = MAX_SYSTEM_FAILURES
    elif self.is_preferred:
      max_count = MAX_PREFERRED_FAILURES
    else:
      max_count = MAX_NORMAL_FAILURES

    self.failed_test_count += 1

    if self.is_system or self.is_preferred:
      # If the preferred host is IPv6 and we have no previous checks, fail quietly.
      if self.is_ipv6 and len(self.checks) <= 1:
        self.disabled = message
      else:
        print "\n* %s failed test #%s/%s: %s" % (self, self.failed_test_count, max_count, message)

    # Fatal doesn't count for system & preferred nameservers.
    if fatal and not (self.is_system or self.is_preferred):
      self.disabled = message
    elif self.failed_test_count >= max_count:
      self.disabled = "Failed %s tests, last: %s" % (self.failed_test_count, message)

  def AddWarning(self, message):
    if not isinstance(message, str):
      print "Tried to add %s to %s (not a string)" % (message, self)
      return None

    self.warnings.add(message)
    if len(self.warnings) >= MAX_WARNINGS:
      self.AddFailure('Too many warnings (%s), probably broken.' % len(self.warnings), fatal=True)

  def CreateRequest(self, record, request_type, return_type):
    """Function to work around any dnspython make_query quirks."""
    return dns.message.make_query(record, request_type, return_type)

  def Query(self, request, timeout):
    return dns.query.udp(request, self.ip, timeout, 53)

  def TimedRequest(self, type_string, record_string, timeout=None, rdataclass=None):
    """Make a DNS request, returning the reply and duration it took.

    Args:
      type_string: DNS record type to query (string)
      record_string: DNS record name to query (string)
      timeout: optional timeout (float)
      rdataclass: optional result class (defaults to rdataclass.IN)

    Returns:
      A tuple of (response, duration in ms [float], error_msg)

    In the case of a DNS response timeout, the response object will be None.
    """
    if not rdataclass:
      rdataclass = dns.rdataclass.IN
    else:
      rdataclass = dns.rdataclass.from_text(rdataclass)

    request_type = dns.rdatatype.from_text(type_string)
    record = dns.name.from_text(record_string, None)
    request = None
    self.request_count += 1

    # Sometimes it takes great effort just to craft a UDP packet.
    try:
      request = self.CreateRequest(record, request_type, rdataclass)
    except ValueError, exc:
      if not request:
        return (None, 0, util.GetLastExceptionString())

    if not timeout:
      timeout = self.timeout

    error_msg = None
    exc = None
    duration = None
    try:
      start_time = self.timer()
      response = self.Query(request, timeout)
      duration = self.timer() - start_time
    except (dns.exception.Timeout), exc:
      response = None
    except (dns.query.BadResponse, dns.message.TrailingJunk,
            dns.query.UnexpectedSource), exc:
      error_msg = util.GetLastExceptionString()
      response = None
    # This is pretty normal if someone runs namebench offline.
    except socket.error:
      response = None
      if ':' in self.ip:
        error_msg = 'socket error: IPv6 may not be available.'
      else:
        error_msg = util.GetLastExceptionString()
    # Pass these exceptions up the food chain
    except (KeyboardInterrupt, SystemExit, SystemError), exc:
      raise exc
    except:
      error_msg = util.GetLastExceptionString()
      print "* Unusual error with %s:%s on %s: %s" % (type_string, record_string, self, error_msg)
      response = None

    if not response:
      self.failure_count += 1

    if not duration:
      duration = self.timer() - start_time

    if exc and not error_msg:
      error_msg = '%s: %s' % (record_string, util.GetLastExceptionString())

    if error_msg:
      key = util.GetLastExceptionString()
      self.error_map[key] = self.error_map.setdefault(key, 0) + 1

    return (response, util.SecondsToMilliseconds(duration), error_msg)

  def RequestVersion(self):
    version = ''
    (response, duration, error_msg) = self.TimedRequest('TXT', 'version.bind.', rdataclass='CHAOS',
                                                        timeout=self.health_timeout*2)
    if response and response.answer:
      response_string = ResponseToAscii(response)
      if (re.search('\d', response_string) or
          (re.search('recursive|ns|server|bind|unbound', response_string, re.I)
           and 'ontact' not in response_string and '...' not in response_string)):
        version = response_string
    self._version = version
    return (version, duration, error_msg)

  def RequestReverseIP(self, ip):
    """Request a hostname for a given IP address."""
    try:
      answer = dns.resolver.query(dns.reversename.from_address(ip), 'PTR')
    except:
      answer = None
    if answer:
      return answer[0].to_text().rstrip('.')
    else:
      return ip

  def RequestNodeId(self):
    """Try to determine the node id for this nameserver (tries many methods)."""
    node = ''
    rdataclass = None
    reverse_lookup = False

    if self.hostname.endswith('ultradns.net') or self.ip.startswith('156.154.7'):
      query_type, record_name = ('A', 'whoareyou.ultradns.net.')
      reverse_lookup = True
    elif self.ip.startswith('8.8'):
      query_type, record_name = ('A', 'self.myresolver.info.')
      reverse_lookup = True
    elif self.hostname.endswith('opendns.com') or self.ip.startswith('208.67.22'):
      query_type, record_name = ('TXT', 'which.opendns.com.')
    else:
      query_type, record_name, rdataclass = ('TXT', 'hostname.bind.', 'CHAOS')

    (response, duration, error_msg) = self.TimedRequest(query_type, record_name, rdataclass=rdataclass,
                                                        timeout=self.health_timeout*2)
    if not response or not response.answer:
      query_type, record_name, rdataclass = ('TXT', 'id.server.', 'CHAOS')
      (response, duration, error_msg) = self.TimedRequest(query_type, record_name, rdataclass=rdataclass,
                                                          timeout=self.health_timeout*2)

    if response and response.answer:
      node = ResponseToAscii(response)
      if reverse_lookup:
        node = self.RequestReverseIP(node)

    # This is what the .node* properties use.
    self._node_ids.add(node)
    return (node, duration, error_msg)

if __name__ == '__main__':
  ns = NameServer(sys.argv[1])
  print "-" * 64
  print "IP:      %s" % ns.ip
  print "Host:    %s" % ns.hostname
  print "Version: %s" % ns.version
  print "Node:    %s" % ns.node_ids
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
# ODOO, Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<http://www.micronaet.it>)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)
from utility import *

_logger = logging.getLogger(__name__)

class PurchaseOrderSql(orm.Model):
    '''Update basic object for import accounting elements
       NOTE: in this procedure there's some fields that are created
             from another module, TODO correct for keep module 'modular'
    '''
    _inherit = 'micronaet.accounting'

    # get_of_lines:
    def get_of_header(self, cr, uid, context=None):
        ''' Return OF_TESTATE
        '''
        table = 'of_testate'
        if self.pool.get('res.company').table_capital_name(
                cr, uid, context=context):
            table = table.upper()

        cursor = self.connect(cr, uid, context=context)

        try:
            cursor.execute('''SELECT * FROM %s;''' % table)
            return cursor
        except:
            _logger.error('Executing query %s: [%s]' % (
                table,
                sys.exc_info(), ))
            return False

    def get_of_line(self, cr, uid, with_desc=False, context=None):
        ''' Return quantity element for product
            Table: OF_RIGHE
            cr: database cursor
            uid: user ID
            with_desc: load also description line
            context: context for parameters
        '''
        table = 'of_righe'
        if self.pool.get('res.company').table_capital_name(
                cr, uid, context=context):
            table = table.upper()

        cursor = self.connect(cr, uid, context=context)
        where = ''
        if not with_desc:
            where = ' WHERE IST_RIGA != \'D\''

        try:
            cursor.execute('''
                SELECT * FROM %s%s;''' % (table, where))
            return cursor # with the query setted up
        except:
            _logger.error('Executing query %s: [%s]' % (
                table,
                sys.exc_info(), ))
            return False

class PurchaseOrderSql(orm.Model):
    '''Update basic object for import accounting elements
       NOTE: in this procedure there's some fields that are created
             from another module, TODO correct for keep module 'modular'
    '''
    _inherit = 'purchase.order'

    # -----------------
    # Utility function:
    # -----------------
    def load_converter(self, cr, uid, carriage_condition, transportation_reason,
            payment, context=None):
        ''' Load useful converters
        '''

        # Create a converter for carriage condition:
        cc_pool = self.pool.get('stock.picking.carriage_condition')
        cc_ids = cc_pool.search(cr, uid, [], context=context)
        for item in cc_pool.browse(cr, uid, cc_ids, context=context):
            if item.account_code:
                carriage_condition[item.account_code] = item.id

        # Create a converter for transportation reason:
        tr_pool = self.pool.get('stock.picking.transportation_reason')
        tr_ids = tr_pool.search(cr, uid, [], context=context)
        for item in tr_pool.browse(cr, uid, tr_ids, context=context):
            if item.import_id:
                transportation_reason[item.import_id] = item.id

        # Create a converter for payment:
        payment_pool = self.pool.get('account.payment.term')
        payment_ids = payment_pool.search(cr, uid, [], context=context)
        for item in payment_pool.browse(cr, uid, payment_ids, context=context):
            if item.import_id:
                payment[item.import_id] = item.id
        return

    def get_uom(self, cr, uid, name, context=None):
        uom_id = self.pool.get('product.uom').search(cr, uid, [
            ('name', '=', name), ])
        if uom_id:
            return uom_id[0] # take the first
        else:
            return False

    # -------------------------------------------------------------------------
    #                           Button event:
    # -------------------------------------------------------------------------
    def schedule_etl_purchase_order(self, cr, uid, context=None):
        ''' Button event for import all order (purchase)
        '''
        _logger.info('Start update extra info OF header')
        query_pool = self.pool.get('micronaet.accounting')
        partner_pool = self.pool.get('res.partner')
        location_pool = self.pool.get('stock.location')

        cr_of = query_pool.get_of_header(cr, uid, context=context)
        if not cr_of:
            _logger.error('Cannot connect to MSSQL OF_TESTATE')
            return

        # -----------
        # Converters:
        # -----------
        orders = {}
        carriage_condition = {}
        transportation_reason = {}
        payment = {}
        self.load_converter(cr, uid, carriage_condition,
            transportation_reason, payment, context=context)

        location_ids = location_pool.search(cr, uid, [
            ('name', '=', 'Stock')], context=context)
        if not location_ids:
            _logger.error('Cannot found location: Stock')
            return

        pricelist_id = 2 # TODO

        # ---------------------------------------------------------------------
        #                           HEADER:
        # ---------------------------------------------------------------------
        for of in cr_of:
            try:
                # -------------------------------------------------------------
                #                          Get data:
                # -------------------------------------------------------------
                # Partner:
                partner_code = of['CKY_CNT_CLFR']
                partner_ids = partner_pool.search(cr, uid, [
                    ('sql_supplier_code', '=', partner_code),
                    ], context=context)
                if not partner_ids:
                    _logger.error(
                        'No partner found, code: %s' % partner_code)
                    continue

                # Destination:
                dest_code = of['CKY_CNT_SPED_ALT']
                dest_id = False
                if dest_code:
                    dest_ids = partner_pool.search(cr, uid, ['|',
                        ('sql_supplier_code', '=', dest_code),
                        ('sql_destination_code', '=', dest_code),
                        ], context=context)
                    if dest_ids:
                        dest_id = dest_ids[0]
                    else:
                        _logger.error('Destination not found: %s' % dest_code)

                # Update transport readon: NKY_CAUM
                transportation_reason_code = of['NKY_CAUM']
                transportation_reason_id = False
                if transportation_reason_code:
                    transportation_reason_id = transportation_reason.get(
                        transportation_reason_code, False)
                    if not transportation_reason_id:
                        _logger.error('Tranportation reason not found: %s' % (
                            transportation_reason_code))

                # Update payment
                payment_term_code = of['NKY_PAG']
                payment_term_id = False
                if payment_term_code:
                    payment_term_id = payment.get(
                        payment_term_code, False)
                    if not payment_term_id:
                        _logger.error('Payment code not found: %s' % (
                            payment_term_code))

                # Update porto:
                #carriage_condition_code = of['IST_PORTO']
                #if carriage_condition_code:
                #    carriage_condition_id = carriage_condition.get(
                #        carriage_condition_code, False)
                #    if carriage_condition_id:
                #        data['carriage_condition_id'] = carriage_condition_id
                #        _logger.info('Carriage condition update: %s' % name)
                #    else:
                #        _logger.warning('Carriage condition not found: %s' % (
                #            carriage_condition_code))

                # Update note:
                # TODO
                #carrier_code = of['CKY_CNT_VETT']

                # Parcels:
                #parcels = of['NGB_TOT_COLLI']
                #if parcels:
                #    data['parcels'] = parcels
                #    _logger.info('Update parcels: %s' % name)

                # Key field:
                name = 'MX-%s/%s' % (
                    of['NGL_DOC'], of['DTT_DOC'].strftime('%Y'))

                of_ids = self.search(cr, uid, [
                    ('name', '=', name),
                    #('accounting_order', '=', True), # No more used!
                    ], context=context)

                header = {
                    'name': name,
                    'partner_id': partner_ids[0],
                    'destination_partner_id': dest_id,

                    'date_order': of['DTT_DOC'].strftime(
                        DEFAULT_SERVER_DATE_FORMAT),
                    'payment_term_id': payment_term_id,
                    'transportation_reason_id': transportation_reason_id,
                    'notes': of['CDS_NOTE'],
                    'location_id': location_ids[0],
                    'pricelist_id': pricelist_id,
                    #'goods_description_id':
                    #'transportation_method':
                    #'carriage_condition_id':
                    #'partner_ref':
                    #'currency_id'
                    #'used_bank_id'
                    #'fiscal_position'
                    #'payment_note':
                    #'delivery_note':
                    #'note'TODO???
                    #'incoterm_id'
                    #'minimum_planned_date'
                    }

                if of_ids:
                    _logger.info('Update header: %s' % name)
                    order_id = of_ids[0]
                    self.write(cr, uid, order_id, header, context=context)
                else:
                    _logger.info('Create header: %s' % name)
                    order_id = self.create(cr, uid, header,
                        context=context)

                # Save for line:
                orders[name] = order_id

            except:
                _logger.error('Problem with record: %s > %s'%(
                    of, sys.exc_info()))

        # ---------------------------------------------------------------------
        #                           LINE:
        # ---------------------------------------------------------------------
        _logger.info('Start import OF lines')
        line_pool = self.pool.get('sale.order.line')
        product_pool = self.pool.get('product.product')

        cr_of_line = query_pool.get_of_line(cr, uid, context=context)
        if not cr_of_line:
            _logger.error('Cannot connect to MSSQL OF_RIGHE')
            return

        for of_line in cr_of_line:
            try:
                # -----------------------------
                # Get product browse from code:
                # -----------------------------
                name = 'MX-%s/%s' % (
                    of_line['NGL_DOC'], of_line['DTT_DOC'].strftime('%Y'))
                order_id = orders.get(name, False)
                if not order_id:
                    _logger.error('Order not found: %s' % name)
                    continue

                # Product:
                product_code = oc_line['CKY_ART'].strip()
                product_ids = product_pool.search(cr, uid, [
                    ('default_code', '=', product_code)], context=context)
                if not product_ids:
                    _logger.info(
                        _('No product found (OC line jumped): %s') % (
                            product_code))
                    continue
                product_browse = product_pool.browse(
                    cr, uid, product_ids, context=context)[0]

                date_planned = oc_line['DTT_SCAD'].strftime(
                    DEFAULT_SERVER_DATE_FORMAT)

                # NOTE ID of line in OC (not sequence=order)
                sequence = oc_line['NPR_RIGA']
                uom_id = product_browse.uom_id.id if product_browse else False
                conversion = (
                    oc_line['NCF_CONV'] if oc_line['NCF_CONV'] else 1.0)

                # pack * unit item * conversion
                quantity = (oc_line['NGB_COLLI'] or 1.0) * (
                    oc_line['NQT_RIGA_O_PLOR'] or 0.0) * 1.0 / conversion

                # HEADER: Save deadline in OC (only first time):
                #if not oc_header[oc_key][1]:
                #    # take the first deadline and save in header
                #    if date_deadline:
                #        oc_header[oc_key][1] = True
                #        mod = self.write(cr, uid, order_id, {
                #            'date_deadline': date_deadline}, context=context)

                # Discount block:
                #discount = False
                #multi_discount_rates = False
                account_scale = oc_line['CSG_SCN'].strip()
                discount = account_scale or False
                #if account_scale:
                #    try:
                #        res = line_pool.on_change_multi_discount(
                #            cr, uid, False, account_scale,
                #            context=context)['value']
                #        discount = res.get('discount', False)
                #        multi_discount_rates = res.get(
                #            'multi_discount_rates', False)
                #    except:
                #        _logger.error(
                #            'Error calculating discount value: %s' % (
                #                account_scale))
                #        pass

                data = {
                    'product_id': product_browse.id,
                    'product_uom_qty': quantity,
                    'order_id': order_id,
                    'date_planned': date_planned,
                    'product_uom': uom_id,
                    'name': oc_line['CDS_VARIAZ_ART'],
                    'price_unit': (
                        oc_line['NPZ_UNIT'] or 0.0) * conversion,
                    # Correct depend on fiscal position
                    'tax_id': [
                        (6, 0, [product_browse.taxes_id[0].id, ])
                        ] if product_browse and product_browse.taxes_id
                            else False, # CSG_IVA
                    'sequence': sequence,
                    'discount': discount,
                    #'multi_discount_rates': multi_discount_rates,

                    # Related fields ------------------------------------------
                    #'partner_id': order_partner.get(order_id, False),
                    }

            except:
                _logger.error('Problem with oc line record: %s\n%s' % (
                    oc_line, sys.exc_info()))

        # TODO testare bene gli ordini di produzione che potrebbero avere delle mancanze!
        _logger.info('End importation OF header and line!')
        return
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>End of line importation check all order data<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
# ODOO, Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<http://www.micronaet.it>)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)
from utility import *

_logger = logging.getLogger(__name__)

class PurchaseOrderSql(orm.Model):
    '''Update basic object for import accounting elements
       NOTE: in this procedure there's some fields that are created
             from another module, TODO correct for keep module 'modular'
    '''
    _inherit = 'micronaet.accounting'

    # get_of_lines:
    def get_of_header(self, cr, uid, context=None):
        ''' Return OF_TESTATE
        '''
        table = 'of_testate'
        if self.pool.get('res.company').table_capital_name(
                cr, uid, context=context):
            table = table.upper()

        cursor = self.connect(cr, uid, context=context)

        try:
            cursor.execute('''SELECT * FROM %s;''' % table)
            return cursor
        except:
            _logger.error('Executing query %s: [%s]' % (
                table,
                sys.exc_info(), ))
            return False

    def get_of_line(self, cr, uid, with_desc=False, context=None):
        ''' Return quantity element for product
            Table: OF_RIGHE
            cr: database cursor
            uid: user ID
            with_desc: load also description line
            context: context for parameters
        '''
        table = 'of_righe'
        if self.pool.get('res.company').table_capital_name(
                cr, uid, context=context):
            table = table.upper()

        cursor = self.connect(cr, uid, context=context)
        where = ''
        if not with_desc:
            where = ' WHERE IST_RIGA != \'D\''

        try:
            cursor.execute('''
                SELECT * FROM %s%s;''' % (table, where))
            return cursor # with the query setted up
        except:
            _logger.error('Executing query %s: [%s]' % (
                table,
                sys.exc_info(), ))
            return False

class PurchaseOrderSql(orm.Model):
    '''Update basic object for import accounting elements
       NOTE: in this procedure there's some fields that are created
             from another module, TODO correct for keep module 'modular'
    '''
    _inherit = 'purchase.order'

    # -----------------
    # Utility function:
    # -----------------
    def load_converter(self, cr, uid, carriage_condition, transportation_reason,
            payment, context=None):
        ''' Load useful converters
        '''

        # Create a converter for carriage condition:
        cc_pool = self.pool.get('stock.picking.carriage_condition')
        cc_ids = cc_pool.search(cr, uid, [], context=context)
        for item in cc_pool.browse(cr, uid, cc_ids, context=context):
            if item.account_code:
                carriage_condition[item.account_code] = item.id

        # Create a converter for transportation reason:
        tr_pool = self.pool.get('stock.picking.transportation_reason')
        tr_ids = tr_pool.search(cr, uid, [], context=context)
        for item in tr_pool.browse(cr, uid, tr_ids, context=context):
            if item.import_id:
                transportation_reason[item.import_id] = item.id

        # Create a converter for payment:
        payment_pool = self.pool.get('account.payment.term')
        payment_ids = payment_pool.search(cr, uid, [], context=context)
        for item in payment_pool.browse(cr, uid, payment_ids, context=context):
            if item.import_id:
                payment[item.import_id] = item.id
        return

    def get_uom(self, cr, uid, name, context=None):
        uom_id = self.pool.get('product.uom').search(cr, uid, [
            ('name', '=', name), ])
        if uom_id:
            return uom_id[0] # take the first
        else:
            return False

    # -------------------------------------------------------------------------
    #                           Button event:
    # -------------------------------------------------------------------------
    def schedule_etl_purchase_order(self, cr, uid, context=None):
        ''' Button event for import all order (purchase)
        '''
        _logger.info('Start update extra info OF header')
        query_pool = self.pool.get('micronaet.accounting')
        partner_pool = self.pool.get('res.partner')
        location_pool = self.pool.get('stock.location')

        cr_of = query_pool.get_of_header(cr, uid, context=context)
        if not cr_of:
            _logger.error('Cannot connect to MSSQL OF_TESTATE')
            return

        # -----------
        # Converters:
        # -----------
        orders = {}
        carriage_condition = {}
        transportation_reason = {}
        payment = {}
        self.load_converter(cr, uid, carriage_condition,
            transportation_reason, payment, context=context)

        location_ids = location_pool.search(cr, uid, [
            ('name', '=', 'Stock')], context=context)
        if not location_ids:
            _logger.error('Cannot found location: Stock')
            return

        pricelist_id = 2 # TODO

        # ---------------------------------------------------------------------
        #                           HEADER:
        # ---------------------------------------------------------------------
        for of in cr_of:
            try:
                # -------------------------------------------------------------
                #                          Get data:
                # -------------------------------------------------------------
                # Partner:
                partner_code = of['CKY_CNT_CLFR']
                partner_ids = partner_pool.search(cr, uid, [
                    ('sql_supplier_code', '=', partner_code),
                    ], context=context)
                if not partner_ids:
                    _logger.error(
                        'No partner found, code: %s' % partner_code)
                    continue

                # Destination:
                dest_code = of['CKY_CNT_SPED_ALT']
                dest_id = False
                if dest_code:
                    dest_ids = partner_pool.search(cr, uid, ['|',
                        ('sql_supplier_code', '=', dest_code),
                        ('sql_destination_code', '=', dest_code),
                        ], context=context)
                    if dest_ids:
                        dest_id = dest_ids[0]
                    else:
                        _logger.error('Destination not found: %s' % dest_code)

                # Update transport readon: NKY_CAUM
                transportation_reason_code = of['NKY_CAUM']
                transportation_reason_id = False
                if transportation_reason_code:
                    transportation_reason_id = transportation_reason.get(
                        transportation_reason_code, False)
                    if not transportation_reason_id:
                        _logger.error('Tranportation reason not found: %s' % (
                            transportation_reason_code))

                # Update payment
                payment_term_code = of['NKY_PAG']
                payment_term_id = False
                if payment_term_code:
                    payment_term_id = payment.get(
                        payment_term_code, False)
                    if not payment_term_id:
                        _logger.error('Payment code not found: %s' % (
                            payment_term_code))

                # Update porto:
                #carriage_condition_code = of['IST_PORTO']
                #if carriage_condition_code:
                #    carriage_condition_id = carriage_condition.get(
                #        carriage_condition_code, False)
                #    if carriage_condition_id:
                #        data['carriage_condition_id'] = carriage_condition_id
                #        _logger.info('Carriage condition update: %s' % name)
                #    else:
                #        _logger.warning('Carriage condition not found: %s' % (
                #            carriage_condition_code))

                # Update note:
                # TODO
                #carrier_code = of['CKY_CNT_VETT']

                # Parcels:
                #parcels = of['NGB_TOT_COLLI']
                #if parcels:
                #    data['parcels'] = parcels
                #    _logger.info('Update parcels: %s' % name)

                # Key field:
                name = 'MX-%s/%s' % (
                    of['NGL_DOC'], of['DTT_DOC'].strftime('%Y'))

                of_ids = self.search(cr, uid, [
                    ('name', '=', name),
                    #('accounting_order', '=', True), # No more used!
                    ], context=context)

                header = {
                    'name': name,
                    'partner_id': partner_ids[0],
                    'destination_partner_id': dest_id,

                    'date_order': of['DTT_DOC'].strftime(
                        DEFAULT_SERVER_DATE_FORMAT),
                    'payment_term_id': payment_term_id,
                    'transportation_reason_id': transportation_reason_id,
                    'notes': of['CDS_NOTE'],
                    'location_id': location_ids[0],
                    'pricelist_id': pricelist_id,
                    #'goods_description_id':
                    #'transportation_method':
                    #'carriage_condition_id':
                    #'partner_ref':
                    #'currency_id'
                    #'used_bank_id'
                    #'fiscal_position'
                    #'payment_note':
                    #'delivery_note':
                    #'note'TODO???
                    #'incoterm_id'
                    #'minimum_planned_date'
                    }

                if of_ids:
                    _logger.info('Update header: %s' % name)
                    order_id = of_ids[0]
                    self.write(cr, uid, order_id, header, context=context)
                else:
                    _logger.info('Create header: %s' % name)
                    order_id = self.create(cr, uid, header,
                        context=context)

                # Save for line:
                orders[of['NGL_DOC']] = order_id

            except:
                _logger.error('Problem with record: %s > %s'%(
                    of, sys.exc_info()))

        # ---------------------------------------------------------------------
        #                           LINE:
        # ---------------------------------------------------------------------
        _logger.info('Start import OF lines')
        line_pool = self.pool.get('purchase.order.line')
        product_pool = self.pool.get('product.product')

        cr_of_line = query_pool.get_of_line(cr, uid, context=context)
        if not cr_of_line:
            _logger.error('Cannot connect to MSSQL OF_RIGHE')
            return

        for of_line in cr_of_line:
            try:
                # -----------------------------
                # Get product browse from code:
                # -----------------------------
                order_id = orders.get(of_line['NGL_DOC'], False)
                if not order_id:
                    _logger.error('Order not found: %s' % name)
                    continue

                # Product:
                product_code = of_line['CKY_ART'].strip()
                product_ids = product_pool.search(cr, uid, [
                    ('default_code', '=', product_code)], context=context)
                if not product_ids:
                    _logger.info(
                        _('No product found (OFC line jumped): %s') % (
                            product_code))
                    continue
                product_browse = product_pool.browse(
                    cr, uid, product_ids, context=context)[0]

                date_planned = of_line['DTT_SCAD'].strftime(
                    DEFAULT_SERVER_DATE_FORMAT)

                # NOTE ID of line in OC (not sequence=order)
                sequence = of_line['NPR_RIGA']
                uom_id = product_browse.uom_id.id if product_browse else False
                conversion = (
                    of_line['NCF_CONV'] if of_line['NCF_CONV'] else 1.0)

                # pack * unit item * conversion
                quantity = (of_line['NGB_COLLI'] or 1.0) * (
                    of_line['NQT_RIGA_O_PLOR'] or 0.0) * 1.0 / conversion

                # HEADER: Save deadline in OC (only first time):
                #if not of_header[of_key][1]:
                #    # take the first deadline and save in header
                #    if date_deadline:
                #        of_header[of_key][1] = True
                #        mod = self.write(cr, uid, order_id, {
                #            'date_deadline': date_deadline}, context=context)

                # Discount block:
                #discount = False
                #multi_discount_rates = False
                account_scale = of_line['CSG_SCN'].strip() or ''
                discount = account_scale.replace(',', '.')
                #if account_scale:
                #    try:
                #        res = line_pool.on_change_multi_discount(
                #            cr, uid, False, account_scale,
                #            context=context)['value']
                #        discount = res.get('discount', False)
                #        multi_discount_rates = res.get(
                #            'multi_discount_rates', False)
                #    except:
                #        _logger.error(
                #            'Error calculating discount value: %s' % (
                #                account_scale))
                #        pass

                data = {
                    'product_id': product_browse.id,
                    'product_qty': quantity,
                    'order_id': order_id,
                    'date_planned': date_planned,
                    'product_uom': uom_id,
                    'name': of_line['CDS_VARIAZ_ART'],
                    'price_unit': (
                        of_line['NPZ_UNIT'] or 0.0) * conversion,
                    # Correct depend on fiscal position
                    'sequence': sequence,

                    # not present:
                    'discount': discount,
                    'taxes_id': [
                        (6, 0, [product_browse.taxes_id[0].id, ])
                        ] if product_browse and product_browse.taxes_id
                            else False, # CSG_IVA
                    #'multi_discount_rates': multi_discount_rates,

                    # Related fields ------------------------------------------
                    #'partner_id': order_partner.get(order_id, False),
                    }
                line_ids = line_pool.search(cr, uid, [
                    ('order_id', '=', order_id),
                    ('sequence', '=', sequence),
                    ], context=context)
                if line_ids:
                    line_pool.write(cr, uid, line_ids[0], data, context=context)
                    _logger.info('Order: %s line %s updated' % (
                        of_line['NGL_DOC'],
                        sequence,
                        ))
                else:
                    line_pool.create(cr, uid, data, context=context)
                    _logger.info('Order: %s line %s created' % (
                        of_line['NGL_DOC'],
                        sequence,
                        ))

            except:
                _logger.error('Problem with of line record: %s\n%s' % (
                    of_line, sys.exc_info()))

        # TODO testare bene gli ordini di produzione che potrebbero avere delle mancanze!
        _logger.info('End importation OF header and line!')
        return

class PurchaseOrderLine(orm.Model):
    ''' Extra fields used
    '''
    _inherit = 'purchase.order.line'

    _columns = {
        # TODO change module position
        'discount': fields.float('Discount', digits=(16, 3)),
        'discount_scale': fields.char('Discount scale', size=40),
        'sequence': fields.integer('Sequence'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>