<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Iterate over results in data/screening and perform evaluation.

The output is stored into data/evaluation.
"""

import os
import argparse
import json
import logging
import csv
import time
import functools
from rdkit.ML.Scoring import Scoring

__author__ = 'Petr Škoda'
__license__ = 'X11'
__email__ = 'skoda@ksi.mff.cuni.cz'

# region Loading files

def find_json_files(root):
    """Recursively find all JSON files.

    :param root:
    :return:
    """
    result = []
    for file_name in os.listdir(root):
        file_path = root + '/' + file_name
        if os.path.isdir(file_path):
            result.extend(find_json_files(file_path))
            continue
        if file_name.endswith('.json'):
            result.append(file_path)
    return result

def find_results():
    """Search and return paths to all screening results.

    :return:
    """
    logging.info('Searching for results ...')
    data_path = os.path.dirname(os.path.realpath(__file__)) + \
                '/../data/screening/'
    results = find_json_files(data_path)
    logging.info('Searching for results ... done (%d)', len(results))
    return results

def load_groups():
    """Find load and return all groups.

    :return:
    """
    logging.info('Loading groups ...')
    datasets_path = os.path.dirname(os.path.realpath(__file__)) + \
                    '/../data/datasets'
    paths = [file for file in find_json_files(datasets_path)
             if file.endswith('group.json')]

    results = {}
    for path in paths:
        with open(path) as stream:
            data = json.load(stream)
            group = ''
            if 'id' in data['metadata']:
                group = data['metadata']['id']
            if 'group' in data['metadata']:
                group = data['metadata']['group']
            id = data['metadata']['dataset'] + \
                 data['metadata']['selection'] + \
                 group
            results[id] = data
    logging.info('Loading groups ... done (%d)', len(results))
    return results

# endregion

# region CSV read and write

def line_to_entry(columns, row):
    """Convert and return line CSV object into dictionary.

    :param columns:
    :param row:
    :return:
    """
    output = {}
    for i in range(0, len(columns)):
        output[columns[i]] = row[i]
    return output

def header():
    """Return a header line.

    :return:
    """
    return ['path', 'dataset', 'selection', 'group', 'instance', 'method',
            'w-auc', 'w-ef-0.005', 'w-ef-0.01', 'w-ef-0.02', 'w-ef-0.05',
            'b-auc', 'b-ef-0.005', 'b-ef-0.01', 'b-ef-0.02', 'b-ef-0.05']

def entry_to_line(entry):
    """Convert entry to the line.

    :param entry:
    :return:
    """
    values = [entry['path'], entry['dataset'], entry['selection'],
              entry['group'], entry['instance'], entry['method'],
              entry['w-auc'],
              entry['w-ef-0.005'],
              entry['w-ef-0.01'],
              entry['w-ef-0.02'],
              entry['w-ef-0.05'],
              entry['b-auc'],
              entry['b-ef-0.005'],
              entry['b-ef-0.01'],
              entry['b-ef-0.02'],
              entry['b-ef-0.05']]
    return [str(item) for item in values]

# endregion

# region Score comparison

def score_compare_worst(left, right):
    if left['score'] == right['score']:
        # Decide based on activity.
        if left['activity'] and right['activity']:
            return 0
        elif left['activity']:
            # Only left is active.
            return -1
        else:
            return 1
    else:
        if left['score'] > right['score']:
            return 1
        else:
            return -1

def score_compare_best(left, right):
    if left['score'] == right['score']:
        # Decide based on activity.
        if left['activity'] and right['activity']:
            return 0
        elif left['activity']:
            # Only left is active.
            return 1
        else:
            return -1
    else:
        if left['score'] > right['score']:
            return 1
        else:
            return -1

# endregion

def compute_measures(scores, params=[0.005, 0.01, 0.02, 0.05]):
    """Evaluate scores and return object with performance metrics,

    :param scores:
    :param params:
    :return:
    """
    output = {
        'auc': Scoring.CalcAUC(scores, 'activity')
    }
    for value in params:
        output['ef-' + str(value)] = \
            Scoring.CalcEnrichment(scores, 'activity', [value])[0]
        # output['bedroc-' + str(value)] = \
        #     Scoring.CalcBEDROC(scores, 'activity', value)
        # output['rie-' + str(value)] = \
        #     Scoring.CalcRIE(scores, 'activity', value)
    return output

def evaluate_result_file(result_path, groups):
    """Evaluate a single result file and return the result object.

    result_path:
    :param groups:
    :return: None in case of an error.
    """
    with open(result_path) as stream:
        try:
            result = json.load(stream)
        except:
            logging.exception('Invalid result file: %s', result_path)
            return None
    group_id = result['dataset'] + result['selection'] + result['group']
    if group_id not in groups:
        logging.warning('Missing group.')
        return None
    group = groups[group_id]
    ligand_names = set()
    for item in group['data']['ligands']:
        ligand_names.add(item['id'])
    decoys_names = set()
    for item in group['data']['decoys']:
        decoys_names.add(item['id'])
    # Add activity to molecules.
    missing_activity = False
    for score in result['scores']:
        is_ligand = score['molecule'] in ligand_names
        is_decoy = score['molecule'] in decoys_names
        if is_ligand and not is_decoy:
            score['activity'] = 1
        elif not is_ligand and is_decoy:
            score['activity'] = 0
        else:
            logging.error('Missing: "%s" in "%s"',
                          score['molecule'], result_path)
            missing_activity = True
    if missing_activity:
        return None
    # Evaluate.
    eval_best = compute_measures(sorted(result['scores'],
                                        key=functools.cmp_to_key(
                                            score_compare_best),
                                        reverse=True))
    eval_worst = compute_measures(sorted(result['scores'],
                                        key=functools.cmp_to_key(
                                            score_compare_worst),
                                         reverse=True))
    # Store results to object.
    return {
        'group': result['group'],
        'dataset': result['dataset'],
        'selection': result['selection'],
        'instance': result['instance'],
        'method': result['method'],
        'w-auc': eval_worst['auc'],
        'w-ef-0.005': eval_worst['ef-0.005'],
        'w-ef-0.01': eval_worst['ef-0.01'],
        'w-ef-0.02': eval_worst['ef-0.02'],
        'w-ef-0.05': eval_worst['ef-0.05'],
        'b-auc': eval_best['auc'],
        'b-ef-0.005': eval_best['ef-0.005'],
        'b-ef-0.01': eval_best['ef-0.01'],
        'b-ef-0.02': eval_best['ef-0.02'],
        'b-ef-0.05': eval_best['ef-0.05']
    }

def read_results(path):
    """Read and return existing results.

    :param path:
    :return:
    """
    results = []
    paths = set()
    with open(path) as stream:
        csv_reader = csv.reader(stream, delimiter=',', quotechar='"')
        #columns = csv_reader.__next__()
        columns = csv_reader.next()
        for row in csv_reader:
            entry = line_to_entry(columns, row)
            results.append(entry)
            paths.add(entry['path'])
    return results, paths

def load_configuration():
    """Load and return the configuration.

    :return:
    """
    parser = argparse.ArgumentParser(
        description='Evaluate runs on local machine.')
    parser.add_argument('-t', type=int, dest='time', required=False, default=-1,
                        help='Time limit in minutes.')
    return vars(parser.parse_args())

def main():
    configuration = load_configuration()
    time_end = -1
    if not configuration['time'] == -1:
        time_end = time.time() + (configuration['time'] * 60)
    # Search for group files and results.
    results = find_results()
    groups = load_groups()
    # Load existing results if they exists.
    eval_dir = os.path.dirname(os.path.realpath(__file__)) + \
               '/../data/evaluation/'
    if not os.path.exists(eval_dir):
        os.makedirs(eval_dir)
    results_path = eval_dir + 'results.csv'
    eval_list = []
    eval_paths = set()
    if os.path.exists(results_path):
        logging.info('Reading results ...')
        eval_list, eval_paths = read_results(results_path)
        logging.info('Reading results ... done')
    # Evaluation.
    logging.info('Evaluating ...')
    with open(results_path, 'w') as output_stream:
        output_stream.write('"')
        output_stream.write('","'.join(header()))
        output_stream.write('"\n')
        # Write old results.
        for item in eval_list:
            output_stream.write('"')
            output_stream.write('","'.join(entry_to_line(item)))
            output_stream.write('"\n')
        eval_list = []
        # Iterate over new results
        for result_path in results:
            relative_path = result_path[
                            result_path.index('/../data/screening/') + 19:]
            # Use path to determine if the file was already processed or not.
            if relative_path in eval_paths:
                continue
            result_entry = evaluate_result_file(result_path, groups)
            if result_entry is None:
                continue
            result_entry['path'] = relative_path
            # Write to file.
            output_stream.write('"')
            output_stream.write('","'.join(entry_to_line(result_entry)))
            output_stream.write('"\n')
            if time_end < time.time():
                logging.info('timeout')
                break
    logging.info('Evaluating ... done')

if __name__ == '__main__':
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(asctime)s [%(levelname)s] - %(message)s',
        datefmt='%H:%M:%S')
    main()
<commit_msg>Bug fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Iterate over results in data/screening and perform evaluation.

The output is stored into data/evaluation.
"""

import os
import argparse
import json
import logging
import csv
import time
import functools
from rdkit.ML.Scoring import Scoring

__author__ = 'Petr Škoda'
__license__ = 'X11'
__email__ = 'skoda@ksi.mff.cuni.cz'

# region Loading files

def find_json_files(root):
    """Recursively find all JSON files.

    :param root:
    :return:
    """
    result = []
    for file_name in os.listdir(root):
        file_path = root + '/' + file_name
        if os.path.isdir(file_path):
            result.extend(find_json_files(file_path))
            continue
        if file_name.endswith('.json'):
            result.append(file_path)
    return result

def find_results():
    """Search and return paths to all screening results.

    :return:
    """
    logging.info('Searching for results ...')
    data_path = os.path.dirname(os.path.realpath(__file__)) + \
                '/../data/screening/'
    results = find_json_files(data_path)
    logging.info('Searching for results ... done (%d)', len(results))
    return results

def load_groups():
    """Find load and return all groups.

    :return:
    """
    logging.info('Loading groups ...')
    datasets_path = os.path.dirname(os.path.realpath(__file__)) + \
                    '/../data/datasets'
    paths = [file for file in find_json_files(datasets_path)
             if file.endswith('group.json')]

    results = {}
    for path in paths:
        with open(path) as stream:
            data = json.load(stream)
            group = ''
            if 'id' in data['metadata']:
                group = data['metadata']['id']
            if 'group' in data['metadata']:
                group = data['metadata']['group']
            id = data['metadata']['dataset'] + \
                 data['metadata']['selection'] + \
                 group
            results[id] = data
    logging.info('Loading groups ... done (%d)', len(results))
    return results

# endregion

# region CSV read and write

def line_to_entry(columns, row):
    """Convert and return line CSV object into dictionary.

    :param columns:
    :param row:
    :return:
    """
    output = {}
    for i in range(0, len(columns)):
        output[columns[i]] = row[i]
    return output

def header():
    """Return a header line.

    :return:
    """
    return ['path', 'dataset', 'selection', 'group', 'instance', 'method',
            'w-auc', 'w-ef-0.005', 'w-ef-0.01', 'w-ef-0.02', 'w-ef-0.05',
            'b-auc', 'b-ef-0.005', 'b-ef-0.01', 'b-ef-0.02', 'b-ef-0.05']

def entry_to_line(entry):
    """Convert entry to the line.

    :param entry:
    :return:
    """
    values = [entry['path'], entry['dataset'], entry['selection'],
              entry['group'], entry['instance'], entry['method'],
              entry['w-auc'],
              entry['w-ef-0.005'],
              entry['w-ef-0.01'],
              entry['w-ef-0.02'],
              entry['w-ef-0.05'],
              entry['b-auc'],
              entry['b-ef-0.005'],
              entry['b-ef-0.01'],
              entry['b-ef-0.02'],
              entry['b-ef-0.05']]
    return [str(item) for item in values]

# endregion

# region Score comparison

def score_compare_worst(left, right):
    if left['score'] == right['score']:
        # Decide based on activity.
        if left['activity'] and right['activity']:
            return 0
        elif left['activity']:
            # Only left is active.
            return -1
        else:
            return 1
    else:
        if left['score'] > right['score']:
            return 1
        else:
            return -1

def score_compare_best(left, right):
    if left['score'] == right['score']:
        # Decide based on activity.
        if left['activity'] and right['activity']:
            return 0
        elif left['activity']:
            # Only left is active.
            return 1
        else:
            return -1
    else:
        if left['score'] > right['score']:
            return 1
        else:
            return -1

# endregion

def compute_measures(scores, params=[0.005, 0.01, 0.02, 0.05]):
    """Evaluate scores and return object with performance metrics,

    :param scores:
    :param params:
    :return:
    """
    output = {
        'auc': Scoring.CalcAUC(scores, 'activity')
    }
    for value in params:
        output['ef-' + str(value)] = \
            Scoring.CalcEnrichment(scores, 'activity', [value])[0]
        # output['bedroc-' + str(value)] = \
        #     Scoring.CalcBEDROC(scores, 'activity', value)
        # output['rie-' + str(value)] = \
        #     Scoring.CalcRIE(scores, 'activity', value)
    return output

def evaluate_result_file(result_path, groups):
    """Evaluate a single result file and return the result object.

    result_path:
    :param groups:
    :return: None in case of an error.
    """
    with open(result_path) as stream:
        try:
            result = json.load(stream)
        except:
            logging.exception('Invalid result file: %s', result_path)
            return None
    group_id = result['dataset'] + result['selection'] + result['group']
    if group_id not in groups:
        logging.warning('Missing group.')
        return None
    group = groups[group_id]
    ligand_names = set()
    for item in group['data']['ligands']:
        ligand_names.add(item['id'])
    decoys_names = set()
    for item in group['data']['decoys']:
        decoys_names.add(item['id'])
    # Add activity to molecules.
    missing_activity = False
    for score in result['scores']:
        is_ligand = score['molecule'] in ligand_names
        is_decoy = score['molecule'] in decoys_names
        if is_ligand and not is_decoy:
            score['activity'] = 1
        elif not is_ligand and is_decoy:
            score['activity'] = 0
        else:
            logging.error('Missing: "%s" in "%s"',
                          score['molecule'], result_path)
            missing_activity = True
    if missing_activity:
        return None
    # Evaluate.
    eval_best = compute_measures(sorted(result['scores'],
                                        key=functools.cmp_to_key(
                                            score_compare_best),
                                        reverse=True))
    eval_worst = compute_measures(sorted(result['scores'],
                                        key=functools.cmp_to_key(
                                            score_compare_worst),
                                         reverse=True))
    # Store results to object.
    return {
        'group': result['group'],
        'dataset': result['dataset'],
        'selection': result['selection'],
        'instance': result['instance'],
        'method': result['method'],
        'w-auc': eval_worst['auc'],
        'w-ef-0.005': eval_worst['ef-0.005'],
        'w-ef-0.01': eval_worst['ef-0.01'],
        'w-ef-0.02': eval_worst['ef-0.02'],
        'w-ef-0.05': eval_worst['ef-0.05'],
        'b-auc': eval_best['auc'],
        'b-ef-0.005': eval_best['ef-0.005'],
        'b-ef-0.01': eval_best['ef-0.01'],
        'b-ef-0.02': eval_best['ef-0.02'],
        'b-ef-0.05': eval_best['ef-0.05']
    }

def read_results(path):
    """Read and return existing results.

    :param path:
    :return:
    """
    results = []
    paths = set()
    with open(path) as stream:
        csv_reader = csv.reader(stream, delimiter=',', quotechar='"')
        #columns = csv_reader.__next__()
        columns = csv_reader.next()
        for row in csv_reader:
            entry = line_to_entry(columns, row)
            results.append(entry)
            paths.add(entry['path'])
    return results, paths

def load_configuration():
    """Load and return the configuration.

    :return:
    """
    parser = argparse.ArgumentParser(
        description='Evaluate runs on local machine.')
    parser.add_argument('-t', type=int, dest='time', required=False, default=-1,
                        help='Time limit in minutes.')
    return vars(parser.parse_args())

def main():
    configuration = load_configuration()
    time_end = -1
    if not configuration['time'] == -1:
        time_end = time.time() + (configuration['time'] * 60)
    # Search for group files and results.
    results = find_results()
    groups = load_groups()
    # Load existing results if they exists.
    eval_dir = os.path.dirname(os.path.realpath(__file__)) + \
               '/../data/evaluation/'
    if not os.path.exists(eval_dir):
        os.makedirs(eval_dir)
    results_path = eval_dir + 'results.csv'
    eval_list = []
    eval_paths = set()
    if os.path.exists(results_path):
        logging.info('Reading results ...')
        eval_list, eval_paths = read_results(results_path)
        logging.info('Reading results ... done')
    # Evaluation.
    logging.info('Evaluating ...')
    with open(results_path, 'w') as output_stream:
        output_stream.write('"')
        output_stream.write('","'.join(header()))
        output_stream.write('"\n')
        # Write old results.
        for item in eval_list:
            output_stream.write('"')
            output_stream.write('","'.join(entry_to_line(item)))
            output_stream.write('"\n')
        eval_list = []
        # Iterate over new results
        for result_path in results:
            relative_path = result_path[
                            result_path.index('/../data/screening/') + 19:]
            # Use path to determine if the file was already processed or not.
            if relative_path in eval_paths:
                continue
            result_entry = evaluate_result_file(result_path, groups)
            if result_entry is None:
                continue
            result_entry['path'] = relative_path
            # Write to file.
            output_stream.write('"')
            output_stream.write('","'.join(entry_to_line(result_entry)))
            output_stream.write('"\n')
            if -1 < time_end < time.time():
                logging.info('timeout')
                break
    logging.info('Evaluating ... done')

if __name__ == '__main__':
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(asctime)s [%(levelname)s] - %(message)s',
        datefmt='%H:%M:%S')
    main()
<|endoftext|>


<commit_before>
def patch_postgis_bad_geomery_escape():
    """
    When using django 1.3.X with PostgreSQL 9.1 some geometry can be badly
    escaped. Monkey patch PostGISAdapter to fix this issue.
    see https://code.djangoproject.com/ticket/16778
    """
    # TODO: Remove when support for django 1.3 is dropped
    import django
    if django.VERSION < (1, 4):
        from django.db import connections
        postgis_engine = 'django.contrib.gis.db.backends.postgis'
        if any(connection.settings_dict['ENGINE'] == postgis_engine
               for connection in connections.all()):
            from psycopg2 import Binary
            from django.contrib.gis.db.backends.postgis.adapter import PostGISAdapter

            PostGISAdapter__init__ = PostGISAdapter.__init__
            def _PostGISAdapter__init__(self, *args, **kwargs):
                PostGISAdapter__init__(self, *args, **kwargs)
                self._adapter = Binary(self.ewkb)
            PostGISAdapter.__init__ = _PostGISAdapter__init__

            def _PostGISAdapter_prepare(self, conn):
                self._adapter.prepare(conn)
            PostGISAdapter.prepare = _PostGISAdapter_prepare

            def _PostGISAdapter_getquoted(self):
                return 'ST_GeomFromEWKB(%s)' % self._adapter.getquoted()
            PostGISAdapter.getquoted = _PostGISAdapter_getquoted

def patch_db_field_compare():
    """
    Field instances cannot be compared to other objects because of attribute
    presence assumptions in it's __cmp__ method. To allow full_clean we must
    override the item_field __cmp__ method to return NotImplemented if the
    object it's compared to isn't a Field instance. Let's monkey patch it!
    see https://code.djangoproject.com/ticket/17851
    """
    # TODO: Remove when support for django 1.4 is dropped
    from django.db.models.fields import Field
    try:
        assert Field() != None
    except AttributeError:
        del Field.__cmp__
        def _Field__lt__(self, other):
            if isinstance(other, Field):
                return self.creation_counter < other.creation_counter
            return NotImplemented
        Field.__lt__ = _Field__lt__
        assert Field() != None

def patch_model_option_verbose_name_raw():
    """
    Until #17763 and all the permission name length issues are fixed we patch
    the `verbose_name_raw` method to return a truncated string in order to
    avoid DatabaseError.
    """
    from django.db.models.options import Options
    verbose_name_raw = Options.verbose_name_raw.fget
    if hasattr(verbose_name_raw, '_patched'):
        return
    def _get_verbose_name_raw(self):
        name = verbose_name_raw(self)
        if len(name) >= 40:
            name = "%s..." % name[0:36]
        return name
    _get_verbose_name_raw.patched = True
    Options.verbose_name_raw = property(_get_verbose_name_raw)

def get_concrete_model(model):
    """
    Prior to django r17573 (django 1.4), `proxy_for_model` returned the
    actual concrete model of a proxy and there was no `concrete_model`
    property so we try to fetch the `concrete_model` from the opts
    and fallback to `proxy_for_model` if it's not defined.
    """
    # TODO: Remove when support for django 1.4 is dropped
    return getattr(model._meta, 'concrete_model', model._meta.proxy_for_model)

def get_real_content_type(model, db=None):
    """
    Prior to #18399 being fixed there was no way to retrieve `ContentType`
    of proxy models. This is a shim that tries to use the newly introduced
    flag and fallback to another method.
    """
    # TODO: Remove when support for django 1.4 is dropped
    from django.contrib.contenttypes.models import ContentType
    from django.utils.encoding import smart_unicode
    cts = ContentType.objects
    if db:
        cts = cts.db_manager(db)
    opts = model._meta
    if opts.proxy:
        try:
            # Attempt to use the `for_concrete_model` kwarg available in
            # django >= 1.5
            return cts.get_for_model(model, for_concrete_model=False)
        except TypeError:
            if model._deferred:
                opts = opts.proxy_for_model._meta
            try:
                ct = cts._get_from_cache(opts)
            except KeyError:
                ct, _created = cts.get_or_create(
                    app_label = opts.app_label,
                    model = opts.object_name.lower(),
                    defaults = {'name': smart_unicode(opts.verbose_name_raw)},
                )
                cts._add_to_cache(cts.db, ct)
                return ct
    else:
        return cts.get_for_model(model)
 <commit_msg>Fix the `get_real_content_type` method to work on django 1.3<commit_after>
def patch_postgis_bad_geomery_escape():
    """
    When using django 1.3.X with PostgreSQL 9.1 some geometry can be badly
    escaped. Monkey patch PostGISAdapter to fix this issue.
    see https://code.djangoproject.com/ticket/16778
    """
    # TODO: Remove when support for django 1.3 is dropped
    import django
    if django.VERSION < (1, 4):
        from django.db import connections
        postgis_engine = 'django.contrib.gis.db.backends.postgis'
        if any(connection.settings_dict['ENGINE'] == postgis_engine
               for connection in connections.all()):
            from psycopg2 import Binary
            from django.contrib.gis.db.backends.postgis.adapter import PostGISAdapter

            PostGISAdapter__init__ = PostGISAdapter.__init__
            def _PostGISAdapter__init__(self, *args, **kwargs):
                PostGISAdapter__init__(self, *args, **kwargs)
                self._adapter = Binary(self.ewkb)
            PostGISAdapter.__init__ = _PostGISAdapter__init__

            def _PostGISAdapter_prepare(self, conn):
                self._adapter.prepare(conn)
            PostGISAdapter.prepare = _PostGISAdapter_prepare

            def _PostGISAdapter_getquoted(self):
                return 'ST_GeomFromEWKB(%s)' % self._adapter.getquoted()
            PostGISAdapter.getquoted = _PostGISAdapter_getquoted

def patch_db_field_compare():
    """
    Field instances cannot be compared to other objects because of attribute
    presence assumptions in it's __cmp__ method. To allow full_clean we must
    override the item_field __cmp__ method to return NotImplemented if the
    object it's compared to isn't a Field instance. Let's monkey patch it!
    see https://code.djangoproject.com/ticket/17851
    """
    # TODO: Remove when support for django 1.4 is dropped
    from django.db.models.fields import Field
    try:
        assert Field() != None
    except AttributeError:
        del Field.__cmp__
        def _Field__lt__(self, other):
            if isinstance(other, Field):
                return self.creation_counter < other.creation_counter
            return NotImplemented
        Field.__lt__ = _Field__lt__
        assert Field() != None

def patch_model_option_verbose_name_raw():
    """
    Until #17763 and all the permission name length issues are fixed we patch
    the `verbose_name_raw` method to return a truncated string in order to
    avoid DatabaseError.
    """
    from django.db.models.options import Options
    verbose_name_raw = Options.verbose_name_raw.fget
    if hasattr(verbose_name_raw, '_patched'):
        return
    def _get_verbose_name_raw(self):
        name = verbose_name_raw(self)
        if len(name) >= 40:
            name = "%s..." % name[0:36]
        return name
    _get_verbose_name_raw.patched = True
    Options.verbose_name_raw = property(_get_verbose_name_raw)

def get_concrete_model(model):
    """
    Prior to django r17573 (django 1.4), `proxy_for_model` returned the
    actual concrete model of a proxy and there was no `concrete_model`
    property so we try to fetch the `concrete_model` from the opts
    and fallback to `proxy_for_model` if it's not defined.
    """
    # TODO: Remove when support for django 1.4 is dropped
    return getattr(model._meta, 'concrete_model', model._meta.proxy_for_model)

def get_real_content_type(model, db=None):
    """
    Prior to #18399 being fixed there was no way to retrieve `ContentType`
    of proxy models. This is a shim that tries to use the newly introduced
    flag and fallback to another method.
    """
    # TODO: Remove when support for django 1.4 is dropped
    from django.contrib.contenttypes.models import ContentType
    from django.utils.encoding import smart_unicode
    cts = ContentType.objects
    if db:
        cts = cts.db_manager(db)
    opts = model._meta
    if opts.proxy:
        try:
            # Attempt to use the `for_concrete_model` kwarg available in
            # django >= 1.5
            return cts.get_for_model(model, for_concrete_model=False)
        except TypeError:
            if model._deferred:
                opts = opts.proxy_for_model._meta
            try:
                try:
                    ct = cts._get_from_cache(opts)
                # TODO: Remove when support for django 1.3 is dropped
                except AttributeError:
                    # django 1.3 doesn't have the `_get_from_cache` method
                    key = (opts.app_label, opts.object_name.lower())
                    return cts.__class__._cache[cts.db][key]
            except KeyError:
                ct, _created = cts.get_or_create(
                    app_label = opts.app_label,
                    model = opts.object_name.lower(),
                    defaults = {'name': smart_unicode(opts.verbose_name_raw)},
                )
                cts._add_to_cache(cts.db, ct)
                return ct
    else:
        return cts.get_for_model(model)
 <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

import locale

from petl.util.base import Table

def fromxlsx(filename, sheet=None, range_string=None, row_offset=0,
             column_offset=0, **kwargs):
    """
    Extract a table from a sheet in an Excel .xlsx file.

    N.B., the sheet name is case sensitive.

    The `sheet` argument can be omitted, in which case the first sheet in
    the workbook is used by default.

    The `range_string` argument can be used to provide a range string
    specifying a range of cells to extract.

    The `row_offset` and `column_offset` arguments can be used to
    specify offsets.

    Any other keyword arguments are passed through to
    :func:`openpyxl.load_workbook()`.

    """

    return XLSXView(filename, sheet=sheet, range_string=range_string,
                    row_offset=row_offset, column_offset=column_offset,
                    **kwargs)

class XLSXView(Table):

    def __init__(self, filename, sheet=None, range_string=None,
                 row_offset=0, column_offset=0, **kwargs):
        self.filename = filename
        self.sheet = sheet
        self.range_string = range_string
        self.row_offset = row_offset
        self.column_offset = column_offset
        self.kwargs = kwargs

    def __iter__(self):
        import openpyxl
        wb = openpyxl.load_workbook(filename=self.filename,
                                    read_only=True, **self.kwargs)
        if self.sheet is None:
            ws = wb[wb.sheetnames[0]]
        elif isinstance(self.sheet, int):
            ws = wb[wb.sheetnames[self.sheet]]
        else:
            ws = wb[str(self.sheet)]

        if self.range_string is not None:
            rows = ws[self.range_string]
        else:
            rows = ws.iter_rows(row_offset=self.row_offset,
                                column_offset=self.column_offset)

        for row in rows:
            yield tuple(cell.value for cell in row)

        try:
            wb._archive.close()
        except AttributeError:
            # just here in case openpyxl stops exposing an _archive property.
            pass

def toxlsx(tbl, filename, sheet=None, encoding=None):
    """
    Write a table to a new Excel .xlsx file.

    """

    import openpyxl
    if encoding is None:
        encoding = locale.getpreferredencoding()
    wb = openpyxl.Workbook(write_only=True)
    ws = wb.create_sheet(title=sheet)
    for row in tbl:
        ws.append(row)
    wb.save(filename)

Table.toxlsx = toxlsx
<commit_msg>fix for upstream openpyxl API changes<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division

import locale

from petl.util.base import Table

def fromxlsx(filename, sheet=None, range_string=None, row_offset=0,
             column_offset=0, **kwargs):
    """
    Extract a table from a sheet in an Excel .xlsx file.

    N.B., the sheet name is case sensitive.

    The `sheet` argument can be omitted, in which case the first sheet in
    the workbook is used by default.

    The `range_string` argument can be used to provide a range string
    specifying a range of cells to extract.

    The `row_offset` and `column_offset` arguments can be used to
    specify offsets.

    Any other keyword arguments are passed through to
    :func:`openpyxl.load_workbook()`.

    """

    return XLSXView(filename, sheet=sheet, range_string=range_string,
                    min_row=row_offset, min_col=column_offset,
                    **kwargs)

class XLSXView(Table):

    def __init__(self, filename, sheet=None, range_string=None,
                 min_row=None, min_col=None, max_row=None, max_col=None, **kwargs):
        self.filename = filename
        self.sheet = sheet
        self.range_string = range_string
        self.min_row = min_row
        self.min_col = min_col
        self.max_row = max_row
        self.max_col = max_col
        self.kwargs = kwargs

    def __iter__(self):
        import openpyxl
        wb = openpyxl.load_workbook(filename=self.filename,
                                    read_only=True, **self.kwargs)
        if self.sheet is None:
            ws = wb[wb.sheetnames[0]]
        elif isinstance(self.sheet, int):
            ws = wb[wb.sheetnames[self.sheet]]
        else:
            ws = wb[str(self.sheet)]

        if self.range_string is not None:
            rows = ws[self.range_string]
        else:
            rows = ws.iter_rows(min_row=self.min_row,
                                min_col=self.min_col,
                                max_row=self.max_row,
                                max_col=self.max_col)

        for row in rows:
            yield tuple(cell.value for cell in row)

        try:
            wb._archive.close()
        except AttributeError:
            # just here in case openpyxl stops exposing an _archive property.
            pass

def toxlsx(tbl, filename, sheet=None, encoding=None):
    """
    Write a table to a new Excel .xlsx file.

    """

    import openpyxl
    if encoding is None:
        encoding = locale.getpreferredencoding()
    wb = openpyxl.Workbook(write_only=True)
    ws = wb.create_sheet(title=sheet)
    for row in tbl:
        ws.append(row)
    wb.save(filename)

Table.toxlsx = toxlsx
<|endoftext|>


<commit_before>"""
.. _ref_ex_custom:

Creating Custom Geometry
------------------------

Calculate section properties of a user-defined section from points and facets.

The following example demonstrates how geometry objects can be created from a
list of points, facets, holes and control points. An straight angle section with
a plate at its base is created from a list of points and facets. The bottom plate
is assigned a separate control point meaning two discrete regions are created.
Creating separate regions allows the user to control the mesh size in each region
and assign material properties to different regions. The geometry is cleaned to
remove the overlapping facet at the junction of the angle and the plate. A
geometric, warping and plastic analysis is then carried out.

The geometry and mesh are plotted before the analysis is carried out. Once the
analysis is complete, a plot of the various calculated centroids is generated
"""

# sphinx_gallery_thumbnail_number = 2

import sectionproperties.pre.sections as sections
from sectionproperties.analysis.cross_section import Section

# %%
# Define parameters for the angle section
a = 1
b = 2
t = 0.1

# %%
# Build the lists of points, facets, holes and control points
points = [[-t/2, -2*a], [t/2, -2*a], [t/2, -t/2], [a, -t/2], [a, t/2],
          [-t/2, t/2], [-b/2, -2*a], [b/2, -2*a], [b/2, -2*a-t],
          [-b/2, -2*a-t]]
facets = [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 0], [6, 7], [7, 8],
          [8, 9], [9, 6]]
holes = []
control_points = [[0, 0], [0, -2 * a - t / 2]]

# %%
# Because we have two separate geometry regions (as indicated by our control_points)
# we create a CompoundGeometry from points
geometry = sections.CompoundGeometry.from_points(points, facets, holes, control_points)

# %%
# Create the mesh and section. For the mesh, use a smaller refinement for the angle region.
geometry.create_mesh(mesh_sizes=[0.0005, 0.001])

section = Section(geometry)
section.plot_mesh()  # plot the generated mesh

# %%
# Perform a geometric, warping and plastic analysis
section.calculate_geometric_properties()
section.calculate_warping_properties()
section.calculate_plastic_properties()

section.plot_centroids()
<commit_msg>Minor edit to docs<commit_after>"""
.. _ref_ex_custom:

Creating Custom Geometry
------------------------

Calculate section properties of a user-defined section from points and facets.

The following example demonstrates how geometry objects can be created from a
list of points, facets, holes and control points. An straight angle section with
a plate at its base is created from a list of points and facets. The bottom plate
is assigned a separate control point meaning two discrete regions are created.
Creating separate regions allows the user to control the mesh size in each region
and assign material properties to different regions. The geometry is cleaned to
remove the overlapping facet at the junction of the angle and the plate. A
geometric, warping and plastic analysis is then carried out.

The geometry and mesh are plotted before the analysis is carried out. Once the
analysis is complete, a plot of the various calculated centroids is generated.
"""

# sphinx_gallery_thumbnail_number = 2

import sectionproperties.pre.sections as sections
from sectionproperties.analysis.cross_section import Section

# %%
# Define parameters for the angle section
a = 1
b = 2
t = 0.1

# %%
# Build the lists of points, facets, holes and control points
points = [[-t/2, -2*a], [t/2, -2*a], [t/2, -t/2], [a, -t/2], [a, t/2],
          [-t/2, t/2], [-b/2, -2*a], [b/2, -2*a], [b/2, -2*a-t],
          [-b/2, -2*a-t]]
facets = [[0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 0], [6, 7], [7, 8],
          [8, 9], [9, 6]]
holes = []
control_points = [[0, 0], [0, -2 * a - t / 2]]

# %%
# Because we have two separate geometry regions (as indicated by our control_points)
# we create a CompoundGeometry from points
geometry = sections.CompoundGeometry.from_points(points, facets, holes, control_points)

# %%
# Create the mesh and section. For the mesh, use a smaller refinement for the angle region.
geometry.create_mesh(mesh_sizes=[0.0005, 0.001])

section = Section(geometry)
section.plot_mesh()  # plot the generated mesh

# %%
# Perform a geometric, warping and plastic analysis
section.calculate_geometric_properties()
section.calculate_warping_properties()
section.calculate_plastic_properties()

section.plot_centroids()
<|endoftext|>


<commit_before>""" pgpdump.py
"""
import calendar
from datetime import datetime

from .packet import Header, SubPacket, String2Key
from .packet import PubKeyAlgo

class PGPDumpFormat(object):
    @staticmethod
    def bytefield(f):
        o = ""

        # python 2.7
        if bytes is str:
            o += ''.join('{:02x} '.format(ord(c)) for c in f)

        # python 3.x
        else:
            o += ''.join('{:02x} '.format(c) for c in f)

        return o

    def __init__(self, pktobj):
        self.out = []
        self.keycreation = 0

        for pkt in pktobj.packets:
            o = ""
            # print the header line first
            o += "{hformat}: {pktname}(tag {tagnum})({length} bytes)\n".format(
                hformat="Old" if pkt.header.format == 0 else "New",
                pktname=pkt.name,
                tagnum=pkt.header.tag.value,
                length=pkt.header.length
            )

            # state machine time!
            if pkt.header.tag == Header.Tag.Signature:
                o += self.signature_fields(pkt)

            if pkt.header.tag in [Header.Tag.PubKey, Header.Tag.PubSubKey,
                                  Header.Tag.PrivKey, Header.Tag.PrivSubKey]:
                o += self.pubkey_fields(pkt)

            if pkt.header.tag in [Header.Tag.PrivKey, Header.Tag.PrivSubKey]:
                o += self.privkey_fields(pkt)

            if pkt.header.tag == Header.Tag.Trust:
                o += "\tTrust - {trustbytes}\n".format(
                    trustbytes=self.bytefield(pkt.trust)
                )

            if pkt.header.tag == Header.Tag.UserID:
                o += "\tUser ID - {userid}\n".format(
                    userid=pkt.data.decode()
                )

            # because python 2.7 is stupid
            if bytes is str:
                o = o.decode()

            # add to self.out
            self.out.append(o[:-1])

    def pkt_ver(self, pkt):
        ##TODO: unhardcode this once PGPy can parse v3 packets
        o = "\tVer 4 - new\n" if pkt.version == 4 else ""
        return o

    def signature_fields(self, pkt):
        o = ""
        o += self.pkt_ver(pkt)
        o += "\tSig type - {tname}({thex}).\n".format(
            tname=str(pkt.type),
            thex="{:#04x}".format(pkt.type.value)
        )
        o += "\tPub alg - {paname}(pub {panum})\n".format(
            paname=str(pkt.key_algorithm),
            panum=pkt.key_algorithm.value
        )
        o += "\tHash alg - {haname}(hash {hanum})\n".format(
            haname=str(pkt.hash_algorithm),
            hanum=pkt.hash_algorithm.value
        )
        o += self.subpkt_fields(pkt)
        o += "\tHash left 2 bytes - {hash2}\n".format(
            hash2=self.bytefield(pkt.hash2)
        )
        o += self.mpi_fields(pkt)
        return o

    def pubkey_fields(self, pkt):
        o = ""
        o += self.pkt_ver(pkt)
        o += "\tPublic key creation time - {keycdate}\n".format(
            keycdate=pkt.key_creation.strftime("%a %b %d %H:%M:%S UTC %Y")
        )
        self.keycreation = calendar.timegm(pkt.key_creation.timetuple())
        o += "\tPub alg - {alg}(pub {algn})\n".format(
            alg=str(pkt.key_algorithm),
            algn=pkt.key_algorithm.value
        )
        o += self.mpi_fields(pkt)

        return o

    def privkey_fields(self, pkt):
        o = ""
        o += self.string2key_fields(pkt)
        if pkt.stokey.id == 0:
            o += self.mpi_fields(pkt, True)

        else:
            o += self.enc_mpi_fields(pkt)

        if pkt.stokey.id == 254:
            o += "\tEncrypted SHA1 hash\n"

        if pkt.stokey.id in [0, 255]:
            if bytes is str:
                chksum = ''.join('{:02x} '.format(ord(c)) for c in pkt.checksum)
            else:
                chksum = ''.join('{:02x} '.format(c) for c in pkt.checksum)
            o += "\tChecksum - {chksum}\n".format(chksum=chksum)

        return o

    def subpkt_fields(self, pkt):
        o = ""

        for st in [pkt.hashed_subpackets, pkt.unhashed_subpackets]:
            for sub in st.subpackets:
                o += "\t{hashed}Sub: {spname}(sub {spid})({splen} bytes)\n".format(
                    hashed="Hashed " if st.hashed else "",
                    spname=str(sub.type),
                    spid=sub.type.value,
                    splen=sub.length - 2
                )

                if sub.type == SubPacket.Type.SigCreationTime:
                    o += "\t\tTime - {date}\n".format(
                        date=sub.payload.strftime("%a %b %e %H:%M:%S UTC %Y")
                    )

                if sub.type == SubPacket.Type.KeyExpirationTime:
                    rt = datetime.utcfromtimestamp(self.keycreation + sub.payload)
                    o += "\t\tTime - {date}\n".format(
                        date=rt.strftime("%a %b %e %H:%M:%S UTC %Y")
                    )

                if sub.type == SubPacket.Type.Issuer:
                    o += "\t\tKey ID - 0x{keyid}\n".format(keyid=sub.payload.decode())

                if sub.type == SubPacket.Type.KeyFlags:
                    for flag in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(flag))

                if sub.type == SubPacket.Type.PreferredSymmetricAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tSym alg - {alg}(sym {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.PreferredHashAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tHash alg - {alg}(hash {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.PreferredCompressionAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tComp alg - {alg}(comp {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.KeyServerPreferences:
                    for pref in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(pref))

                if sub.type == SubPacket.Type.Features:
                    for feature in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(feature))

                if sub.type == SubPacket.Type.Revocable:
                    o += "\t\tRevocable - {rev}\n".format(
                        rev="Yes" if sub.payload else "No"
                    )

                if sub.type == SubPacket.Type.PolicyURL:
                    o += "\t\tURL - {url}\n".format(url=sub.payload.decode())
        return o

    def string2key_fields(self, pkt):
        o = ""
        if pkt.stokey.id in [254, 255]:
            o += "\tSym alg - {salg}(sym {salgn})\n".format(
                salg=str(pkt.stokey.alg),
                salgn=pkt.stokey.alg.value
            )

            o += "\t{stokeytype}(s2k {stokeytypenum}):\n".format(
                stokeytype=str(pkt.stokey.type),
                stokeytypenum=pkt.stokey.type.value
            )

            o += "\t\tHash alg - {hash}(hash {hashn})\n".format(
                hash=str(pkt.stokey.hash),
                hashn=pkt.stokey.hash.value
            )

            if pkt.stokey.type in [String2Key.Type.Salted, String2Key.Type.Iterated]:
                if bytes is str:
                    saltbytes = ''.join('{:02x} '.format(ord(c)) for c in pkt.stokey.salt)
                else:
                    saltbytes = ''.join('{:02x} '.format(c) for c in pkt.stokey.salt)
                o += "\t\tSalt - {saltbytes}\n".format(saltbytes=saltbytes)

            if pkt.stokey.type == String2Key.Type.Iterated:
                o += "\t\tCount - {count}(coded count {c})\n".format(
                    count=pkt.stokey.count,
                    c=pkt.stokey.c
                )

        if pkt.stokey.id != 0:
            if bytes is str:
                ivbytes = ''.join('{:02x} '.format(ord(c)) for c in pkt.stokey.iv)
            else:
                ivbytes = ''.join('{:02x} '.format(c) for c in pkt.stokey.iv)

            o += "\tIV - {ivbytes}\n".format(ivbytes=ivbytes)

        return o

    def mpi_fields(self, pkt, sec=False):
        o = ""

        if pkt.header.tag == Header.Tag.Signature:
            mpis = pkt.signature

        if pkt.header.tag in [Header.Tag.PubKey, Header.Tag.PubSubKey,
                              Header.Tag.PrivKey, Header.Tag.PrivSubKey] and not sec:
            mpis = pkt.key_material

        if pkt.header.tag in [Header.Tag.PrivKey, Header.Tag.PrivSubKey] and sec:
            mpis = pkt.seckey_material

        for mpi in mpis.fields.values():
            o += "\t{mname}({bitlen} bits) - {keybytes}\n".format(
                mname=mpi['name'],
                bitlen=mpi['bitlen'],
                keybytes=self.bytefield(mpi['bytes']),
            )

        # Only print the encoding if it's a signature packet
        if pkt.header.tag == Header.Tag.Signature:
            o += "\t\t-> {enc}\n".format(enc=mpis.encoding)

        return o

    def enc_mpi_fields(self, pkt):
        o = ""

        if pkt.key_algorithm == PubKeyAlgo.RSAEncryptOrSign:
            o += "\tEncrypted RSA d\n"
            o += "\tEncrypted RSA p\n"
            o += "\tEncrypted RSA q\n"
            o += "\tEncrypted RSA u\n"

        if pkt.key_algorithm == PubKeyAlgo.DSA:
            o += "\tEncrypted DSA x\n"

        if pkt.key_algorithm == PubKeyAlgo.ElGamal:
            o += "\tEncrypted ElGamal x\n"

        return o<commit_msg>typo fix<commit_after>""" pgpdump.py
"""
import calendar
from datetime import datetime

from .packet import Header, SubPacket, String2Key
from .packet import PubKeyAlgo

class PGPDumpFormat(object):
    @staticmethod
    def bytefield(f):
        o = ""

        # python 2.7
        if bytes is str:
            o += ''.join('{:02x} '.format(ord(c)) for c in f)

        # python 3.x
        else:
            o += ''.join('{:02x} '.format(c) for c in f)

        return o

    def __init__(self, pktobj):
        self.out = []
        self.keycreation = 0

        for pkt in pktobj.packets:
            o = ""
            # print the header line first
            o += "{hformat}: {pktname}(tag {tagnum})({length} bytes)\n".format(
                hformat="Old" if pkt.header.format == 0 else "New",
                pktname=pkt.name,
                tagnum=pkt.header.tag.value,
                length=pkt.header.length
            )

            # state machine time!
            if pkt.header.tag == Header.Tag.Signature:
                o += self.signature_fields(pkt)

            if pkt.header.tag in [Header.Tag.PubKey, Header.Tag.PubSubKey,
                                  Header.Tag.PrivKey, Header.Tag.PrivSubKey]:
                o += self.pubkey_fields(pkt)

            if pkt.header.tag in [Header.Tag.PrivKey, Header.Tag.PrivSubKey]:
                o += self.privkey_fields(pkt)

            if pkt.header.tag == Header.Tag.Trust:
                o += "\tTrust - {trustbytes}\n".format(
                    trustbytes=self.bytefield(pkt.trust)
                )

            if pkt.header.tag == Header.Tag.UserID:
                o += "\tUser ID - {userid}\n".format(
                    userid=pkt.data.decode()
                )

            # because python 2.7 is stupid
            if bytes is str:
                o = o.decode()

            # add to self.out
            self.out.append(o[:-1])

    def pkt_ver(self, pkt):
        ##TODO: unhardcode this once PGPy can parse v3 packets
        o = "\tVer 4 - new\n" if pkt.version == 4 else ""
        return o

    def signature_fields(self, pkt):
        o = ""
        o += self.pkt_ver(pkt)
        o += "\tSig type - {tname}({thex}).\n".format(
            tname=str(pkt.type),
            thex="{:#04x}".format(pkt.type.value)
        )
        o += "\tPub alg - {paname}(pub {panum})\n".format(
            paname=str(pkt.key_algorithm),
            panum=pkt.key_algorithm.value
        )
        o += "\tHash alg - {haname}(hash {hanum})\n".format(
            haname=str(pkt.hash_algorithm),
            hanum=pkt.hash_algorithm.value
        )
        o += self.subpkt_fields(pkt)
        o += "\tHash left 2 bytes - {hash2}\n".format(
            hash2=self.bytefield(pkt.hash2)
        )
        o += self.mpi_fields(pkt)
        return o

    def pubkey_fields(self, pkt):
        o = ""
        o += self.pkt_ver(pkt)
        o += "\tPublic key creation time - {keycdate}\n".format(
            keycdate=pkt.key_creation.strftime("%a %b %e %H:%M:%S UTC %Y")
        )
        self.keycreation = calendar.timegm(pkt.key_creation.timetuple())
        o += "\tPub alg - {alg}(pub {algn})\n".format(
            alg=str(pkt.key_algorithm),
            algn=pkt.key_algorithm.value
        )
        o += self.mpi_fields(pkt)

        return o

    def privkey_fields(self, pkt):
        o = ""
        o += self.string2key_fields(pkt)
        if pkt.stokey.id == 0:
            o += self.mpi_fields(pkt, True)

        else:
            o += self.enc_mpi_fields(pkt)

        if pkt.stokey.id == 254:
            o += "\tEncrypted SHA1 hash\n"

        if pkt.stokey.id in [0, 255]:
            if bytes is str:
                chksum = ''.join('{:02x} '.format(ord(c)) for c in pkt.checksum)
            else:
                chksum = ''.join('{:02x} '.format(c) for c in pkt.checksum)
            o += "\tChecksum - {chksum}\n".format(chksum=chksum)

        return o

    def subpkt_fields(self, pkt):
        o = ""

        for st in [pkt.hashed_subpackets, pkt.unhashed_subpackets]:
            for sub in st.subpackets:
                o += "\t{hashed}Sub: {spname}(sub {spid})({splen} bytes)\n".format(
                    hashed="Hashed " if st.hashed else "",
                    spname=str(sub.type),
                    spid=sub.type.value,
                    splen=sub.length - 2
                )

                if sub.type == SubPacket.Type.SigCreationTime:
                    o += "\t\tTime - {date}\n".format(
                        date=sub.payload.strftime("%a %b %e %H:%M:%S UTC %Y")
                    )

                if sub.type == SubPacket.Type.KeyExpirationTime:
                    rt = datetime.utcfromtimestamp(self.keycreation + sub.payload)
                    o += "\t\tTime - {date}\n".format(
                        date=rt.strftime("%a %b %e %H:%M:%S UTC %Y")
                    )

                if sub.type == SubPacket.Type.Issuer:
                    o += "\t\tKey ID - 0x{keyid}\n".format(keyid=sub.payload.decode())

                if sub.type == SubPacket.Type.KeyFlags:
                    for flag in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(flag))

                if sub.type == SubPacket.Type.PreferredSymmetricAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tSym alg - {alg}(sym {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.PreferredHashAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tHash alg - {alg}(hash {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.PreferredCompressionAlgorithms:
                    for alg in sub.payload:
                        o += "\t\tComp alg - {alg}(comp {algn})\n".format(
                            alg=str(alg),
                            algn=alg.value
                        )

                if sub.type == SubPacket.Type.KeyServerPreferences:
                    for pref in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(pref))

                if sub.type == SubPacket.Type.Features:
                    for feature in sub.payload:
                        o += "\t\tFlag - {flag}\n".format(flag=str(feature))

                if sub.type == SubPacket.Type.Revocable:
                    o += "\t\tRevocable - {rev}\n".format(
                        rev="Yes" if sub.payload else "No"
                    )

                if sub.type == SubPacket.Type.PolicyURL:
                    o += "\t\tURL - {url}\n".format(url=sub.payload.decode())
        return o

    def string2key_fields(self, pkt):
        o = ""
        if pkt.stokey.id in [254, 255]:
            o += "\tSym alg - {salg}(sym {salgn})\n".format(
                salg=str(pkt.stokey.alg),
                salgn=pkt.stokey.alg.value
            )

            o += "\t{stokeytype}(s2k {stokeytypenum}):\n".format(
                stokeytype=str(pkt.stokey.type),
                stokeytypenum=pkt.stokey.type.value
            )

            o += "\t\tHash alg - {hash}(hash {hashn})\n".format(
                hash=str(pkt.stokey.hash),
                hashn=pkt.stokey.hash.value
            )

            if pkt.stokey.type in [String2Key.Type.Salted, String2Key.Type.Iterated]:
                if bytes is str:
                    saltbytes = ''.join('{:02x} '.format(ord(c)) for c in pkt.stokey.salt)
                else:
                    saltbytes = ''.join('{:02x} '.format(c) for c in pkt.stokey.salt)
                o += "\t\tSalt - {saltbytes}\n".format(saltbytes=saltbytes)

            if pkt.stokey.type == String2Key.Type.Iterated:
                o += "\t\tCount - {count}(coded count {c})\n".format(
                    count=pkt.stokey.count,
                    c=pkt.stokey.c
                )

        if pkt.stokey.id != 0:
            if bytes is str:
                ivbytes = ''.join('{:02x} '.format(ord(c)) for c in pkt.stokey.iv)
            else:
                ivbytes = ''.join('{:02x} '.format(c) for c in pkt.stokey.iv)

            o += "\tIV - {ivbytes}\n".format(ivbytes=ivbytes)

        return o

    def mpi_fields(self, pkt, sec=False):
        o = ""

        if pkt.header.tag == Header.Tag.Signature:
            mpis = pkt.signature

        if pkt.header.tag in [Header.Tag.PubKey, Header.Tag.PubSubKey,
                              Header.Tag.PrivKey, Header.Tag.PrivSubKey] and not sec:
            mpis = pkt.key_material

        if pkt.header.tag in [Header.Tag.PrivKey, Header.Tag.PrivSubKey] and sec:
            mpis = pkt.seckey_material

        for mpi in mpis.fields.values():
            o += "\t{mname}({bitlen} bits) - {keybytes}\n".format(
                mname=mpi['name'],
                bitlen=mpi['bitlen'],
                keybytes=self.bytefield(mpi['bytes']),
            )

        # Only print the encoding if it's a signature packet
        if pkt.header.tag == Header.Tag.Signature:
            o += "\t\t-> {enc}\n".format(enc=mpis.encoding)

        return o

    def enc_mpi_fields(self, pkt):
        o = ""

        if pkt.key_algorithm == PubKeyAlgo.RSAEncryptOrSign:
            o += "\tEncrypted RSA d\n"
            o += "\tEncrypted RSA p\n"
            o += "\tEncrypted RSA q\n"
            o += "\tEncrypted RSA u\n"

        if pkt.key_algorithm == PubKeyAlgo.DSA:
            o += "\tEncrypted DSA x\n"

        if pkt.key_algorithm == PubKeyAlgo.ElGamal:
            o += "\tEncrypted ElGamal x\n"

        return o<|endoftext|>


<commit_before>"""
Predicate Directed Acyclic Graph.
"""

__author__ = "Eldar Abusalimov"
__date__ = "2012-11-29"

__all__ = [
    "Pdag",
    "PdagNode",
    "PdagContext",
    "PdagContextError",
    "And",
    "Or",
    "Not",
    "Atom",
]

import abc
from collections import namedtuple
from operator import attrgetter

import logs as log

class PdagContext(object):
    """
    Defines context protocol for interacting with Pdag nodes.
    """
    __metaclass__ = abc.ABCMeta
    __slots__ = ()

    @abc.abstractproperty
    def _dict(self):
        """
        Implementation must provide a dictionary-like object which is used as
        an internal storage of the context.
        """
        raise NotImplementedError

    def __getitem__(self, pnode):
        """
        Fetch the state of a given pnode in the current context.

        Returns:
            Tristate (None, True or False) indicating the current value of the
            specified pnode.

        Never raises KeyError.
        """
        try:
            return self._dict[pnode]
        except KeyError:
            return None

    def __setitem__(self, pnode, value):
        """
        Calls 'store' method with 'notify_pnode=True'.
        """
        self.store(pnode, value, notify_pnode=True)

    def store(self, pnode, value, notify_pnode=False):
        """
        Set value of a given pnode.

        Depending on a 'notify_pnode' argument, setting a *new* value may
        result in calling 'context_setting' on the target pnode which may in
        turn set values for other nodes.

        Args:
            pnode:
                Obviously, the target pnode to set the value for.
            value (bool):
                The value for the target pnode, which must not conflict with an
                existing value, if any. Otherwise a PdagContextError is raised.
                In other words, operation succeeds iff
                    ctx[pnode] is None or ctx[pnode] == value
            notify_pnode (bool):
                Tells whether to call 'PdagNode.context_setting' or not.

        Returns:
            An old value (which is the same as new one), if any,
            or None otherwise.

        Raises:
            PdagContextError:
                When another value has already been set for this pnode.
        """
        assert isinstance(value, bool)

        self_dict = self._dict

        try:
            old_value = self_dict[pnode]

        except KeyError:
            self_dict[pnode] = value

            if notify_pnode:
                pnode.context_setting(self, value)

        else:
            if old_value != value:
                raise PdagContextError

            return old_value

class Pdag(object):

    atoms = property(attrgetter('_atoms'))
    nodes = property(attrgetter('_nodes'))

    def __init__(self, *atoms):
        self._atoms = atoms = frozenset(atoms)
        for atom in atoms:
            if not isinstance(atom, Atom):
                raise TypeError
        self._nodes = frozenset(self._hull(atoms))

    @classmethod
    def _hull(cls, nodes):
        unvisited = set(nodes)
        visited = set()

        while unvisited:
            node = unvisited.pop()
            visited.add(node)
            outgoing = node._outgoing = frozenset(node._outgoing)
            unvisited |= node._outgoing - visited

        return visited

class PdagNode(object):
    """docstring for PdagNode"""
    __slots__ = '_outgoing'

    costs = (0, 0) # cost = pnode.costs[value] # value is either True or False

    def __init__(self):
        self._outgoing = set()

    def _new_incoming(self, incoming):
        incoming._outgoing.add(self)

    def _incoming_setting(self, incoming, ctx, value):
        """Here 'value' is either True or False."""
        raise NotImplementedError

    def context_setting(self, ctx, value):
        log.debug("pdag: outgoing: [%s]",
                  ', '.join(str(out.bind(ctx)) for out in self._outgoing))
        for out in self._outgoing:
            out._incoming_setting(self, ctx, value)

    def bind(self, ctx):
        return PnodeInContext(self, ctx)

class LatticeOp(PdagNode):
    """Associative, commutative and idempotent operation."""
    __slots__ = '_operands'

    def __init__(self, *operands):
        super(LatticeOp, self).__init__()

        self._operands = set()
        for operand in operands:
            self._operands.add(operand)
            self._new_incoming(operand)

    def _incoming_setting(self, incoming, ctx, value):
        with log.debug("pdag: %s: %s, operand %s", type(self).__name__,
                       self.bind(ctx), incoming.bind(ctx)):

            if value is self._zero:
                log.debug("pdag: operand value is zero")
                ctx.store(self, value)

            else:
                log.debug("pdag: operand value is identity")
                self._eval_operands(ctx)

    def context_setting(self, ctx, value):
        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):

            if value is self._identity:
                log.debug("pdag: new value is identity")
                for operand in self._operands:
                    ctx[operand] = value

            else:
                log.debug("pdag: new value is zero")
                self._eval_operands(ctx)

            super(LatticeOp, self).context_setting(ctx, value)

    def _eval_operands(self, ctx):
        with log.debug("pdag: %s: %s, evaluating: [%s]",
                       type(self).__name__, self.bind(ctx),
                       ', '.join(str(i.bind(ctx)) for i in self._operands)):

            zero = self._zero

            last_unset_operand = None
            for operand in self._operands:
                value = ctx[operand]

                if value is zero:
                    log.debug("pdag: operand value is zero")
                    ctx.store(self, zero)
                    return

                if value is None:
                    if last_unset_operand is not None:
                        log.debug("pdag: too many unset operands")
                        return
                    last_unset_operand = operand

            if last_unset_operand is None:
                log.debug("pdag: all operands are identity")
                ctx.store(self, self._identity)

            elif ctx[self] is zero:
                log.debug("pdag: sole unset operand left: %s",
                          last_unset_operand)
                ctx[last_unset_operand] = zero

            else:
                log.debug("pdag: sole operand left, but self value is not zero")

    # def __repr__(self):
    #     return self._repr_sign.join(map(repr, self._operands)).join('()')
    def __str__(self):
        return self._repr_sign.join(map(str, self._operands)).join('()')

class And(LatticeOp):
    _identity = True
    _zero     = False

    _repr_sign = '&'

class Or(LatticeOp):
    _identity = False
    _zero     = True

    _repr_sign = '|'

class Not(PdagNode):
    __slots__ = '_operand'

    def __init__(self, operand):
        super(Not, self).__init__()
        self._operand = operand
        self._new_incoming(operand)

    def _incoming_setting(self, incoming, ctx, value):
        assert incoming is self._operand

        with log.debug("pdag: %s: %s, operand %s", type(self).__name__,
                       self.bind(ctx), incoming.bind(ctx)):

            ctx.store(self, not value)

    def context_setting(self, ctx, value):
        assert self._operand is not None

        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):

            ctx[self._operand] = not value
            super(Not, self).context_setting(ctx, value)

    # def __repr__(self):
    #     return '~%r' % self._operand
    def __str__(self):
        return '~%s' % self._operand

class Atom(PdagNode):
    """To be extended by the client."""
    __slots__ = ()
    costs = (0, 1)

    def context_setting(self, ctx, value):
        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):
            super(Atom, self).context_setting(ctx, value)

class PnodeInContext(namedtuple('_PnodeInContext', 'node context')):
    def __str__(self):
        node = self.node
        value = self.context[node]
        return '%s=%s' % (node, value) if value is not None else str(node)

class PdagContextError(ValueError):
    """Raised by PdagContext on an attempt to set an inappropriate value."""

<commit_msg>first draft of Implies node (if-then).<commit_after>"""
Predicate Directed Acyclic Graph.
"""

__author__ = "Eldar Abusalimov"
__date__ = "2012-11-29"

__all__ = [
    "Pdag",
    "PdagNode",
    "PdagContext",
    "PdagContextError",
    "And",
    "Or",
    "Not",
    "Atom",
]

import abc
from collections import namedtuple
from operator import attrgetter

import logs as log

class PdagContext(object):
    """
    Defines context protocol for interacting with Pdag nodes.
    """
    __metaclass__ = abc.ABCMeta
    __slots__ = ()

    @abc.abstractproperty
    def _dict(self):
        """
        Implementation must provide a dictionary-like object which is used as
        an internal storage of the context.
        """
        raise NotImplementedError

    def __getitem__(self, pnode):
        """
        Fetch the state of a given pnode in the current context.

        Returns:
            Tristate (None, True or False) indicating the current value of the
            specified pnode.

        Never raises KeyError.
        """
        try:
            return self._dict[pnode]
        except KeyError:
            return None

    def __setitem__(self, pnode, value):
        """
        Calls 'store' method with 'notify_pnode=True'.
        """
        self.store(pnode, value, notify_pnode=True)

    def store(self, pnode, value, notify_pnode=False):
        """
        Set value of a given pnode.

        Depending on a 'notify_pnode' argument, setting a *new* value may
        result in calling 'context_setting' on the target pnode which may in
        turn set values for other nodes.

        Args:
            pnode:
                Obviously, the target pnode to set the value for.
            value (bool):
                The value for the target pnode, which must not conflict with an
                existing value, if any. Otherwise a PdagContextError is raised.
                In other words, operation succeeds iff
                    ctx[pnode] is None or ctx[pnode] == value
            notify_pnode (bool):
                Tells whether to call 'PdagNode.context_setting' or not.

        Returns:
            An old value (which is the same as new one), if any,
            or None otherwise.

        Raises:
            PdagContextError:
                When another value has already been set for this pnode.
        """
        assert isinstance(value, bool)

        self_dict = self._dict

        try:
            old_value = self_dict[pnode]

        except KeyError:
            self_dict[pnode] = value

            if notify_pnode:
                pnode.context_setting(self, value)

        else:
            if old_value != value:
                raise PdagContextError

            return old_value

class Pdag(object):

    atoms = property(attrgetter('_atoms'))
    nodes = property(attrgetter('_nodes'))

    def __init__(self, *atoms):
        self._atoms = atoms = frozenset(atoms)
        for atom in atoms:
            if not isinstance(atom, Atom):
                raise TypeError
        self._nodes = frozenset(self._hull(atoms))

    @classmethod
    def _hull(cls, nodes):
        unvisited = set(nodes)
        visited = set()

        while unvisited:
            node = unvisited.pop()
            visited.add(node)
            outgoing = node._outgoing = frozenset(node._outgoing)
            unvisited |= node._outgoing - visited

        return visited

class PdagNode(object):
    """docstring for PdagNode"""
    __slots__ = '_outgoing'

    costs = (0, 0) # cost = pnode.costs[value] # value is either True or False

    def __init__(self):
        self._outgoing = set()

    def _new_incoming(self, incoming):
        incoming._outgoing.add(self)

    def _incoming_setting(self, incoming, ctx, value):
        """Here 'value' is either True or False."""
        raise NotImplementedError

    def context_setting(self, ctx, value):
        log.debug("pdag: outgoing: [%s]",
                  ', '.join(str(out.bind(ctx)) for out in self._outgoing))
        for out in self._outgoing:
            out._incoming_setting(self, ctx, value)

    def bind(self, ctx):
        return PnodeInContext(self, ctx)

class LatticeOp(PdagNode):
    """Associative, commutative and idempotent operation."""
    __slots__ = '_operands'

    def __init__(self, *operands):
        super(LatticeOp, self).__init__()

        self._operands = set()
        for operand in operands:
            self._operands.add(operand)
            self._new_incoming(operand)

    def _incoming_setting(self, incoming, ctx, value):
        with log.debug("pdag: %s: %s, operand %s", type(self).__name__,
                       self.bind(ctx), incoming.bind(ctx)):

            if value is self._zero:
                log.debug("pdag: operand value is zero")
                ctx.store(self, value)

            else:
                log.debug("pdag: operand value is identity")
                self._eval_operands(ctx)

    def context_setting(self, ctx, value):
        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):

            if value is self._identity:
                log.debug("pdag: new value is identity")
                for operand in self._operands:
                    ctx[operand] = value

            else:
                log.debug("pdag: new value is zero")
                self._eval_operands(ctx)

            super(LatticeOp, self).context_setting(ctx, value)

    def _eval_operands(self, ctx):
        with log.debug("pdag: %s: %s, evaluating: [%s]",
                       type(self).__name__, self.bind(ctx),
                       ', '.join(str(i.bind(ctx)) for i in self._operands)):

            zero = self._zero

            last_unset_operand = None
            for operand in self._operands:
                value = ctx[operand]

                if value is zero:
                    log.debug("pdag: operand value is zero")
                    ctx.store(self, zero)
                    return

                if value is None:
                    if last_unset_operand is not None:
                        log.debug("pdag: too many unset operands")
                        return
                    last_unset_operand = operand

            if last_unset_operand is None:
                log.debug("pdag: all operands are identity")
                ctx.store(self, self._identity)

            elif ctx[self] is zero:
                log.debug("pdag: sole unset operand left: %s",
                          last_unset_operand)
                ctx[last_unset_operand] = zero

            else:
                log.debug("pdag: sole operand left, but self value is not zero")

    # def __repr__(self):
    #     return self._repr_sign.join(map(repr, self._operands)).join('()')
    def __str__(self):
        return self._repr_sign.join(map(str, self._operands)).join('()')

class And(LatticeOp):
    _identity = True
    _zero     = False

    _repr_sign = '&'

class Or(LatticeOp):
    _identity = False
    _zero     = True

    _repr_sign = '|'

class Not(PdagNode):
    __slots__ = '_operand'

    def __init__(self, operand):
        super(Not, self).__init__()
        self._operand = operand
        self._new_incoming(operand)

    def _incoming_setting(self, incoming, ctx, value):
        assert incoming is self._operand

        with log.debug("pdag: %s: %s, operand %s", type(self).__name__,
                       self.bind(ctx), incoming.bind(ctx)):

            ctx.store(self, not value)

    def context_setting(self, ctx, value):
        assert self._operand is not None

        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):

            ctx[self._operand] = not value
            super(Not, self).context_setting(ctx, value)

    # def __repr__(self):
    #     return '~%r' % self._operand
    def __str__(self):
        return '~%s' % self._operand

class Implies(PdagNode):
    __slots__ = '_if', '_then'

    def __init__(self, if_, then):
        super(Implies, self).__init__()

        self._if = if_
        self._new_incoming(if_)

        self._then = then
        self._new_incoming(then)

    def _incoming_setting(self, incoming, ctx, value):
        with log.debug("pdag: %s: %s, operand %s", type(self).__name__,
                       self.bind(ctx), incoming.bind(ctx)):

            if (incoming is self._if) == value:
                ctx.store(self, value)

    # def context_setting(self, ctx, value):
    #     assert self._operand is not None

    #     with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):

    #         ctx[self._operand] = not value
    #         super(Not, self).context_setting(ctx, value)

    # def __str__(self):
    #     return '~%s' % self._operand

class Atom(PdagNode):
    """To be extended by the client."""
    __slots__ = ()
    costs = (0, 1)

    def context_setting(self, ctx, value):
        with log.debug("pdag: %s: %s", type(self).__name__, self.bind(ctx)):
            super(Atom, self).context_setting(ctx, value)

class PnodeInContext(namedtuple('_PnodeInContext', 'node context')):
    def __str__(self):
        node = self.node
        value = self.context[node]
        return '%s=%s' % (node, value) if value is not None else str(node)

class PdagContextError(ValueError):
    """Raised by PdagContext on an attempt to set an inappropriate value."""

<|endoftext|>