<commit_before>#!/usr/bin/env python

# Copyright (C) 2017 SignalFx, Inc. All rights reserved.

import os
import sys
import unittest

sys.path.insert(0, os.path.join(
    os.path.dirname(os.path.abspath(__file__)), '..'))

# import the signalfx pyformance library
import signalfx.pyformance as pyf # noqa

class TestPyformance(unittest.TestCase):
    def tearDown(self):
        pyf.clear()

    def test_gauge(self):
        reg = pyf.MetricsRegistry()
        reg.gauge('test_gauge').set_value(1)
        reg.gauge('test_gauge_with_dim', default=3,
                  gauge_dim='hello_gauge').set_value(2)
        self.assertEqual(
            reg.metadata.get_metadata(
                'gauge_dim=hello_gauge.test_gauge_with_dim'),
            {
                'dimensions': {'gauge_dim': 'hello_gauge'},
                'metric': 'test_gauge_with_dim',
            })
        self.assertEqual(reg.dump_metrics(), {
                'test_gauge': {'value': 1},
                'gauge_dim=hello_gauge.test_gauge_with_dim': {'value': 2},
            }
        )
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_gauge(self):
        pyf.gauge('test_gauge').set_value(1)
        pyf.gauge('test_gauge_with_dim', default=3,
                  gauge_dim='hello_gauge').set_value(2)
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'gauge_dim=hello_gauge.test_gauge_with_dim'),
            {
                'dimensions': {'gauge_dim': 'hello_gauge'},
                'metric': 'test_gauge_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
                'test_gauge': {'value': 1},
                'gauge_dim=hello_gauge.test_gauge_with_dim': {'value': 2},
            }
        )

    def test_counter(self):
        reg = pyf.MetricsRegistry()
        reg.counter('test_counter').inc()
        reg.counter('test_counter_with_dim',
                    counter_dim='hello_counter').inc()
        self.assertEqual(
            reg.metadata.get_metadata(
                'counter_dim=hello_counter.test_counter_with_dim'),
            {
                'dimensions': {'counter_dim': 'hello_counter'},
                'metric': 'test_counter_with_dim',
            })
        self.assertEqual(reg.dump_metrics(), {
            'test_counter': {'count': 1},
            'counter_dim=hello_counter.test_counter_with_dim': {'count': 1},
        })
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_counter(self):
        pyf.counter('test_counter').inc()
        pyf.counter('test_counter_with_dim',
                    counter_dim='hello_counter').inc()
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'counter_dim=hello_counter.test_counter_with_dim'),
            {
                'dimensions': {'counter_dim': 'hello_counter'},
                'metric': 'test_counter_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'test_counter': {'count': 1},
            'counter_dim=hello_counter.test_counter_with_dim': {'count': 1},
        })

    def test_counter_decorator(self):
        @pyf.count_calls
        def callme():
            pass

        @pyf.count_calls_with_dims(counter_dim='hello_counter')
        def callme_with_dims():
            pass

        callme()
        callme_with_dims()
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'counter_dim=hello_counter.callme_with_dims_calls'),
            {
                'dimensions': {'counter_dim': 'hello_counter'},
                'metric': 'callme_with_dims_calls',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'callme_calls': {'count': 1},
            'counter_dim=hello_counter.callme_with_dims_calls': {'count': 1},
        })

    def test_histogram(self):
        reg = pyf.MetricsRegistry()
        h1 = reg.histogram('test_histogram')
        h1.add(1)
        h1.add(1)
        h1.add(1)
        h2 = reg.histogram('test_histogram_with_dim',
                           histogram_dim='hello_histogram')
        h2.add(1)
        h2.add(1)
        h2.add(1)
        self.assertEqual(
            reg.metadata.get_metadata(
                'histogram_dim=hello_histogram.test_histogram_with_dim'),
            {
                'dimensions': {'histogram_dim': 'hello_histogram'},
                'metric': 'test_histogram_with_dim',
            })

        metrics = reg.dump_metrics()
        self.assertEqual(metrics, {
            'test_histogram': {'count': 3, '999_percentile': 1,
                               '99_percentile': 1, 'min': 1,
                               '95_percentile': 1, '75_percentile': 1,
                               'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.test_histogram_with_dim':
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
             '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
             'max': 1, 'avg': 1.0},
        })
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_histogram(self):
        h1 = pyf.histogram('test_histogram')
        h1.add(1)
        h1.add(1)
        h1.add(1)
        h2 = pyf.histogram('test_histogram_with_dim',
                           histogram_dim='hello_histogram')
        h2.add(1)
        h2.add(1)
        h2.add(1)
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'histogram_dim=hello_histogram.test_histogram_with_dim'),
            {
                'dimensions': {'histogram_dim': 'hello_histogram'},
                'metric': 'test_histogram_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'test_histogram': {'count': 3, '999_percentile': 1,
                               '99_percentile': 1, 'min': 1,
                               '95_percentile': 1, '75_percentile': 1,
                               'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.test_histogram_with_dim':
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
             '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
             'max': 1, 'avg': 1.0},
        })

    def test_histogram_decorator(self):
        @pyf.hist_calls
        def callme():
            return 1

        @pyf.hist_calls_with_dims(histogram_dim='hello_histogram')
        def callme_with_dims():
            return 1

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'histogram_dim=hello_histogram.callme_with_dims_calls'),
            {
                'dimensions': {'histogram_dim': 'hello_histogram'},
                'metric': 'callme_with_dims_calls',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'callme_calls': {'count': 3, '999_percentile': 1,
                             '99_percentile': 1, 'min': 1,
                             '95_percentile': 1, '75_percentile': 1,
                             'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.callme_with_dims_calls':
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
             '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
             'max': 1, 'avg': 1.0},
        })

    def test_meter(self):
        reg = pyf.MetricsRegistry()
        reg.meter('test_meter')
        reg.meter('test_meter_with_dim',
                  meter_dim='hello_meter')
        self.assertEqual(
            reg.metadata.get_metadata(
                'meter_dim=hello_meter.test_meter_with_dim'),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': 'test_meter_with_dim',
            })
        self.assertEqual(len(reg.dump_metrics()), 2)
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_meter(self):
        pyf.meter('test_meter')
        pyf.meter('test_meter_with_dim', meter_dim='hello_meter')
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'meter_dim=hello_meter.test_meter_with_dim'),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': 'test_meter_with_dim',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_meter_decorator(self):
        @pyf.meter_calls
        def callme():
            return 1

        @pyf.meter_calls_with_dims(meter_dim='hello_meter')
        def callme_with_dims():
            return 1

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()

        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'meter_dim=hello_meter.callme_with_dims_calls'),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': 'callme_with_dims_calls',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_timer(self):
        reg = pyf.MetricsRegistry()
        reg.timer('test_timer')
        reg.timer('test_timer_with_dim',
                  timer_dim='hello_timer')
        self.assertEqual(
            reg.metadata.get_metadata(
                'timer_dim=hello_timer.test_timer_with_dim'),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': 'test_timer_with_dim',
            })
        self.assertEqual(len(reg.dump_metrics()), 2)
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_timer(self):
        pyf.timer('test_timer')
        pyf.timer('test_timer_with_dim', timer_dim='hello_timer')
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'timer_dim=hello_timer.test_timer_with_dim'),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': 'test_timer_with_dim',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_timer_decorator(self):
        @pyf.time_calls
        def callme():
            return 1

        @pyf.time_calls_with_dims(timer_dim='hello_timer')
        def callme_with_dims():
            return 1

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()

        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'timer_dim=hello_timer.callme_with_dims_calls'),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': 'callme_with_dims_calls',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

if __name__ == '__main__':
    unittest.main()
<commit_msg>fix pyformance unittests for python3<commit_after>#!/usr/bin/env python

# Copyright (C) 2017 SignalFx, Inc. All rights reserved.

from pyformance.registry import get_qualname
import os
import sys
import unittest

sys.path.insert(0, os.path.join(
    os.path.dirname(os.path.abspath(__file__)), '..'))

# import the signalfx pyformance library
import signalfx.pyformance as pyf # noqa

class TestPyformance(unittest.TestCase):
    def tearDown(self):
        pyf.clear()

    def test_gauge(self):
        reg = pyf.MetricsRegistry()
        reg.gauge('test_gauge').set_value(1)
        reg.gauge('test_gauge_with_dim', default=3,
                  gauge_dim='hello_gauge').set_value(2)
        self.assertEqual(
            reg.metadata.get_metadata(
                'gauge_dim=hello_gauge.test_gauge_with_dim'),
            {
                'dimensions': {'gauge_dim': 'hello_gauge'},
                'metric': 'test_gauge_with_dim',
            })
        self.assertEqual(reg.dump_metrics(), {
                'test_gauge': {'value': 1},
                'gauge_dim=hello_gauge.test_gauge_with_dim': {'value': 2},
            }
        )
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_gauge(self):
        pyf.gauge('test_gauge').set_value(1)
        pyf.gauge('test_gauge_with_dim', default=3,
                  gauge_dim='hello_gauge').set_value(2)
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'gauge_dim=hello_gauge.test_gauge_with_dim'),
            {
                'dimensions': {'gauge_dim': 'hello_gauge'},
                'metric': 'test_gauge_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
                'test_gauge': {'value': 1},
                'gauge_dim=hello_gauge.test_gauge_with_dim': {'value': 2},
            }
        )

    def test_counter(self):
        reg = pyf.MetricsRegistry()
        reg.counter('test_counter').inc()
        reg.counter('test_counter_with_dim',
                    counter_dim='hello_counter').inc()
        self.assertEqual(
            reg.metadata.get_metadata(
                'counter_dim=hello_counter.test_counter_with_dim'),
            {
                'dimensions': {'counter_dim': 'hello_counter'},
                'metric': 'test_counter_with_dim',
            })
        self.assertEqual(reg.dump_metrics(), {
            'test_counter': {'count': 1},
            'counter_dim=hello_counter.test_counter_with_dim': {'count': 1},
        })
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_counter(self):
        pyf.counter('test_counter').inc()
        pyf.counter('test_counter_with_dim',
                    counter_dim='hello_counter').inc()
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'counter_dim=hello_counter.test_counter_with_dim'),
            {
                'dimensions': {'counter_dim': 'hello_counter'},
                'metric': 'test_counter_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'test_counter': {'count': 1},
            'counter_dim=hello_counter.test_counter_with_dim': {'count': 1},
        })

    def test_counter_decorator(self):
        @pyf.count_calls
        def callme():
            pass

        qcallme = get_qualname(callme)

        @pyf.count_calls_with_dims(counter_dim='hello_counter')
        def callme_with_dims():
            pass

        qcallme_with_dims = get_qualname(callme_with_dims)

        callme()
        callme_with_dims()
        if sys.version_info[0] < 3:
            self.assertEqual(
                pyf.global_registry().metadata.get_metadata(
                    'counter_dim=hello_counter.{0}_calls'.format(
                        qcallme_with_dims)),
                {
                    'dimensions': {'counter_dim': 'hello_counter'},
                    'metric': '{0}_calls'.format(qcallme_with_dims),
                })
            self.assertEqual(pyf.dump_metrics(), {
                '{0}_calls'.format(qcallme): {'count': 1},
                'counter_dim=hello_counter.{0}_calls'.format(
                    qcallme_with_dims):
                {'count': 1},
            })

    def test_histogram(self):
        reg = pyf.MetricsRegistry()
        h1 = reg.histogram('test_histogram')
        h1.add(1)
        h1.add(1)
        h1.add(1)
        h2 = reg.histogram('test_histogram_with_dim',
                           histogram_dim='hello_histogram')
        h2.add(1)
        h2.add(1)
        h2.add(1)

        metrics = reg.dump_metrics()
        self.assertEqual(metrics, {
            'test_histogram': {'count': 3, '999_percentile': 1,
                               '99_percentile': 1, 'min': 1,
                               '95_percentile': 1, '75_percentile': 1,
                               'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.test_histogram_with_dim':
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
             '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
             'max': 1, 'avg': 1.0},
        })
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_histogram(self):
        h1 = pyf.histogram('test_histogram')
        h1.add(1)
        h1.add(1)
        h1.add(1)
        h2 = pyf.histogram('test_histogram_with_dim',
                           histogram_dim='hello_histogram')
        h2.add(1)
        h2.add(1)
        h2.add(1)
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'histogram_dim=hello_histogram.test_histogram_with_dim'),
            {
                'dimensions': {'histogram_dim': 'hello_histogram'},
                'metric': 'test_histogram_with_dim',
            })
        self.assertEqual(pyf.dump_metrics(), {
            'test_histogram': {'count': 3, '999_percentile': 1,
                               '99_percentile': 1, 'min': 1,
                               '95_percentile': 1, '75_percentile': 1,
                               'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.test_histogram_with_dim':
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
             '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
             'max': 1, 'avg': 1.0},
        })

    def test_histogram_decorator(self):
        @pyf.hist_calls
        def callme():
            return 1

        qcallme = get_qualname(callme)

        @pyf.hist_calls_with_dims(histogram_dim='hello_histogram')
        def callme_with_dims():
            return 1

        qcallme_with_dims = get_qualname(callme_with_dims)

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()

        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'histogram_dim=hello_histogram.{0}_calls'.format(
                    qcallme_with_dims)),
            {
                'dimensions': {'histogram_dim': 'hello_histogram'},
                'metric': '{0}_calls'.format(qcallme_with_dims),
            })
        self.assertEqual(pyf.dump_metrics(), {
            '{0}_calls'.format(qcallme): {
                'count': 3, '999_percentile': 1,
                '99_percentile': 1, 'min': 1,
                '95_percentile': 1, '75_percentile': 1,
                'std_dev': 0.0, 'max': 1, 'avg': 1.0},
            'histogram_dim=hello_histogram.{0}_calls'.format(
                qcallme_with_dims):
            {'count': 3, '999_percentile': 1, '99_percentile': 1, 'min': 1,
                '95_percentile': 1, '75_percentile': 1, 'std_dev': 0.0,
                'max': 1, 'avg': 1.0},
        })

    def test_meter(self):
        reg = pyf.MetricsRegistry()
        reg.meter('test_meter')
        reg.meter('test_meter_with_dim',
                  meter_dim='hello_meter')
        self.assertEqual(
            reg.metadata.get_metadata(
                'meter_dim=hello_meter.test_meter_with_dim'),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': 'test_meter_with_dim',
            })
        self.assertEqual(len(reg.dump_metrics()), 2)
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_meter(self):
        pyf.meter('test_meter')
        pyf.meter('test_meter_with_dim', meter_dim='hello_meter')
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'meter_dim=hello_meter.test_meter_with_dim'),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': 'test_meter_with_dim',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_meter_decorator(self):
        @pyf.meter_calls
        def callme():
            return 1

        @pyf.meter_calls_with_dims(meter_dim='hello_meter')
        def callme_with_dims():
            return 1

        qcallme_with_dims = get_qualname(callme_with_dims)

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()

        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'meter_dim=hello_meter.{0}_calls'.format(qcallme_with_dims)),
            {
                'dimensions': {'meter_dim': 'hello_meter'},
                'metric': '{0}_calls'.format(qcallme_with_dims),
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_timer(self):
        reg = pyf.MetricsRegistry()
        reg.timer('test_timer')
        reg.timer('test_timer_with_dim',
                  timer_dim='hello_timer')
        self.assertEqual(
            reg.metadata.get_metadata(
                'timer_dim=hello_timer.test_timer_with_dim'),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': 'test_timer_with_dim',
            })
        self.assertEqual(len(reg.dump_metrics()), 2)
        reg.clear()
        self.assertEqual(reg.dump_metrics(), {})
        self.assertEqual(len(reg.metadata._metadata), 0)

    def test_global_timer(self):
        pyf.timer('test_timer')
        pyf.timer('test_timer_with_dim', timer_dim='hello_timer')
        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'timer_dim=hello_timer.test_timer_with_dim'),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': 'test_timer_with_dim',
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

    def test_timer_decorator(self):
        @pyf.time_calls
        def callme():
            return 1

        @pyf.time_calls_with_dims(timer_dim='hello_timer')
        def callme_with_dims():
            return 1

        qcallme_with_dims = get_qualname(callme_with_dims)

        callme()
        callme()
        callme()
        callme_with_dims()
        callme_with_dims()
        callme_with_dims()

        self.assertEqual(
            pyf.global_registry().metadata.get_metadata(
                'timer_dim=hello_timer.{0}_calls'.format(qcallme_with_dims)),
            {
                'dimensions': {'timer_dim': 'hello_timer'},
                'metric': '{0}_calls'.format(qcallme_with_dims),
            })
        self.assertEqual(len(pyf.dump_metrics()), 2)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>
from django.db import connection
from south.db import generic

class DatabaseOperations(generic.DatabaseOperations):

    """
    SQLite3 implementation of database operations.
    """

    # SQLite ignores foreign key constraints. I wish I could.
    supports_foreign_keys = False

    # You can't add UNIQUE columns with an ALTER TABLE.
    def add_column(self, table_name, name, field, *args, **kwds):
        # Run ALTER TABLE with no unique column
        unique, field._unique, field.db_index = field.unique, False, False
        generic.DatabaseOperations.add_column(self, table_name, name, field, *args, **kwds)
        # If it _was_ unique, make an index on it.
        if unique:
            self.create_index(table_name, [name], unique=True)

    # SQLite doesn't have ALTER COLUMN
    def alter_column(self, table_name, name, field, explicit_name=True):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support altering columns.")

    # Nor DROP COLUMN
    def delete_column(self, table_name, name):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support deleting columns.")

    # Nor RENAME COLUMN
    def rename_column(self, table_name, old, new):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support renaming columns.")<commit_msg>Fix delete_table for sqlite.<commit_after>
from django.db import connection
from south.db import generic

class DatabaseOperations(generic.DatabaseOperations):

    """
    SQLite3 implementation of database operations.
    """

    # SQLite ignores foreign key constraints. I wish I could.
    supports_foreign_keys = False

    # You can't add UNIQUE columns with an ALTER TABLE.
    def add_column(self, table_name, name, field, *args, **kwds):
        # Run ALTER TABLE with no unique column
        unique, field._unique, field.db_index = field.unique, False, False
        generic.DatabaseOperations.add_column(self, table_name, name, field, *args, **kwds)
        # If it _was_ unique, make an index on it.
        if unique:
            self.create_index(table_name, [name], unique=True)

    # SQLite doesn't have ALTER COLUMN
    def alter_column(self, table_name, name, field, explicit_name=True):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support altering columns.")

    # Nor DROP COLUMN
    def delete_column(self, table_name, name):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support deleting columns.")

    # Nor RENAME COLUMN
    def rename_column(self, table_name, old, new):
        """
        Not supported under SQLite.
        """
        raise NotImplementedError("SQLite does not support renaming columns.")

    # No cascades on deletes
    def delete_table(self, table_name, cascade=True):
        generic.DatabaseOperations.delete_table(self, table_name, False)
    <|endoftext|>


<commit_before>__author__ = "Felipe Zapata"

__all__ = ["calculate_mos", "create_dict_CGFs", "create_point_folder",
           "split_file_geometries"]

# ================> Python Standard  and third-party <==========
from collections import namedtuple
from noodles import (gather, schedule)
from os.path import join

import fnmatch
import h5py
import logging
import os

# ==================> Internal modules <==========
from nac.basisSet.basisNormalization import createNormalizedCGFs
from nac.schedule.scheduleCp2k import prepare_job_cp2k
from qmworks.common import InputKey
from qmworks.hdf5 import dump_to_hdf5
from qmworks.hdf5.quantumHDF5 import (cp2k2hdf5, turbomole2hdf5)
from qmworks.utils import chunksOf
from qmworks.warnings_qmworks import SCF_Convergence_Warning

# Type Hints
from typing import (Dict, List, Tuple)

# ==============================<>=========================
# Tuple contanining file paths
JobFiles = namedtuple("JobFiles", ("get_xyz", "get_inp", "get_out", "get_MO"))

# Starting logger
logger = logging.getLogger(__name__)
# ==============================> Tasks <=====================================

def calculate_mos(package_name, all_geometries, project_name, path_hdf5,
                  folders, package_args, guess_args=None,
                  calc_new_wf_guess_on_points=None, enumerate_from=0,
                  package_config=None):
    """
    Look for the MO in the HDF5 file if they do not exists calculate them by
    splitting the jobs in batches given by the ``restart_chunk`` variables.
    Only the first job is calculated from scratch while the rest of the
    batch uses as guess the wave function of the first calculation in
    the batch.

    :param all_geometries: list of molecular geometries
    :type all_geometries: String list
    :param project_name: Name of the project used as root path for storing
    data in HDF5.
    :type project_name: String
    :param path_hdf5: Path to the HDF5 file that contains the
    numerical results.
    :type path_hdf5: String
    :param folders: path to the directories containing the MO outputs
    :type folders: String list
    :param package_args: Settings for the job to run.
    :type package_args: Settings
    :param calc_new_wf_guess_on_points: Calculate a new Wave function guess in
    each of the geometries indicated. By Default only an initial guess is
    computed.
    :type calc_new_wf_guess_on_points: [Int]
    :param enumerate_from: Number from where to start enumerating the folders
    create for each point in the MD
    :type enumerate_from: Int
    :returns: path to nodes in the HDF5 file to MO energies
              and MO coefficients.
    """
    def search_data_in_hdf5(xs):
        """
        Search if the node exists in the HDF5 file.
        """
        if os.path.exists(path_hdf5):
            with h5py.File(path_hdf5, 'r') as f5:
                if isinstance(xs, list):
                    return all(path in f5 for path in xs)
                else:
                    return xs in f5
        else:
            return False

    # First calculation has no initial guess
    guess_job = None

    # calculate the rest of the job using the previous point as initial guess
    orbitals = []  # list to the nodes in the HDF5 containing the MOs
    for j, gs in enumerate(all_geometries):

        # number of the point with respect to all the trajectory
        k = j + enumerate_from

        # Path where the MOs will be store in the HDF5
        root = join(project_name, 'point_{}'.format(k), package_name, 'mo')
        hdf5_orb_path = [join(root, 'eigenvalues'), join(root, 'coefficients')]

        # If the MOs are already store in the HDF5 format return the path
        # to them and skip the calculation
        if search_data_in_hdf5(hdf5_orb_path):
            logger.info("point_{} has already been calculated".format(k))
            orbitals.append(hdf5_orb_path)
        else:
            logger.info("Computing Molecular orbitals of: point_{}".format(k))

            # Path to I/O files
            point_dir = folders[j]
            job_files = create_file_names(point_dir, k)
            job_name = 'point_{}'.format(k)

            # Compute the MOs and return a new guess
            promise_qm = compute_orbitals(
                guess_job, package_name, project_name, path_hdf5,
                package_args, guess_args, package_config,
                calc_new_wf_guess_on_points, point_dir, job_files, k, gs)

            # Check if the job finishes succesfully
            promise_qm = schedule_check(
                promise_qm, job_name, package_name, project_name, path_hdf5,
                package_args, guess_args, package_config, point_dir, job_files, k, gs)

            # Store the computation
            path_MOs = store_in_hdf5(project_name, path_hdf5, promise_qm,
                                     hdf5_orb_path, job_name)

            guess_job = promise_qm
            orbitals.append(path_MOs)

    return gather(*orbitals)

@schedule
def store_in_hdf5(project_name: str, path_hdf5: str, promise_qm,
                  node_paths: str, job_name: str) -> None:
    """
    Store the MOs in the HDF5
    """
    # Molecular Orbitals
    mos = promise_qm.orbitals
    if mos is not None:
        # Store in the HDF5
        try:
            with h5py.File(path_hdf5, 'r+') as f5:
                dump_to_hdf5(
                    mos, 'cp2k', f5, project_name=project_name, job_name=job_name)
        # Remove the ascii MO file
        finally:
            work_dir = promise_qm.archive['work_dir']
            path_MOs = fnmatch.filter(os.listdir(work_dir), 'mo_*MOLog')
            os.remove(path_MOs)

    return node_paths

def compute_orbitals(
        guess_job, package_name: str, project_name: str, path_hdf5: str,
        package_args: Dict, guess_args: Dict, package_config: Dict,
        calc_new_wf_guess_on_points: List, point_dir: str, job_files: Tuple,
        k: int, gs: List):
    """
    Call a Quantum chemisty package to compute the MOs required to calculate
    the nonadiabatic coupling. When finish store the MOs in the HdF5 and
    returns a new guess.
    """
    prepare_and_schedule = {'cp2k': prepare_job_cp2k}

    call_schedule_qm = prepare_and_schedule[package_name]

    job_files = create_file_names(point_dir, k)

    # Calculating initial guess
    compute_guess = calc_new_wf_guess_on_points is not None

    # A job  is a restart if guess_job is None and the list of
    # wf guesses are not empty
    is_restart = guess_job is None and compute_guess

    pred = (k in calc_new_wf_guess_on_points) or is_restart

    if pred:
        guess_job = call_schedule_qm(
            gs, job_files, guess_args,
            k, point_dir, wfn_restart_job=guess_job,
            package_config=package_config)

    promise_qm = call_schedule_qm(
        gs, job_files, package_args,
        k, point_dir, wfn_restart_job=guess_job,
        package_config=package_config)

    return promise_qm

@schedule
def schedule_check(promise_qm, job_name: str, package_name: str,
                   project_name: str, path_hdf5: str, package_args: Dict,
                   guess_args: Dict, package_config: Dict, point_dir: str,
                   job_files: Tuple, k: int, gs: List):
    """
    Check wether a calculation finishes succesfully otherwise run a new guess.
    """
    # Warnings of the computation
    warnings = promise_qm.warnings

    # Check for SCF convergence errors
    if warnings is not None and any(
            w == SCF_Convergence_Warning for msg, w in warnings.items()):
        # Report the failure
        msg = "Job: {} Finished with Warnings: {}".format(job_name, warnings)
        logger.warning(msg)

        # recompute a new guess
        msg1 = "Computing a new wave function guess for job: {}".format(job_name)
        logger.warning(msg1)

        # Remove the previous ascii file containing the MOs
        msg2 = "removing file containig the previous failed MOs of {}".format(job_name)
        logger.warning(msg2)
        path = fnmatch.filter(os.listdir(point_dir), 'mo*MOLog')[0]
        os.remove(join(point_dir, path))

        # Compute new guess at point k
        calc_new_wf_guess_on_points = [k]
        return compute_orbitals(
            None, package_name, project_name, path_hdf5,
            package_args, guess_args, package_config,
            calc_new_wf_guess_on_points, point_dir, job_files, k, gs)
    else:
        return promise_qm

def create_point_folder(work_dir, n, enumerate_from):
    """
    Create a new folder for each point in the MD trajectory.

    :returns: Paths lists.
    """
    folders = []
    for k in range(enumerate_from, n + enumerate_from):
        new_dir = join(work_dir, 'point_{}'.format(k))
        if not os.path.exists(new_dir):
            os.makedirs(new_dir)
        folders.append(new_dir)

    return folders

def split_file_geometries(pathXYZ):
    """
    Reads a set of molecular geometries in xyz format and returns
    a list of string, where is element a molecular geometry

    :returns: String list containing the molecular geometries.
    """
    # Read Cartesian Coordinates
    with open(pathXYZ) as f:
        xss = f.readlines()

    numat = int(xss[0].split()[0])
    return list(map(''.join, chunksOf(xss, numat + 2)))

def create_dict_CGFs(path_hdf5, basisname, xyz, package_name='cp2k',
                     package_config=None):
    """
    Try to read the basis from the HDF5 otherwise read it from a file and store
    it in the HDF5 file. Finally, it reads the basis Set from HDF5 and
    calculate the CGF for each atom.

    :param path_hdf5: Path to the HDF5 file that contains the
    numerical results.
    type path_hdf5: String
    :param basisname: Name of the Gaussian basis set.
    :type basisname: String
    :param xyz: List of Atoms.
    :type xyz: [nac.common.AtomXYZ]
    """
    functions = {'cp2k': cp2k2hdf5, 'turbomole': turbomole2hdf5}

    basis_location = join(package_name, 'basis')
    with h5py.File(path_hdf5) as f5:
        if basis_location not in f5:
            # Search Path to the file containing the basis set
            pathBasis = package_config["basis"]
            keyBasis = InputKey("basis", [pathBasis])
            # Store the basis sets
            functions[package_name](f5, [keyBasis])

        return createNormalizedCGFs(f5, basisname, package_name, xyz)

def create_file_names(work_dir, i):
    """
    Creates a namedTuple with the name of the 4 files used
    for each point in the trajectory

    :returns: Namedtuple containing the IO files
    """
    file_xyz = join(work_dir, 'coordinates_{}.xyz'.format(i))
    file_inp = join(work_dir, 'point_{}.inp'.format(i))
    file_out = join(work_dir, 'point_{}.out'.format(i))
    file_MO = join(work_dir, 'mo_coeff_{}.out'.format(i))

    return JobFiles(file_xyz, file_inp, file_out, file_MO)
<commit_msg>remove the first element of the list<commit_after>__author__ = "Felipe Zapata"

__all__ = ["calculate_mos", "create_dict_CGFs", "create_point_folder",
           "split_file_geometries"]

# ================> Python Standard  and third-party <==========
from collections import namedtuple
from noodles import (gather, schedule)
from os.path import join

import fnmatch
import h5py
import logging
import os

# ==================> Internal modules <==========
from nac.basisSet.basisNormalization import createNormalizedCGFs
from nac.schedule.scheduleCp2k import prepare_job_cp2k
from qmworks.common import InputKey
from qmworks.hdf5 import dump_to_hdf5
from qmworks.hdf5.quantumHDF5 import (cp2k2hdf5, turbomole2hdf5)
from qmworks.utils import chunksOf
from qmworks.warnings_qmworks import SCF_Convergence_Warning

# Type Hints
from typing import (Dict, List, Tuple)

# ==============================<>=========================
# Tuple contanining file paths
JobFiles = namedtuple("JobFiles", ("get_xyz", "get_inp", "get_out", "get_MO"))

# Starting logger
logger = logging.getLogger(__name__)
# ==============================> Tasks <=====================================

def calculate_mos(package_name, all_geometries, project_name, path_hdf5,
                  folders, package_args, guess_args=None,
                  calc_new_wf_guess_on_points=None, enumerate_from=0,
                  package_config=None):
    """
    Look for the MO in the HDF5 file if they do not exists calculate them by
    splitting the jobs in batches given by the ``restart_chunk`` variables.
    Only the first job is calculated from scratch while the rest of the
    batch uses as guess the wave function of the first calculation in
    the batch.

    :param all_geometries: list of molecular geometries
    :type all_geometries: String list
    :param project_name: Name of the project used as root path for storing
    data in HDF5.
    :type project_name: String
    :param path_hdf5: Path to the HDF5 file that contains the
    numerical results.
    :type path_hdf5: String
    :param folders: path to the directories containing the MO outputs
    :type folders: String list
    :param package_args: Settings for the job to run.
    :type package_args: Settings
    :param calc_new_wf_guess_on_points: Calculate a new Wave function guess in
    each of the geometries indicated. By Default only an initial guess is
    computed.
    :type calc_new_wf_guess_on_points: [Int]
    :param enumerate_from: Number from where to start enumerating the folders
    create for each point in the MD
    :type enumerate_from: Int
    :returns: path to nodes in the HDF5 file to MO energies
              and MO coefficients.
    """
    def search_data_in_hdf5(xs):
        """
        Search if the node exists in the HDF5 file.
        """
        if os.path.exists(path_hdf5):
            with h5py.File(path_hdf5, 'r') as f5:
                if isinstance(xs, list):
                    return all(path in f5 for path in xs)
                else:
                    return xs in f5
        else:
            return False

    # First calculation has no initial guess
    guess_job = None

    # calculate the rest of the job using the previous point as initial guess
    orbitals = []  # list to the nodes in the HDF5 containing the MOs
    for j, gs in enumerate(all_geometries):

        # number of the point with respect to all the trajectory
        k = j + enumerate_from

        # Path where the MOs will be store in the HDF5
        root = join(project_name, 'point_{}'.format(k), package_name, 'mo')
        hdf5_orb_path = [join(root, 'eigenvalues'), join(root, 'coefficients')]

        # If the MOs are already store in the HDF5 format return the path
        # to them and skip the calculation
        if search_data_in_hdf5(hdf5_orb_path):
            logger.info("point_{} has already been calculated".format(k))
            orbitals.append(hdf5_orb_path)
        else:
            logger.info("Computing Molecular orbitals of: point_{}".format(k))

            # Path to I/O files
            point_dir = folders[j]
            job_files = create_file_names(point_dir, k)
            job_name = 'point_{}'.format(k)

            # Compute the MOs and return a new guess
            promise_qm = compute_orbitals(
                guess_job, package_name, project_name, path_hdf5,
                package_args, guess_args, package_config,
                calc_new_wf_guess_on_points, point_dir, job_files, k, gs)

            # Check if the job finishes succesfully
            promise_qm = schedule_check(
                promise_qm, job_name, package_name, project_name, path_hdf5,
                package_args, guess_args, package_config, point_dir, job_files, k, gs)

            # Store the computation
            path_MOs = store_in_hdf5(project_name, path_hdf5, promise_qm,
                                     hdf5_orb_path, job_name)

            guess_job = promise_qm
            orbitals.append(path_MOs)

    return gather(*orbitals)

@schedule
def store_in_hdf5(project_name: str, path_hdf5: str, promise_qm,
                  node_paths: str, job_name: str) -> None:
    """
    Store the MOs in the HDF5
    """
    # Molecular Orbitals
    mos = promise_qm.orbitals
    if mos is not None:
        # Store in the HDF5
        try:
            with h5py.File(path_hdf5, 'r+') as f5:
                dump_to_hdf5(
                    mos, 'cp2k', f5, project_name=project_name, job_name=job_name)
        # Remove the ascii MO file
        finally:
            work_dir = promise_qm.archive['work_dir']
            path_MOs = fnmatch.filter(os.listdir(work_dir), 'mo_*MOLog')[0]
            os.remove(path_MOs)

    return node_paths

def compute_orbitals(
        guess_job, package_name: str, project_name: str, path_hdf5: str,
        package_args: Dict, guess_args: Dict, package_config: Dict,
        calc_new_wf_guess_on_points: List, point_dir: str, job_files: Tuple,
        k: int, gs: List):
    """
    Call a Quantum chemisty package to compute the MOs required to calculate
    the nonadiabatic coupling. When finish store the MOs in the HdF5 and
    returns a new guess.
    """
    prepare_and_schedule = {'cp2k': prepare_job_cp2k}

    call_schedule_qm = prepare_and_schedule[package_name]

    job_files = create_file_names(point_dir, k)

    # Calculating initial guess
    compute_guess = calc_new_wf_guess_on_points is not None

    # A job  is a restart if guess_job is None and the list of
    # wf guesses are not empty
    is_restart = guess_job is None and compute_guess

    pred = (k in calc_new_wf_guess_on_points) or is_restart

    if pred:
        guess_job = call_schedule_qm(
            gs, job_files, guess_args,
            k, point_dir, wfn_restart_job=guess_job,
            package_config=package_config)

    promise_qm = call_schedule_qm(
        gs, job_files, package_args,
        k, point_dir, wfn_restart_job=guess_job,
        package_config=package_config)

    return promise_qm

@schedule
def schedule_check(promise_qm, job_name: str, package_name: str,
                   project_name: str, path_hdf5: str, package_args: Dict,
                   guess_args: Dict, package_config: Dict, point_dir: str,
                   job_files: Tuple, k: int, gs: List):
    """
    Check wether a calculation finishes succesfully otherwise run a new guess.
    """
    # Warnings of the computation
    warnings = promise_qm.warnings

    # Check for SCF convergence errors
    if warnings is not None and any(
            w == SCF_Convergence_Warning for msg, w in warnings.items()):
        # Report the failure
        msg = "Job: {} Finished with Warnings: {}".format(job_name, warnings)
        logger.warning(msg)

        # recompute a new guess
        msg1 = "Computing a new wave function guess for job: {}".format(job_name)
        logger.warning(msg1)

        # Remove the previous ascii file containing the MOs
        msg2 = "removing file containig the previous failed MOs of {}".format(job_name)
        logger.warning(msg2)
        path = fnmatch.filter(os.listdir(point_dir), 'mo*MOLog')[0]
        os.remove(join(point_dir, path))

        # Compute new guess at point k
        calc_new_wf_guess_on_points = [k]
        return compute_orbitals(
            None, package_name, project_name, path_hdf5,
            package_args, guess_args, package_config,
            calc_new_wf_guess_on_points, point_dir, job_files, k, gs)
    else:
        return promise_qm

def create_point_folder(work_dir, n, enumerate_from):
    """
    Create a new folder for each point in the MD trajectory.

    :returns: Paths lists.
    """
    folders = []
    for k in range(enumerate_from, n + enumerate_from):
        new_dir = join(work_dir, 'point_{}'.format(k))
        if not os.path.exists(new_dir):
            os.makedirs(new_dir)
        folders.append(new_dir)

    return folders

def split_file_geometries(pathXYZ):
    """
    Reads a set of molecular geometries in xyz format and returns
    a list of string, where is element a molecular geometry

    :returns: String list containing the molecular geometries.
    """
    # Read Cartesian Coordinates
    with open(pathXYZ) as f:
        xss = f.readlines()

    numat = int(xss[0].split()[0])
    return list(map(''.join, chunksOf(xss, numat + 2)))

def create_dict_CGFs(path_hdf5, basisname, xyz, package_name='cp2k',
                     package_config=None):
    """
    Try to read the basis from the HDF5 otherwise read it from a file and store
    it in the HDF5 file. Finally, it reads the basis Set from HDF5 and
    calculate the CGF for each atom.

    :param path_hdf5: Path to the HDF5 file that contains the
    numerical results.
    type path_hdf5: String
    :param basisname: Name of the Gaussian basis set.
    :type basisname: String
    :param xyz: List of Atoms.
    :type xyz: [nac.common.AtomXYZ]
    """
    functions = {'cp2k': cp2k2hdf5, 'turbomole': turbomole2hdf5}

    basis_location = join(package_name, 'basis')
    with h5py.File(path_hdf5) as f5:
        if basis_location not in f5:
            # Search Path to the file containing the basis set
            pathBasis = package_config["basis"]
            keyBasis = InputKey("basis", [pathBasis])
            # Store the basis sets
            functions[package_name](f5, [keyBasis])

        return createNormalizedCGFs(f5, basisname, package_name, xyz)

def create_file_names(work_dir, i):
    """
    Creates a namedTuple with the name of the 4 files used
    for each point in the trajectory

    :returns: Namedtuple containing the IO files
    """
    file_xyz = join(work_dir, 'coordinates_{}.xyz'.format(i))
    file_inp = join(work_dir, 'point_{}.inp'.format(i))
    file_out = join(work_dir, 'point_{}.out'.format(i))
    file_MO = join(work_dir, 'mo_coeff_{}.out'.format(i))

    return JobFiles(file_xyz, file_inp, file_out, file_MO)
<|endoftext|>


<commit_before>import os
import sys
import subprocess
from random import random
import tempfile
from nalaf import print_warning, print_verbose, print_debug

class SVMLightTreeKernels:
    """
    Base class for interaction with Alessandro Moschitti's Tree Kernels in SVM Light
    """

    def __init__(self, svmlight_dir_path='', model_path=tempfile.NamedTemporaryFile().name, use_tree_kernel=True):
        self.svmlight_dir_path = svmlight_dir_path
        """
        The directory where the executables svm_classify and svm_learn are located.
        Defaults to the empty string '', which then means that the svmlight executables must be in your binary path
        """

        executables_extension = '' if sys.platform.startswith('linux') or sys.platform.startswith('darwin') else '.exe'
        self.svm_learn_call = os.path.join(self.svmlight_dir_path, ('svm_learn' + executables_extension))
        self.svm_classify_call = os.path.join(self.svmlight_dir_path, ('svm_classify' + executables_extension))

        self.model_path = model_path
        """the model (path) to read from / write to"""

        self.use_tree_kernel = use_tree_kernel
        """whether to use tree kernels or not"""

    def create_input_file(self, dataset, mode, features, minority_class=None, undersampling=1):
        string = ''

        if mode == 'train':
            for edge in dataset.edges():
                if minority_class is None or edge.target == minority_class or random() < undersampling:
                    string += str(edge.target)
                    if self.use_tree_kernel:
                        string += ' |BT| '
                        string += edge.part.sentence_parse_trees[edge.sentence_id]
                        string += ' |ET|'
                    values = set(features.values())
                    for key in sorted(edge.features.keys()):
                        if key in values:
                            value = edge.features[key]
                            string += ' ' + str(key) + ':' + str(value)
                    string += '\n'

        elif mode == 'test':
            for edge in dataset.edges():
                string += str(edge.target)
                if self.use_tree_kernel:
                    string += ' |BT| '
                    string += edge.part.sentence_parse_trees[edge.sentence_id]
                    string += ' |ET|'
                values = set(features.values())
                for key in sorted(edge.features.keys()):
                    if key in values:
                        value = edge.features[key]
                        string += ' ' + str(key) + ':' + str(value)
                string += '\n'

        elif mode == 'predict':
            for edge in dataset.edges():
                string += '0'  # http://svmlight.joachims.org "A class label of 0 indicates that this example should be classified using transduction"
                if self.use_tree_kernel:
                    string += ' |BT| '
                    string += edge.part.sentence_parse_trees[edge.sentence_id]
                    string += ' |ET|'
                for key in sorted(edge.features.keys()):
                    if key in features.values():
                        value = edge.features[key]
                        string += ' ' + str(key) + ':' + str(value)
                string += '\n'

        instancesfile = tempfile.NamedTemporaryFile('w', delete=False)
        print_debug("Instances file for mode={} -> {}".format(mode, instancesfile.name))
        instancesfile.write(string)
        instancesfile.flush()
        # Note, we do not close the file

        return instancesfile

    def learn(self, instancesfile, c=0.5):

        with instancesfile:

            if self.use_tree_kernel:
                subprocess.call([
                    self.svm_learn_call,
                    '-v', '0',
                    '-t', '5',
                    '-T', '1',
                    '-W', 'S',
                    '-V', 'S',
                    '-C', '+',
                    '-c', str(c),
                    instancesfile.name,
                    self.model_path
                ])

            else:
                subprocess.call([
                    self.svm_learn_call,
                    '-c', str(c),
                    '-v', '0',
                    instancesfile.name,
                    self.model_path
                ])

            return self.model_path

    def tag(self, instancesfile):

        predictionsfile = tempfile.NamedTemporaryFile('r+', delete=False)
        print_debug("Predictions file: " + predictionsfile.name)

        call = [
            self.svm_classify_call,
            '-v', '0',
            instancesfile.name,
            self.model_path,
            predictionsfile.name
        ]
        exitcode = subprocess.call(call)

        if exitcode != 0:
            raise Exception("Error when tagging: " + ' '.join(call))

        predictionsfile.flush()
        # Note, we do not close the file

        return predictionsfile

    def read_predictions(self, dataset, predictionsfile):
        values = []
        with predictionsfile:
            predictionsfile.seek(0)

            for line in predictionsfile:
                if float(line.strip()) > 0:  # http://svmlight.joachims.org For classification, the sign of this value determines the predicted class -- CAUTION, relna (Ashish), had it set before to exactly: '-0.1' (was this a bug or a conscious decision to move the threshold of classification?)
                    values.append(1)
                else:
                    values.append(-1)

            if (len(values) > 1):
                for index, edge in enumerate(dataset.edges()):
                    edge.target = values[index]
            else:
                raise Exception("EMPTY PREDICTIONS FILE -- This may be due to too small dataset or too few of features. Predictions file: " + predictionsfile.name)

        return dataset.form_predicted_relations()
<commit_msg>set threshold as a parameter in reading of svm predictions -- https://github.com/Rostlab/relna/issues/21<commit_after>import os
import sys
import subprocess
from random import random
import tempfile
from nalaf import print_warning, print_verbose, print_debug

class SVMLightTreeKernels:
    """
    Base class for interaction with Alessandro Moschitti's Tree Kernels in SVM Light
    """

    def __init__(self, svmlight_dir_path='', model_path=tempfile.NamedTemporaryFile().name, use_tree_kernel=True):
        self.svmlight_dir_path = svmlight_dir_path
        """
        The directory where the executables svm_classify and svm_learn are located.
        Defaults to the empty string '', which then means that the svmlight executables must be in your binary path
        """

        executables_extension = '' if sys.platform.startswith('linux') or sys.platform.startswith('darwin') else '.exe'
        self.svm_learn_call = os.path.join(self.svmlight_dir_path, ('svm_learn' + executables_extension))
        self.svm_classify_call = os.path.join(self.svmlight_dir_path, ('svm_classify' + executables_extension))

        self.model_path = model_path
        """the model (path) to read from / write to"""

        self.use_tree_kernel = use_tree_kernel
        """whether to use tree kernels or not"""

    def create_input_file(self, dataset, mode, features, minority_class=None, undersampling=1):
        string = ''

        if mode == 'train':
            for edge in dataset.edges():
                if minority_class is None or edge.target == minority_class or random() < undersampling:
                    string += str(edge.target)
                    if self.use_tree_kernel:
                        string += ' |BT| '
                        string += edge.part.sentence_parse_trees[edge.sentence_id]
                        string += ' |ET|'
                    values = set(features.values())
                    for key in sorted(edge.features.keys()):
                        if key in values:
                            value = edge.features[key]
                            string += ' ' + str(key) + ':' + str(value)
                    string += '\n'

        elif mode == 'test':
            for edge in dataset.edges():
                string += str(edge.target)
                if self.use_tree_kernel:
                    string += ' |BT| '
                    string += edge.part.sentence_parse_trees[edge.sentence_id]
                    string += ' |ET|'
                values = set(features.values())
                for key in sorted(edge.features.keys()):
                    if key in values:
                        value = edge.features[key]
                        string += ' ' + str(key) + ':' + str(value)
                string += '\n'

        elif mode == 'predict':
            for edge in dataset.edges():
                string += '0'  # http://svmlight.joachims.org "A class label of 0 indicates that this example should be classified using transduction"
                if self.use_tree_kernel:
                    string += ' |BT| '
                    string += edge.part.sentence_parse_trees[edge.sentence_id]
                    string += ' |ET|'
                for key in sorted(edge.features.keys()):
                    if key in features.values():
                        value = edge.features[key]
                        string += ' ' + str(key) + ':' + str(value)
                string += '\n'

        instancesfile = tempfile.NamedTemporaryFile('w', delete=False)
        print_debug("Instances file for mode={} -> {}".format(mode, instancesfile.name))
        instancesfile.write(string)
        instancesfile.flush()
        # Note, we do not close the file

        return instancesfile

    def learn(self, instancesfile, c=0.5):

        with instancesfile:

            if self.use_tree_kernel:
                subprocess.call([
                    self.svm_learn_call,
                    '-v', '0',
                    '-t', '5',
                    '-T', '1',
                    '-W', 'S',
                    '-V', 'S',
                    '-C', '+',
                    '-c', str(c),
                    instancesfile.name,
                    self.model_path
                ])

            else:
                subprocess.call([
                    self.svm_learn_call,
                    '-c', str(c),
                    '-v', '0',
                    instancesfile.name,
                    self.model_path
                ])

            return self.model_path

    def tag(self, instancesfile):

        predictionsfile = tempfile.NamedTemporaryFile('r+', delete=False)
        print_debug("Predictions file: " + predictionsfile.name)

        call = [
            self.svm_classify_call,
            '-v', '0',
            instancesfile.name,
            self.model_path,
            predictionsfile.name
        ]
        exitcode = subprocess.call(call)

        if exitcode != 0:
            raise Exception("Error when tagging: " + ' '.join(call))

        predictionsfile.flush()
        # Note, we do not close the file

        return predictionsfile

    def read_predictions(self, dataset, predictionsfile, threshold=0):
        values = []
        with predictionsfile:
            predictionsfile.seek(0)

            for line in predictionsfile:
                # http://svmlight.joachims.org For classification, the sign of this value determines the predicted class -- CAUTION, relna (Ashish), had it set before to exactly: '-0.1' (was this a bug or a conscious decision to move the threshold of classification?)
                # See more information in: https://github.com/Rostlab/relna/issues/21
                if float(line.strip()) > threshold:
                    values.append(1)
                else:
                    values.append(-1)

            if (len(values) > 1):
                for index, edge in enumerate(dataset.edges()):
                    edge.target = values[index]
            else:
                raise Exception("EMPTY PREDICTIONS FILE -- This may be due to too small dataset or too few of features. Predictions file: " + predictionsfile.name)

        return dataset.form_predicted_relations()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.0.1'
__all__ = ["name_expansion"]

from name_expansion import NameExpansionCore
<commit_msg>rename NameExpansionCore to NameExpansion<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.0.1'
__all__ = ["name_expansion"]

from name_expansion import NameExpansion
<|endoftext|>


<commit_before>#!/usr/bin/env python
from __future__ import print_function

import sys
import os
import re
import sqlite3
import qrcode
import codecs
import argparse

default_db = 'databases'
# https://code.google.com/p/google-authenticator/wiki/KeyUriFormat

def make_qrimage(url):
    qr = qrcode.QRCode()
    qr.add_data(url)
    qr.make()
    return qr

def query_db(args):
    conn = sqlite3.connect(args.db)
    c = conn.cursor()
    qry = '''SELECT _id, email, secret, issuer, original_name FROM accounts '''
    if args.id:
        qry += ''' WHERE _id={id}'''.format(id=args.id)
    qry += ''' ORDER BY _id'''
    resp = c.execute(qry)
    return resp

def list_entries(args):
    resp = query_db(args)
    fmt = "% 3s: %-20s: %s"
    title = fmt % ('ID', 'Issuer', 'e-mail')
    print("%s\n%s" % (title, len(title)))
    for row in resp:
        print(fmt % (row[0], row[3] if row[3] is not None else row[4], row[1]))

def process_db(args):
    if args.list_entries:
        return list_entries(args)

    isatty = os.isatty(sys.stdout.fileno())
    if not isatty:
        # Stdout is not a terminal. Force it to be utf-8.
        sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

    resp = query_db(args)
    for row in resp:
        qrurl = "otpauth://totp/%s?secret=%s&issuer=%s" % (row[1], row[2], row[3] if row[3] is not None else row[4])
        if args.url_only:
            print(qrurl)
        qr = make_qrimage(qrurl)

        if args.tty:
            print("%s: %s" % (row[3] if row[3] is not None else row[4], row[1]))
            if isatty:
                qr.print_tty()
            else:
                qr.print_ascii(out=sys.stdout, invert=True)
            print()

        if args.png:
            outname = re.sub('[:/]', '_', row[1]) + '.png'
            print(outname)
            img = qr.make_image()
            img.save(outname)

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument("--db", "--ga-db", help="Name of the Google Authenticator database file")
    parser.add_argument("--list-entries", action="store_true", help="List the entries in the database file.")
    parser.add_argument("--png", action="store_true", help="Save generated codes as PNG files.")
    parser.add_argument("--tty", action="store_true", help="Dump entries to console as ANSI graphics")
    parser.add_argument("--url-only", action="store_true", help="Only print the otpauth:// URLs.")
    parser.add_argument("--id", help="Dump only the one which has the id")

    args = parser.parse_args()
    if not args.db:
        args.db = default_db
    if not os.path.exists(default_db):
        print("Database file '%s' not exists.")
        sys.exit(1)

    process_db(args)

if __name__ == '__main__':
    main()
<commit_msg>db query returns Row, instead of tuple. URI created by separate function<commit_after>#!/usr/bin/env python
from __future__ import print_function

import sys
import os
import re
import sqlite3
import qrcode
import codecs
import argparse

default_db = 'databases'
# https://code.google.com/p/google-authenticator/wiki/KeyUriFormat

def make_qrimage(url):
    qr = qrcode.QRCode()
    qr.add_data(url)
    qr.make()
    return qr

def query_db(args):
    conn = sqlite3.connect(args.db)
    conn.row_factory = sqlite3.Row
    c = conn.cursor()
    qry = '''SELECT * FROM accounts '''
    if args.id:
        qry += ''' WHERE _id={id}'''.format(id=args.id)
    qry += ''' ORDER BY _id'''
    resp = c.execute(qry)
    return resp

def list_entries(args):
    resp = query_db(args)
    fmt = "% 3s: %-20s: %s"
    title = fmt % ('ID', 'Issuer', 'e-mail')
    print("%s\n%s" % (title, '-' * len(title)))
    for row in resp:
        print(fmt % (row['_id'], row['issuer'] if row['issuer'] is not None else row['original_name'], row['email']))

def create_otpauth_url(atype, secret, email, **kwds):
    if isinstance(atype, int):
        if atype == 0:
            atype = 'totp'
        elif atype == 1:
            atype == 'hotp'
        else:
            raise ValueError("Invalid OTP type '%s'" % atype)

    url = "otpauth://" + atype + "/" + email + "?secret=" + secret

    if atype == 'totp':
        if 'period' in kwds:
            url += '&period=' + kwds['period']

        if 'digits' in kwds:
            url += '&digits=' + kwds['digits']
    elif atype == 'hotp':
        if 'counter' in kwds:
            url +='&counter=' + kwds['counter']

    if 'issuer' in kwds:
        url += "&issuer=" + kwds['issuer']

    return url

def process_db(args):
    if args.list_entries:
        return list_entries(args)

    isatty = os.isatty(sys.stdout.fileno())
    if not isatty:
        # Stdout is not a terminal. Force it to be utf-8.
        sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

    resp = query_db(args)
    for row in resp:
        qrurl = create_otpauth_url(atype=row['type'], secret=row['secret'], email=row['email'],
                                   issuer=row['issuer'] if row['issuer'] is not None else row['original_name'],
                                   counter=row['counter'])
        if args.url_only:
            print(qrurl)
        qr = make_qrimage(qrurl)

        if args.tty:
            print("%s: %s" % (row[3] if row[3] is not None else row[4], row[1]))
            if isatty:
                qr.print_tty()
            else:
                qr.print_ascii(out=sys.stdout, invert=True)
            print()

        if args.png:
            outname = re.sub('[:/]', '_', row['email']) + '.png'
            print(outname)
            img = qr.make_image()
            img.save(outname)

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument("--db", "--ga-db", help="Name of the Google Authenticator database file")
    parser.add_argument("--list-entries", action="store_true", help="List the entries in the database file.")
    parser.add_argument("--png", action="store_true", help="Save generated codes as PNG files.")
    parser.add_argument("--tty", action="store_true", help="Dump entries to console as ANSI graphics")
    parser.add_argument("--url-only", action="store_true", help="Only print the otpauth:// URLs.")
    parser.add_argument("--id", help="Dump only the one which has the id")

    args = parser.parse_args()
    if not args.db:
        args.db = default_db
    if not os.path.exists(default_db):
        print("Database file '%s' not exists.")
        sys.exit(1)

    process_db(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import abc

import six

class CipherAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this mode (e.g. "AES", "Camellia").
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The size of the key being used as an integer in bits (e.g. 128, 256).
        """

class BlockCipherAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def block_size(self):
        """
        The size of a block as an integer in bits (e.g. 64, 128).
        """

class Mode(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this mode (e.g. "ECB", "CBC").
        """

    @abc.abstractmethod
    def validate_for_algorithm(self, algorithm):
        """
        Checks that all the necessary invariants of this (mode, algorithm)
        combination are met.
        """

class ModeWithInitializationVector(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def initialization_vector(self):
        """
        The value of the initialization vector for this mode as bytes.
        """

class ModeWithNonce(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def nonce(self):
        """
        The value of the nonce for this mode as bytes.
        """

class ModeWithAuthenticationTag(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def tag(self):
        """
        The value of the tag supplied to the constructor of this mode.
        """

class CipherContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes through the cipher and returns the results
        as bytes.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Returns the results of processing the final block as bytes.
        """

class AEADCipherContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def authenticate_additional_data(self, data):
        """
        Authenticates the provided bytes.
        """

class AEADEncryptionContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def tag(self):
        """
        Returns tag bytes. This is only available after encryption is
        finalized.
        """

class PaddingContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Pads the provided bytes and returns any available data as bytes.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Finalize the padding, returns bytes.
        """

class HashAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this algorithm (e.g. "sha256", "md5").
        """

    @abc.abstractproperty
    def digest_size(self):
        """
        The size of the resulting digest in bytes.
        """

    @abc.abstractproperty
    def block_size(self):
        """
        The internal block size of the hash algorithm in bytes.
        """

class HashContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def algorithm(self):
        """
        A HashAlgorithm that will be used by this context.
        """

    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes through the hash.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Finalizes the hash context and returns the hash digest as bytes.
        """

    @abc.abstractmethod
    def copy(self):
        """
        Return a HashContext that is a copy of the current context.
        """

class RSAPrivateKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The public modulus of the RSA key.
        """

    @abc.abstractproperty
    def public_exponent(self):
        """
        The public exponent of the RSA key.
        """

    @abc.abstractproperty
    def private_exponent(self):
        """
        The private exponent of the RSA key.
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The bit length of the public modulus.
        """

    @abc.abstractmethod
    def public_key(self):
        """
        The RSAPublicKey associated with this private key.
        """

    @abc.abstractproperty
    def n(self):
        """
        The public modulus of the RSA key. Alias for modulus.
        """

    @abc.abstractproperty
    def p(self):
        """
        One of the two primes used to generate d.
        """

    @abc.abstractproperty
    def q(self):
        """
        One of the two primes used to generate d.
        """

    @abc.abstractproperty
    def d(self):
        """
        The private exponent. This can be calculated using p and q. Alias for
        private_exponent.
        """

    @abc.abstractproperty
    def dmp1(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations.  Calculated as: d mod (p-1)
        """

    @abc.abstractproperty
    def dmq1(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations.  Calculated as: d mod (q-1)
        """

    @abc.abstractproperty
    def iqmp(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations. The modular inverse of q modulo p
        """

    @abc.abstractproperty
    def e(self):
        """
        The public exponent of the RSA key. Alias for public_exponent.
        """

class RSAPublicKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The public modulus of the RSA key.
        """

    @abc.abstractproperty
    def public_exponent(self):
        """
        The public exponent of the RSA key.
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The bit length of the public modulus.
        """

    @abc.abstractproperty
    def n(self):
        """
        The public modulus of the RSA key. Alias for modulus.
        """

    @abc.abstractproperty
    def e(self):
        """
        The public exponent of the RSA key. Alias for public_exponent.
        """

class DSAParams(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def divisor(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def generator(self):
        """
        The generator that is used in generating the DSA keypair.
        """

    @abc.abstractmethod
    def generate(self):
        """
        Generate DSAPrivateKey from the object's parameters
        """

    @abc.abstractproperty
    def p(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        Alias for modulus.
        """

    @abc.abstractproperty
    def q(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        Alias for divisor.
        """

    @abc.abstractproperty
    def g(self):
        """
        The generator that is used in generating the DSA keypair.
        Alias for generator.
        """

class DSAPrivateKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def divisor(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def generator(self):
        """
        The generator that is used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def modulus_length(self):
        """
        The bit length of the prime modulus.
        """

    @abc.abstractproperty
    def divisor_length(self):
        """
        The bit length of the divisor.
        """

    @abc.abstractproperty
    def priv_key(self):
        """
        The private key in the DSA structure.
        """

    @abc.abstractmethod
    def public_key(self):
        """
        The DSAPublicKey associated with this private key.
        """

    @abc.abstractproperty
    def p(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        Alias for modulus.
        """

    @abc.abstractproperty
    def q(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        Alias for divisor.
        """

    @abc.abstractproperty
    def g(self):
        """
        The generator that is used in generating the DSA keypair.
        Alias for generator.
        """

    @abc.abstractproperty
    def L(self):
        """
        The bit length of the prime modulus. Alias for modulus_length.
        """

    @abc.abstractproperty
    def N(self):
        """
        The bit length of the divisor. Alias for divisor_length.
        """

    @abc.abstractproperty
    def y(self):
        """
        The DSAPublicKey associated with this private key.
        Alias for public_key.
        """

    @abc.abstractproperty
    def params(self):
        """
        The params associated with a DSA keypair
        """

class DSAPublicKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def divisor(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def generator(self):
        """
        The generator that is used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def pub_key(self):
        """
        The pub_key that's in the DSA structure, also known as y.
        """
    @abc.abstractproperty
    def p(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        Alias for modulus.
        """

    @abc.abstractproperty
    def q(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        Alias for divisor.
        """

    @abc.abstractproperty
    def g(self):
        """
        The generator that is used in generating the DSA keypair.
        Alias for generator.
        """

    @abc.abstractproperty
    def params(self):
        """
        The params associated with a DSA keypair
        """

class AsymmetricSignatureContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes and returns nothing.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Returns the signature as bytes.
        """

class AsymmetricVerificationContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes and returns nothing.
        """

    @abc.abstractmethod
    def verify(self):
        """
        Raises an exception if the bytes provided to update do not match the
        signature or the signature does not match the public key.
        """

class AsymmetricPadding(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this padding (e.g. "PSS", "PKCS1").
        """

class KeyDerivationFunction(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def derive(self, key_material):
        """
        Deterministically generates and returns a new key based on the existing
        key material.
        """

    @abc.abstractmethod
    def verify(self, key_material, expected_key):
        """
        Checks whether the key generated by the key material matches the
        expected derived key. Raises an exception if they do not match.
        """
<commit_msg>Change property names to be consistent with RSA<commit_after># Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import abc

import six

class CipherAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this mode (e.g. "AES", "Camellia").
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The size of the key being used as an integer in bits (e.g. 128, 256).
        """

class BlockCipherAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def block_size(self):
        """
        The size of a block as an integer in bits (e.g. 64, 128).
        """

class Mode(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this mode (e.g. "ECB", "CBC").
        """

    @abc.abstractmethod
    def validate_for_algorithm(self, algorithm):
        """
        Checks that all the necessary invariants of this (mode, algorithm)
        combination are met.
        """

class ModeWithInitializationVector(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def initialization_vector(self):
        """
        The value of the initialization vector for this mode as bytes.
        """

class ModeWithNonce(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def nonce(self):
        """
        The value of the nonce for this mode as bytes.
        """

class ModeWithAuthenticationTag(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def tag(self):
        """
        The value of the tag supplied to the constructor of this mode.
        """

class CipherContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes through the cipher and returns the results
        as bytes.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Returns the results of processing the final block as bytes.
        """

class AEADCipherContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def authenticate_additional_data(self, data):
        """
        Authenticates the provided bytes.
        """

class AEADEncryptionContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def tag(self):
        """
        Returns tag bytes. This is only available after encryption is
        finalized.
        """

class PaddingContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Pads the provided bytes and returns any available data as bytes.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Finalize the padding, returns bytes.
        """

class HashAlgorithm(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this algorithm (e.g. "sha256", "md5").
        """

    @abc.abstractproperty
    def digest_size(self):
        """
        The size of the resulting digest in bytes.
        """

    @abc.abstractproperty
    def block_size(self):
        """
        The internal block size of the hash algorithm in bytes.
        """

class HashContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def algorithm(self):
        """
        A HashAlgorithm that will be used by this context.
        """

    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes through the hash.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Finalizes the hash context and returns the hash digest as bytes.
        """

    @abc.abstractmethod
    def copy(self):
        """
        Return a HashContext that is a copy of the current context.
        """

class RSAPrivateKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The public modulus of the RSA key.
        """

    @abc.abstractproperty
    def public_exponent(self):
        """
        The public exponent of the RSA key.
        """

    @abc.abstractproperty
    def private_exponent(self):
        """
        The private exponent of the RSA key.
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The bit length of the public modulus.
        """

    @abc.abstractmethod
    def public_key(self):
        """
        The RSAPublicKey associated with this private key.
        """

    @abc.abstractproperty
    def n(self):
        """
        The public modulus of the RSA key. Alias for modulus.
        """

    @abc.abstractproperty
    def p(self):
        """
        One of the two primes used to generate d.
        """

    @abc.abstractproperty
    def q(self):
        """
        One of the two primes used to generate d.
        """

    @abc.abstractproperty
    def d(self):
        """
        The private exponent. This can be calculated using p and q. Alias for
        private_exponent.
        """

    @abc.abstractproperty
    def dmp1(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations.  Calculated as: d mod (p-1)
        """

    @abc.abstractproperty
    def dmq1(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations.  Calculated as: d mod (q-1)
        """

    @abc.abstractproperty
    def iqmp(self):
        """
        A Chinese remainder theorem coefficient used to speed up RSA
        calculations. The modular inverse of q modulo p
        """

    @abc.abstractproperty
    def e(self):
        """
        The public exponent of the RSA key. Alias for public_exponent.
        """

class RSAPublicKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The public modulus of the RSA key.
        """

    @abc.abstractproperty
    def public_exponent(self):
        """
        The public exponent of the RSA key.
        """

    @abc.abstractproperty
    def key_size(self):
        """
        The bit length of the public modulus.
        """

    @abc.abstractproperty
    def n(self):
        """
        The public modulus of the RSA key. Alias for modulus.
        """

    @abc.abstractproperty
    def e(self):
        """
        The public exponent of the RSA key. Alias for public_exponent.
        """

class DSAParams(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def modulus(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def divisor(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        """

    @abc.abstractproperty
    def generator(self):
        """
        The generator that is used in generating the DSA keypair.
        """

    @abc.abstractmethod
    def generate(self):
        """
        Generate DSAPrivateKey from the object's parameters
        """

    @abc.abstractproperty
    def p(self):
        """
        The prime modulus that's used in generating the DSA keypair.
        Alias for modulus.
        """

    @abc.abstractproperty
    def q(self):
        """
        The prime divisor of (p-1) that's used in generating the DSA keypair.
        Alias for divisor.
        """

    @abc.abstractproperty
    def g(self):
        """
        The generator that is used in generating the DSA keypair.
        Alias for generator.
        """

class DSAPrivateKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def key_size(self):
        """
        The bit length of the prime modulus.
        """

    @abc.abstractproperty
    def divisor_size(self):
        """
        The bit length of the divisor.
        """

    @abc.abstractproperty
    def priv_key(self):
        """
        The private key in the DSA structure.
        """

    @abc.abstractmethod
    def public_key(self):
        """
        The DSAPublicKey associated with this private key.
        """

    @abc.abstractproperty
    def L(self):
        """
        The bit length of the prime modulus. Alias for modulus_length.
        """

    @abc.abstractproperty
    def N(self):
        """
        The bit length of the divisor. Alias for divisor_length.
        """

    @abc.abstractproperty
    def y(self):
        """
        The DSAPublicKey associated with this private key.
        Alias for public_key.
        """

    @abc.abstractproperty
    def params(self):
        """
        The params associated with a DSA keypair
        """

class DSAPublicKey(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def pub_key(self):
        """
        The pub_key that's in the DSA structure, also known as y.
        """

    @abc.abstractproperty
    def params(self):
        """
        The params associated with a DSA keypair
        """

class AsymmetricSignatureContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes and returns nothing.
        """

    @abc.abstractmethod
    def finalize(self):
        """
        Returns the signature as bytes.
        """

class AsymmetricVerificationContext(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def update(self, data):
        """
        Processes the provided bytes and returns nothing.
        """

    @abc.abstractmethod
    def verify(self):
        """
        Raises an exception if the bytes provided to update do not match the
        signature or the signature does not match the public key.
        """

class AsymmetricPadding(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractproperty
    def name(self):
        """
        A string naming this padding (e.g. "PSS", "PKCS1").
        """

class KeyDerivationFunction(six.with_metaclass(abc.ABCMeta)):
    @abc.abstractmethod
    def derive(self, key_material):
        """
        Deterministically generates and returns a new key based on the existing
        key material.
        """

    @abc.abstractmethod
    def verify(self, key_material, expected_key):
        """
        Checks whether the key generated by the key material matches the
        expected derived key. Raises an exception if they do not match.
        """
<|endoftext|>


<commit_before>#! /usr/bin/python3

# Import everything needed.
import web

#Define code paths (for now, just /)
urls = (
    '/', 'index'
)

class index:
    def GET(self):
        return 42

if __name__ == "__main__":
    app = web.application(urls, globals())
    app.run()
<commit_msg>Removed server api, it was useless<commit_after><|endoftext|>


<commit_before># coding: utf-8
# zhangxiaoyang.hit#gmail.com
# github.com/zhangxiaoyang

import socket
import traceback

class GstoreConnector:

    def _connect(self):
        try:
            self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock.connect((self.ip, self.port))
            return True
        except Exception, e:
            print 'socket connection error. @GstoreConnector.connect'
            traceback.print_exc()
        return False

    def _disconnect(self):
        try:
            self._sock.close()
            return True
        except Exception, e:
            print 'socket disconnection error. @GstoreConnector.disconnect'
            traceback.print_exc()
        return False

    def _send(self, msg):
        data = self._pack(msg)
        self._sock.send(data)
        return True

    def _recv(self):
        head = self._sock.recv(4)
        context_len = 0
        for i in range(4):
            context_len |= (ord(head[i]) & 0xFF) << i * 8

        data = bytearray()
        recv_len = 0
        while recv_len < context_len:
            chunk = self._sock.recv(context_len - recv_len)
            data.extend(chunk)
            recv_len += len(chunk)
        return data.rstrip('\x00').decode('utf-8')

    def _pack(self, msg):
        data_context = bytearray()
        data_context.extend(msg)
        context_len = len(data_context) + 1
        data_len = context_len + 4

        data = bytearray(data_len)
        for i in range(4):
            data[i] = chr((context_len >> i * 8) & 0xFF)
        for i, _ in enumerate(data_context):
            data[i + 4] = data_context[i]
        data[data_len - 1] = 0
        return data

    def _communicate(f):
        def wrapper(self, *args, **kwargs):
            if not self._connect():
                print 'connect to server error. @GstoreConnector.%s' % f.__name__
                return False

            if f.__name__ == 'build':
                cmd = 'import'
            elif f.__name__ == 'show':
                if args[0]:
                    cmd = 'show all'
                else:
                    cmd = 'show databases'
            else:
                cmd = f.__name__
            if f.__name__ == 'query':
                #DEBUG:only sparql query should be used, output should not be included in parameters
                params = ' '.join([args[1], args[0]])
            else:
                params = ' '.join(map(lambda x:str(x), args))
            full_cmd = ' '.join([
                cmd,
                params
            ]).strip()

            if not self._send(full_cmd):
                print 'send %s command error. @GstoreConnector.build' % cmd
                return False

            recv_msg = self._recv()
            self._disconnect()

            succ = {
                'test': 'OK',
                'load': 'load database done.',
                'unload': 'unload database done.',
                'import': 'import RDF file to database done.',
                'drop': 'drop database done.',
                'stop': 'server stopped.',
                'query': None,
                'show all': None,
                'show databases': None,
            }
            if cmd in succ:
                if succ[cmd] == recv_msg:
                    return True
                else:
                    return recv_msg
            return False
        return wrapper

    def __init__(self, ip = '127.0.0.1', port = 3305):
        self.ip = ip
        self.port = port

    @_communicate
    def test(self):
        pass

    @_communicate
    def load(self, db_name):
        pass

    @_communicate
    def unload(self, db_name):
        pass

    @_communicate
    def build(self, db_name, rdf_file_path):
        pass

    @_communicate
    def drop(self, db_name):
        pass

    @_communicate
    def query(self, sparql, output = '/'):
        pass

    @_communicate
    def show(self, _type = False):
        pass
<commit_msg>fix run error bug in python api<commit_after># coding: utf-8
# zhangxiaoyang.hit#gmail.com
# github.com/zhangxiaoyang

import socket
import traceback

class GstoreConnector:

    def _connect(self):
        try:
            self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._sock.connect((self.ip, self.port))
            return True
        except Exception, e:
            print 'socket connection error. @GstoreConnector.connect'
            traceback.print_exc()
        return False

    def _disconnect(self):
        try:
            self._sock.close()
            return True
        except Exception, e:
            print 'socket disconnection error. @GstoreConnector.disconnect'
            traceback.print_exc()
        return False

    def _send(self, msg):
        data = self._pack(msg)
        self._sock.send(data)
        return True

    def _recv(self):
        head = self._sock.recv(4)
        context_len = 0
        for i in range(4):
            context_len |= (ord(head[i]) & 0xFF) << i * 8

        data = bytearray()
        recv_len = 0
        while recv_len < context_len:
            chunk = self._sock.recv(context_len - recv_len)
            data.extend(chunk)
            recv_len += len(chunk)
        return data.rstrip('\x00').decode('utf-8')

    def _pack(self, msg):
        data_context = bytearray()
        data_context.extend(msg)
        context_len = len(data_context) + 1
        data_len = context_len + 4

        data = bytearray(data_len)
        for i in range(4):
            data[i] = chr((context_len >> i * 8) & 0xFF)
        for i, _ in enumerate(data_context):
            data[i + 4] = data_context[i]
        data[data_len - 1] = 0
        return data

    def _communicate(f):
        def wrapper(self, *args, **kwargs):
            if not self._connect():
                print 'connect to server error. @GstoreConnector.%s' % f.__name__
                return False

            if f.__name__ == 'build':
                cmd = 'import'
            elif f.__name__ == 'show':
                if args[0]:
                    cmd = 'show all'
                else:
                    cmd = 'show databases'
            else:
                cmd = f.__name__
            if f.__name__ == 'query':
                #DEBUG:only sparql query should be used, output should not be included in parameters
                print "the first argument:"
                print args[0]
                # print args[1]
                params = ' '.join([args[0]])
            else:
                params = ' '.join(map(lambda x:str(x), args))
            full_cmd = ' '.join([
                cmd,
                params
            ]).strip()
            print "command to send to server:"
            print full_cmd

            if not self._send(full_cmd):
                print 'send %s command error. @GstoreConnector.build' % cmd
                return False

            recv_msg = self._recv()
            self._disconnect()

            succ = {
                'test': 'OK',
                'load': 'load database done.',
                'unload': 'unload database done.',
                'import': 'import RDF file to database done.',
                'drop': 'drop database done.',
                'stop': 'server stopped.',
                'query': None,
                'show all': None,
                'show databases': None,
            }
            if cmd in succ:
                if succ[cmd] == recv_msg:
                    return True
                else:
                    return recv_msg
            return False
        return wrapper

    def __init__(self, ip = '127.0.0.1', port = 3305):
        self.ip = ip
        self.port = port

    @_communicate
    def test(self):
        pass

    @_communicate
    def load(self, db_name):
        pass

    @_communicate
    def unload(self, db_name):
        pass

    @_communicate
    def build(self, db_name, rdf_file_path):
        pass

    @_communicate
    def drop(self, db_name):
        pass

    @_communicate
    def query(self, sparql, output = '/'):
        pass

    @_communicate
    def show(self, _type = False):
        pass
<|endoftext|>