<commit_before># coding=utf-8
# Filename: hdf5.py
# pylint: disable=C0103,R0903
# vim:set ts=4 sts=4 sw=4 et:
"""
Pumps for the EVT simulation dataformat.

"""
from __future__ import division, absolute_import, print_function

import os.path

import numpy as np
import tables

import km3pipe
from km3pipe import Pump, Module
from km3pipe.dataclasses import HitSeries, TrackSeries, EventInfo
from km3pipe.logger import logging

log = logging.getLogger(__name__)  # pylint: disable=C0103

__author__ = "Tamas Gal and Moritz Lotze"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal and Moritz Lotze"
__email__ = "tgal@km3net.de"
__status__ = "Development"

class HDF5Sink(Module):
    def __init__(self, **context):
        """A Module to convert (KM3NeT) ROOT files to HDF5."""
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename') or 'dump.h5'
        self.index = 1
        self.h5file = tables.open_file(self.filename, mode="w", title="KM3NeT")
        self.filters = tables.Filters(complevel=5, shuffle=True,
                                      fletcher32=True)
        desc = self.get('tables') or []
        self._descriptions = [
                ('Hits', '/hits', HitSeries.dtype),
                ('MCHits', '/mc_hits', HitSeries.dtype),
                ('MCTracks', '/mc_tracks', TrackSeries.dtype),
                ('EventInfo', '/event_info', EventInfo.dtype),
                ]
        self._descriptions.extend(desc)
        self._tables = {}
        for key, path, dtype in self._descriptions:
            loc, tabname = os.path.split(path)
            tab = self.h5file.create_table(loc, tabname, description=dtype,
            title=key, filters=self.filters, createparents=True)
            self._tables[key] = tab

    def _write_reco_track(self, track, reco_row):
        for colname, val in track.items():
            reco_row[colname] = val
        reco_row.append()

    def _gen_dtype(self, track):
        keys = [(k, np.float64) for k in track.keys() if k != 'event_id' if k != 'did_converge']
        keys.extend([('event_id', np.uint32), ('did_converge', np.bool_)])
        return np.dtype(sorted(keys))

    def _write_reco(self, reco_dict, reco_group):
        for recname, track in reco_dict.items():
            if recname not in self._tables:
                dtype = self._gen_dtype(track)
                reco_table = self.h5file.create_table(
                    reco_group, recname.lower(),
                    dtype,      #np.dtype(recname_to_dtype[recname]),
                    recname, createparents=True, filters=self.filters
                )
                self._tables[recname] = reco_table
            reco_table = self._tables[recname]
            self._write_reco_track(track, reco_table.row)

    def process(self, blob):
        for key, tab in self._tables.items():
			if key in blob:
				tab.append(blob[key].as_table())
        if 'Reco' in blob:
            # this is a group, not a single table
            self._write_reco(blob['Reco'], '/reco')

        if not self.index % 1000:
            for tab in self._tables.values():
                tab.flush()

        self.index += 1
        return blob

    def finish(self):
        for tab in self._tables.values():
            tab.cols.event_id.create_index()
        self.h5file.root._v_attrs.km3pipe = str(km3pipe.__version__)
        self.h5file.root._v_attrs.pytables = str(tables.__version__)
        self.h5file.close()

class HDF5Pump(Pump):
    """Provides a pump for KM3NeT HDF5 files"""
    def __init__(self, filename, **context):
        super(self.__class__, self).__init__(**context)
        self.filename = filename
        if os.path.isfile(self.filename):
            self.h5_file = tables.File(self.filename)
        else:
            raise IOError("No such file or directory: '{0}'"
                          .format(self.filename))
        self.index = None
        self._reset_index()

        try:
            event_info = self.h5_file.get_node('/', 'event_info')
            self.event_ids = event_info.cols.event_id[:]
        except tables.NoSuchNodeError:
            log.critical("No /event_info table found.")
            raise SystemExit

        self._n_events = len(self.event_ids)

    def process(self, blob):
        try:
            blob = self.get_blob(self.index)
        except KeyError:
            self._reset_index()
            raise StopIteration
        self.index += 1
        return blob

    def _get_hits(self, event_id, table_name='hits', where='/'):
        table = self.h5_file.get_node(where, table_name)
        rows = table.read_where('event_id == %d' % event_id)
        return HitSeries.from_table(rows, event_id)

    def _get_tracks(self, event_id, table_name='tracks', where='/'):
        table = self.h5_file.get_node(where, table_name)
        rows = table.read_where('event_id == %d' % event_id)
        return TrackSeries.from_table(rows, event_id)

    def _get_event_info(self, event_id, table_name='event_info', where='/'):
        table = self.h5_file.get_node(where, table_name)
        return EventInfo.from_table(table[event_id])

    def _get_reco(self, event_id, group_name='reco', where='/'):
        group = self.h5_file.get_node(where, group_name)
        out = {}
        for table in group:
            tabname = table.title
            data = table.read_where('event_id == %d' % event_id)
            out[tabname] = data
        return out

    def get_blob(self, index):
        event_id = self.event_ids[index]
        blob = {}
        blob['Hits'] = self._get_hits(event_id, table_name='hits')
        blob['MCHits'] = self._get_hits(event_id, table_name='mc_hits')
        blob['MCTracks'] = self._get_tracks(event_id, table_name='mc_tracks')
        blob['EventInfo'] = self._get_event_info(event_id,
                                                 table_name='event_info')
        blob['Reco'] = self._get_reco(event_id, group_name='reco')
        return blob

    def finish(self):
        """Clean everything up"""
        self.h5_file.close()

    def _reset_index(self):
        """Reset index to default value"""
        self.index = 0

    def __len__(self):
        return self._n_events

    def __iter__(self):
        return self

    def next(self):
        """Python 2/3 compatibility for iterators"""
        return self.__next__()

    def __next__(self):
        if self.index >= self._n_events:
            self._reset_index()
            raise StopIteration
        blob = self.get_blob(self.index)
        self.index += 1
        return blob

    def __getitem__(self, index):
        if isinstance(index, int):
            return self.get_blob(index)
        elif isinstance(index, slice):
            return self._slice_generator(index)
        else:
            raise TypeError("index must be int or slice")

    def _slice_generator(self, index):
        """A simple slice generator for iterations"""
        start, stop, step = index.indices(len(self))
        for i in range(start, stop, step):
            yield self.get_blob(i)
<commit_msg>add first tabinit sketch<commit_after># coding=utf-8
# Filename: hdf5.py
# pylint: disable=C0103,R0903
# vim:set ts=4 sts=4 sw=4 et:
"""
Pumps for the EVT simulation dataformat.

"""
from __future__ import division, absolute_import, print_function

import os.path

import numpy as np
import tables

import km3pipe
from km3pipe import Pump, Module
from km3pipe.dataclasses import HitSeries, TrackSeries, EventInfo
from km3pipe.logger import logging
from km3pipe.tools import camelise

log = logging.getLogger(__name__)  # pylint: disable=C0103

__author__ = "Tamas Gal and Moritz Lotze"
__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Tamas Gal and Moritz Lotze"
__email__ = "tgal@km3net.de"
__status__ = "Development"

class HDF5Sink(Module):
    def __init__(self, **context):
        """A Module to convert (KM3NeT) ROOT files to HDF5."""
        super(self.__class__, self).__init__(**context)
        self.filename = self.get('filename') or 'dump.h5'
        self.index = 1
        self.h5file = tables.open_file(self.filename, mode="w", title="KM3NeT")
        self.filters = tables.Filters(complevel=5, shuffle=True,
                                      fletcher32=True)
        self._tables = {}
        desc = self.get('tables') or []
        self._descriptions = [
                ('Hits', '/hits', HitSeries.dtype),
                ('MCHits', '/mc_hits', HitSeries.dtype),
                ('MCTracks', '/mc_tracks', TrackSeries.dtype),
                ('EventInfo', '/event_info', EventInfo.dtype),
                ]
        self._descriptions.extend(desc)
        for key, path, dtype in self._descriptions:
            loc, tabname = os.path.split(path)
            tab = self.h5file.create_table(loc, tabname, description=dtype,
            title=key, filters=self.filters, createparents=True)
            self._tables[key] = tab

    def _write_reco_track(self, track, reco_row):
        for colname, val in track.items():
            reco_row[colname] = val
        reco_row.append()

    def _gen_dtype(self, track):
        keys = [(k, np.float64) for k in track.keys() if k != 'event_id' if k != 'did_converge']
        keys.extend([('event_id', np.uint32), ('did_converge', np.bool_)])
        return np.dtype(sorted(keys))

    def _write_reco(self, reco_dict, reco_group):
        for recname, track in reco_dict.items():
            if recname not in self._tables:
                dtype = self._gen_dtype(track)
                reco_table = self.h5file.create_table(
                    reco_group, recname.lower(),
                    dtype,      #np.dtype(recname_to_dtype[recname]),
                    recname, createparents=True, filters=self.filters
                )
                self._tables[recname] = reco_table
            reco_table = self._tables[recname]
            self._write_reco_track(track, reco_table.row)

    def process(self, blob):
        for key, entry in blob.items():
            if key == 'Reco':
                self._write_reco(entry, '/reco')
            # init h5 tables. maybe do this on 1st iter step only?
            try:
                dtype = entry.dtype
            except AttributeError:
                continue
            try:
                loc = entry.loc
            except AttributeError:
                loc = ''
            tabname + camelise(key)
            where = loc + '/' + camelise(key)
            if where not in self._tables.keys():
                self._tables[where] = self.h5file.create_table(
                    loc, tabname, description=dtype, title=key,
                    filters=self.filters, createparents=True)
            tab = self._tables[where]
            try:
                data = entry.as_table()
            except AttributeError:
                data = entry
            tab.append(data)

        if not self.index % 1000:
            for tab in self._tables.values():
                tab.flush()

        self.index += 1
        return blob

    def finish(self):
        for tab in self._tables.values():
            tab.cols.event_id.create_index()
        self.h5file.root._v_attrs.km3pipe = str(km3pipe.__version__)
        self.h5file.root._v_attrs.pytables = str(tables.__version__)
        self.h5file.close()

class HDF5Pump(Pump):
    """Provides a pump for KM3NeT HDF5 files"""
    def __init__(self, filename, **context):
        super(self.__class__, self).__init__(**context)
        self.filename = filename
        if os.path.isfile(self.filename):
            self.h5_file = tables.File(self.filename)
        else:
            raise IOError("No such file or directory: '{0}'"
                          .format(self.filename))
        self.index = None
        self._reset_index()

        try:
            event_info = self.h5_file.get_node('/', 'event_info')
            self.event_ids = event_info.cols.event_id[:]
        except tables.NoSuchNodeError:
            log.critical("No /event_info table found.")
            raise SystemExit

        self._n_events = len(self.event_ids)

    def process(self, blob):
        try:
            blob = self.get_blob(self.index)
        except KeyError:
            self._reset_index()
            raise StopIteration
        self.index += 1
        return blob

    def _get_hits(self, event_id, table_name='hits', where='/'):
        table = self.h5_file.get_node(where, table_name)
        rows = table.read_where('event_id == %d' % event_id)
        return HitSeries.from_table(rows, event_id)

    def _get_tracks(self, event_id, table_name='tracks', where='/'):
        table = self.h5_file.get_node(where, table_name)
        rows = table.read_where('event_id == %d' % event_id)
        return TrackSeries.from_table(rows, event_id)

    def _get_event_info(self, event_id, table_name='event_info', where='/'):
        table = self.h5_file.get_node(where, table_name)
        return EventInfo.from_table(table[event_id])

    def _get_reco(self, event_id, group_name='reco', where='/'):
        group = self.h5_file.get_node(where, group_name)
        out = {}
        for table in group:
            tabname = table.title
            data = table.read_where('event_id == %d' % event_id)
            out[tabname] = data
        return out

    def get_blob(self, index):
        event_id = self.event_ids[index]
        blob = {}
        blob['Hits'] = self._get_hits(event_id, table_name='hits')
        blob['MCHits'] = self._get_hits(event_id, table_name='mc_hits')
        blob['MCTracks'] = self._get_tracks(event_id, table_name='mc_tracks')
        blob['EventInfo'] = self._get_event_info(event_id,
                                                 table_name='event_info')
        blob['Reco'] = self._get_reco(event_id, group_name='reco')
        return blob

    def finish(self):
        """Clean everything up"""
        self.h5_file.close()

    def _reset_index(self):
        """Reset index to default value"""
        self.index = 0

    def __len__(self):
        return self._n_events

    def __iter__(self):
        return self

    def next(self):
        """Python 2/3 compatibility for iterators"""
        return self.__next__()

    def __next__(self):
        if self.index >= self._n_events:
            self._reset_index()
            raise StopIteration
        blob = self.get_blob(self.index)
        self.index += 1
        return blob

    def __getitem__(self, index):
        if isinstance(index, int):
            return self.get_blob(index)
        elif isinstance(index, slice):
            return self._slice_generator(index)
        else:
            raise TypeError("index must be int or slice")

    def _slice_generator(self, index):
        """A simple slice generator for iterations"""
        start, stop, step = index.indices(len(self))
        for i in range(start, stop, step):
            yield self.get_blob(i)
<|endoftext|>


<commit_before>'''time series prediction'''
from abc import ABCMeta, abstractmethod
import collections
import pdb
import numbers
import unittest

class CreateFeatures(object):
    'create features file from a master file and possibly many associated information files'
    def __init__(self):
        self.n_input_records = None
        self.n_output_records = None
        self.skipped = None  # collections.Counter for reasons input records were skipped
        pass

    def create(
        self,
        feature_makers=None,
        master_file_records=None,
        selected=None,   # lambda index, master_record: (use_record, maybe_error_message)
        # start optional arguments
        verbose=False,
    ):
        'yield sequence (features:Dict, index, master_record) of output records with features derived from the master file'
        def skip(msg):
            entire_msg = ('skipping %d %s; %s' % (self.n_input_records, index, msg))
            self.skipped[msg] += 1
            if verbose:
                print entire_msg

        def error(msg):
            print 'error in feature maker %s feature_name %s feature_value %s' % (
                feature_maker.name,
                feature_name,
                feature_value,
            )
            print msg
            print 'entering pdb'
            pdb.set_trace()

        assert feature_makers is not None
        assert master_file_records is not None
        assert selected is not None
        self.n_input_records = 0
        self.n_output_records = 0
        self.skipped = collections.Counter()
        for index, master_record in master_file_records:
            self.n_input_records += 1
            if self.n_input_records % 10000 == 1:
                print 'creating features from master record %d index %s' % (self.n_input_records, index)
            (use_record, msg) = selected(index, master_record)
            if not use_record:
                skip(msg)
                continue
            # create features from the master_record
            # the feature makers may incorporate data from other records
            features_made = {}
            stopped_early = False
            # accumulate all the features from the feature makers
            # check for errors on the way
            for feature_maker in feature_makers:
                maybe_features = feature_maker.make_features(index, master_record)
                if isinstance(maybe_features, str):
                    skip('no features from feature maker %s' % feature_maker.name)
                    stopped_early = True
                    break
                elif isinstance(maybe_features, dict):
                    for feature_name, feature_value in maybe_features.iteritems():
                        if feature_name in features_made:
                            error('duplicate feature name')
                        elif not feature_name.startswith('id_') and not isinstance(feature_value, numbers.Number):
                            error('feature value is not numeric')
                        elif feature_name.endswith('_size') and feature_value < 0.0:
                            error('size feature is negative')
                        else:
                            features_made[feature_name] = feature_value
                else:
                    error('unexpected return type from a feature_maker')
            if stopped_early:
                continue
            self.n_output_records += 1
            yield features_made, index, master_record

class FeatureMaker(object):
    __metaclass__ = ABCMeta

    def __init__(self, name=None):
        self.name = name  # used in error message; informal name of the feature maker

    @abstractmethod
    def make_features(ticker_index, tickercusip, ticker_record):
        'return errors:str or Dict[feature_name:str, feature_value:nmber]'
        pass

FitPredictResult = collections.namedtuple(
    'FitPredictResult',
    (
        'query_index',
        'query_features',
        'predicted_feature_name',
        'predicted_feature_value',
        'model_spec',
        'prediction',
        'fitted_model',
        'n_training_samples',
    )
)

class FitPredict(object):
    'fit models and predict targets'

    def fit_predict(
        self,
        df_features=None,
        df_targets=None,
        make_model=None,
        model_specs=None,
        timestamp_feature_name=None,
        already_seen_lambda=None,    # lambda query_index, model_spec, predicted_feature_name: Bool
    ):
        'yield either (True, result:FitPredictResult) or (False, error_msg:str)'
        # df_targets: a sorted sequence of targets, sorted in timestamp order (y values)
        sorted_targets = df_targets.sort_values(by=timestamp_feature_name)
        for query_index in sorted_targets.index:
            if query_index not in df_features.index:
                yield False, 'no query feature for query index %s' % query_index
                continue
            query_features = df_features.loc[[query_index]]  # must be a DataFrame
            assert len(query_features) == 1
            timestamp = query_features.iloc[0][timestamp_feature_name]
            mask = sorted_targets[timestamp_feature_name] < timestamp
            training_targets = sorted_targets.loc[mask]
            if len(training_targets) == 0:
                yield False, 'no training_targets for query index %s timestamp %s' % (query_index, timestamp)
                continue
            training_features = df_features.loc[training_targets.index]
            if len(training_features) == 0:
                yield False, 'no training_features for query index %s timestamp %s' % (query_index, timestamp)
                continue
            for predicted_feature_name, predicted_feature_value in sorted_targets.loc[query_index].iteritems():
                if predicted_feature_name.startswith('id_'):
                    continue  # skip identifiers, as these are not features
                if predicted_feature_name.endswith('_decreased') or predicted_feature_name.endswith('_increased'):
                    yield False, 'classification not yet implemented; target feature name %s' % predicted_feature_name
                    continue
                for model_spec in model_specs:
                    if already_seen_lambda(query_index, model_spec, predicted_feature_name):
                        yield False, 'already seen: %s %s %s' % (query_index, model_spec, predicted_feature_name)
                        continue
                    m = make_model(model_spec, predicted_feature_name)
                    try:
                        # TODO: turn into keywords
                        m.fit(training_features, training_targets)
                    except Exception as e:
                        yield False, 'exception raised during fitting: %s' % e
                    predictions = m.predict(query_features)
                    assert len(predictions) == 1
                    yield (
                        True,
                        FitPredictResult(
                            query_index=query_index,
                            query_features=query_features,
                            predicted_feature_name=predicted_feature_name,
                            predicted_feature_value=predicted_feature_value,
                            model_spec=model_spec,
                            prediction=predictions[0],
                            fitted_model=m,
                            n_training_samples=len(training_features),
                        )
                    )

class FitPredictOutput(object):
    'content of output file for program fit_predict.py'
    __metaclass__ = ABCMeta

    @abstractmethod
    def as_dict(self):
        'return a dict with all the fields'
        pass

class HpChoices(object):
    'iterated over HpSpec instances'
    __metaclass__ = ABCMeta

    @abstractmethod
    def __iter__(self):
        'yield sequence of HpSpec objects'
        pass

class Model(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def fit(self, df_training_samples_features, df_training_samples_targets):
        'mutate self; set attribute importances: Dict[feature_name:str, feature_importance:Number]'
        pass

    @abstractmethod
    def predict(self, df_query_samples_features):
        'return predictions'
        pass

class ModelSpec(object):
    'specification of a model name and its associated hyperparamters'
    __metaclass__ = ABCMeta

    @abstractmethod
    def __str__(self):
        'return parsable string representation'
        # Hint: Use method self._to_str(value) to convert individual values to strings
        # That will make all the string representations use the same encoding of values to strings
        pass

    @staticmethod
    @abstractmethod
    def make_from_str(s):
        'parse the representation returned by str(self) to create an instance'
        pass

    @abstractmethod
    def iteritems(self):
        'yield each (hyparameter name:str, hyperparameter value)'
        pass

    @abstractmethod
    def __eq__(self, other):
        pass

    @abstractmethod
    def __hash__(self):
        pass

    @abstractmethod
    def __lt__(self, other):
        pass

    def _to_str(self, value):
        'internal method. Convert value to a string. Use me in your __str__ method'
        def remove_trailing_zeroes(s):
            return (
                s if s[-1] != '0' else
                remove_trailing_zeroes(s[:-1])
            )
        if value is None:
            return ''
        elif isinstance(value, float):
            return remove_trailing_zeroes(('%f' % value).replace('.', '_'))
        elif isinstance(value, int):
            return '%d' % value
        else:
            return str(value)

class TestHpChoices(unittest.TestCase):
    def test_construction(self):
        self.assertRaises(Exception, HpChoices, None)

class TestHpSpeC(unittest.TestCase):
    def test_construction(self):
        self.assertRaises(Exception, HpSpec, None)

if __name__ == '__main__':
    unittest.main()
<commit_msg>better error handling<commit_after>'''time series prediction'''
from abc import ABCMeta, abstractmethod
import collections
import pdb
import numbers
import unittest

class CreateFeatures(object):
    'create features file from a master file and possibly many associated information files'
    def __init__(self):
        self.n_input_records = None
        self.n_output_records = None
        self.skipped = None  # collections.Counter for reasons input records were skipped
        pass

    def create(
        self,
        feature_makers=None,
        master_file_records=None,
        selected=None,   # lambda index, master_record: (use_record, maybe_error_message)
        # start optional arguments
        verbose=False,
    ):
        'yield sequence (features:Dict, index, master_record) of output records with features derived from the master file'
        def skip(msg):
            entire_msg = ('skipping %d %s; %s' % (self.n_input_records, index, msg))
            self.skipped[msg] += 1
            if verbose:
                print entire_msg

        def error(msg):
            print 'error in feature maker %s feature_name %s feature_value %s' % (
                feature_maker.name,
                feature_name,
                feature_value,
            )
            print msg
            print 'entering pdb'
            pdb.set_trace()

        assert feature_makers is not None
        assert master_file_records is not None
        assert selected is not None
        self.n_input_records = 0
        self.n_output_records = 0
        self.skipped = collections.Counter()
        for index, master_record in master_file_records:
            self.n_input_records += 1
            if self.n_input_records % 10000 == 1:
                print 'creating features from master record %d index %s' % (self.n_input_records, index)
            (use_record, msg) = selected(index, master_record)
            if not use_record:
                skip(msg)
                continue
            # create features from the master_record
            # the feature makers may incorporate data from other records
            features_made = {}
            stopped_early = False
            # accumulate all the features from the feature makers
            # check for errors on the way
            for feature_maker in feature_makers:
                maybe_features = feature_maker.make_features(index, master_record)
                if isinstance(maybe_features, str):
                    skip('no features from feature maker %s' % feature_maker.name)
                    stopped_early = True
                    break
                elif isinstance(maybe_features, dict):
                    for feature_name, feature_value in maybe_features.iteritems():
                        if feature_name in features_made:
                            error('duplicate feature name')
                        elif not feature_name.startswith('id_') and not isinstance(feature_value, numbers.Number):
                            error('feature value is not numeric')
                        elif feature_name.endswith('_size') and feature_value < 0.0:
                            error('size feature is negative')
                        else:
                            features_made[feature_name] = feature_value
                else:
                    print feature_maker.name
                    print feature_maker
                    print maybe_features
                    print type(maybe_features)
                    error('unexpected return type from a feature_maker')
            if stopped_early:
                continue
            self.n_output_records += 1
            yield features_made, index, master_record

class FeatureMaker(object):
    __metaclass__ = ABCMeta

    def __init__(self, name=None):
        self.name = name  # used in error message; informal name of the feature maker

    @abstractmethod
    def make_features(ticker_index, tickercusip, ticker_record):
        'return errors:str or Dict[feature_name:str, feature_value:nmber]'
        pass

FitPredictResult = collections.namedtuple(
    'FitPredictResult',
    (
        'query_index',
        'query_features',
        'predicted_feature_name',
        'predicted_feature_value',
        'model_spec',
        'prediction',
        'fitted_model',
        'n_training_samples',
    )
)

class FitPredict(object):
    'fit models and predict targets'

    def fit_predict(
        self,
        df_features=None,
        df_targets=None,
        make_model=None,
        model_specs=None,
        timestamp_feature_name=None,
        already_seen_lambda=None,    # lambda query_index, model_spec, predicted_feature_name: Bool
    ):
        'yield either (True, result:FitPredictResult) or (False, error_msg:str)'
        # df_targets: a sorted sequence of targets, sorted in timestamp order (y values)
        sorted_targets = df_targets.sort_values(by=timestamp_feature_name)
        for query_index in sorted_targets.index:
            if query_index not in df_features.index:
                yield False, 'no query feature for query index %s' % query_index
                continue
            query_features = df_features.loc[[query_index]]  # must be a DataFrame
            assert len(query_features) == 1
            timestamp = query_features.iloc[0][timestamp_feature_name]
            mask = sorted_targets[timestamp_feature_name] < timestamp
            training_targets = sorted_targets.loc[mask]
            if len(training_targets) == 0:
                yield False, 'no training_targets for query index %s timestamp %s' % (query_index, timestamp)
                continue
            training_features = df_features.loc[training_targets.index]
            if len(training_features) == 0:
                yield False, 'no training_features for query index %s timestamp %s' % (query_index, timestamp)
                continue
            for predicted_feature_name, predicted_feature_value in sorted_targets.loc[query_index].iteritems():
                if predicted_feature_name.startswith('id_'):
                    continue  # skip identifiers, as these are not features
                if predicted_feature_name.endswith('_decreased') or predicted_feature_name.endswith('_increased'):
                    yield False, 'classification not yet implemented; target feature name %s' % predicted_feature_name
                    continue
                for model_spec in model_specs:
                    if already_seen_lambda(query_index, model_spec, predicted_feature_name):
                        yield False, 'already seen: %s %s %s' % (query_index, model_spec, predicted_feature_name)
                        continue
                    m = make_model(model_spec, predicted_feature_name)
                    try:
                        # TODO: turn into keywords
                        m.fit(training_features, training_targets)
                    except Exception as e:
                        yield False, 'exception raised during fitting: %s' % e
                    predictions = m.predict(query_features)
                    assert len(predictions) == 1
                    yield (
                        True,
                        FitPredictResult(
                            query_index=query_index,
                            query_features=query_features,
                            predicted_feature_name=predicted_feature_name,
                            predicted_feature_value=predicted_feature_value,
                            model_spec=model_spec,
                            prediction=predictions[0],
                            fitted_model=m,
                            n_training_samples=len(training_features),
                        )
                    )

class FitPredictOutput(object):
    'content of output file for program fit_predict.py'
    __metaclass__ = ABCMeta

    @abstractmethod
    def as_dict(self):
        'return a dict with all the fields'
        pass

class HpChoices(object):
    'iterated over HpSpec instances'
    __metaclass__ = ABCMeta

    @abstractmethod
    def __iter__(self):
        'yield sequence of HpSpec objects'
        pass

class Model(object):
    __metaclass__ = ABCMeta

    @abstractmethod
    def fit(self, df_training_samples_features, df_training_samples_targets):
        'mutate self; set attribute importances: Dict[feature_name:str, feature_importance:Number]'
        pass

    @abstractmethod
    def predict(self, df_query_samples_features):
        'return predictions'
        pass

class ModelSpec(object):
    'specification of a model name and its associated hyperparamters'
    __metaclass__ = ABCMeta

    @abstractmethod
    def __str__(self):
        'return parsable string representation'
        # Hint: Use method self._to_str(value) to convert individual values to strings
        # That will make all the string representations use the same encoding of values to strings
        pass

    @staticmethod
    @abstractmethod
    def make_from_str(s):
        'parse the representation returned by str(self) to create an instance'
        pass

    @abstractmethod
    def iteritems(self):
        'yield each (hyparameter name:str, hyperparameter value)'
        pass

    @abstractmethod
    def __eq__(self, other):
        pass

    @abstractmethod
    def __hash__(self):
        pass

    @abstractmethod
    def __lt__(self, other):
        pass

    def _to_str(self, value):
        'internal method. Convert value to a string. Use me in your __str__ method'
        def remove_trailing_zeroes(s):
            return (
                s if s[-1] != '0' else
                remove_trailing_zeroes(s[:-1])
            )
        if value is None:
            return ''
        elif isinstance(value, float):
            return remove_trailing_zeroes(('%f' % value).replace('.', '_'))
        elif isinstance(value, int):
            return '%d' % value
        else:
            return str(value)

class TestHpChoices(unittest.TestCase):
    def test_construction(self):
        self.assertRaises(Exception, HpChoices, None)

class TestHpSpeC(unittest.TestCase):
    def test_construction(self):
        self.assertRaises(Exception, HpSpec, None)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import module
import youtube
from irc import IRCMessage, IRCUser
import sys
sys.path.append ("../")
import toolbox
from toolbox import irccode
import requests, re

class URLModule(module.Module):
	def run (self):
		if self.message.content:
			contentParams = self.message.content.split()
			for url in contentParams:
				# check if url is YOUTUBE url
				match = toolbox.is_youtube_url (url)
				#print match
				if match:
					#ding ding: youtube URL, pass it to YT module
					new_message = self.message
					new_message.content = match.group()
					youtube_module = youtube.YouTubeModule(sender=self.sender, message=new_message, ircSock=self.ircSock)
					youtube_module.run()
					continue
				if toolbox.is_proper_url (url):
					try:
						# get URL's title
						resp = requests.get(url)
						# print it
						title = resp.text[resp.text.find("<title>")+7:resp.text.find("</title>")]
						# send it back
						title = re.sub (r'([^ ]&\s)', '', title)
						if title:
							self.pm (self.message.get_reply_to(), irccode("DARK_MAGNETA") + "Link Title: " + irccode("PURPLE") + title)
					except Exception:
						pass<commit_msg>Made regular expression clearer and more general when sanitizing whitespace from website titles.<commit_after># -*- coding: utf-8 -*-
import module
import youtube
from irc import IRCMessage, IRCUser
import sys
sys.path.append ("../")
import toolbox
from toolbox import irccode
import requests, re

class URLModule(module.Module):
	def run (self):
		if self.message.content:
			contentParams = self.message.content.split()
			for url in contentParams:
				# check if url is YOUTUBE url
				match = toolbox.is_youtube_url (url)
				#print match
				if match:
					#ding ding: youtube URL, pass it to YT module
					new_message = self.message
					new_message.content = match.group()
					youtube_module = youtube.YouTubeModule(sender=self.sender, message=new_message, ircSock=self.ircSock)
					youtube_module.run()
					continue
				if toolbox.is_proper_url (url):
					try:
						# get URL's title
						resp = requests.get(url)
						# print it
						title = resp.text[resp.text.find("<title>")+7:resp.text.find("</title>")]
						# Remove non-space whitespace
						title = re.sub(r'([\r\n\t])', '', title)
						# send it back
						if title:
							self.pm (self.message.get_reply_to(), irccode("DARK_MAGNETA") + "Link Title: " + irccode("PURPLE") + title)
					except Exception:
						pass
<|endoftext|>


<commit_before>import numpy
import math
from mta.dataset import Dataset
from copy import copy

class Profile:
    @classmethod
    def rating_dist(self, dataset, stacks=11, stack_range=1000):
        dist = numpy.zeros(stacks, dtype=int)
        labels = numpy.array(list( i*stack_range for i in range(stacks)))
        for u_id, i_id, rating in dataset.ratings.to_list():
            if rating >= stacks* stack_range:
                dist[stacks-1] +=1
            else:
                index = math.floor(rating/stack_range)
                dist[index] +=1
        return dist, labels

    @classmethod
    def rating_std_dist_on_items(self, dataset, stacks=11, stack_range=100):
        item_size = dataset.ratings.matrix_shape()[1]
        rating_stds  = numpy.zeros(item_size, dtype=float)
        rating_matrix = dataset.ratings.to_matrix()
        dist = numpy.zeros(stacks, dtype=int)
        labels = numpy.array(list( i*stack_range for i in range(stacks)))

        for i_id in range(len(rating_matrix[0])):
            rating_stds[i_id] = rating_matrix[:,i_id].std(ddof=1)

        for std_value in rating_stds:
            if std_value >= stacks* stack_range:
                dist[stacks-1]+=1
            else:
                index = math.floor(std_value/stack_range)
                dist[index]+=1
        return dist, labels

    #get the distribution of number of touched factors in each users
    @classmethod
    def factor_dist_on_users(self, dataset, stacks=21, stack_range=1):
        user_size = dataset.matrix_shape()[0]
        factor_size = dataset.matrix_shape()[1]
        user_num_factors = numpy.zeros(user_size, dtype=int)
        dist = numpy.zeros(stacks, dtype=int)
        labels = numpy.array(list( i*stack_range for i in range(stacks)))

        user_factor_matrix = dataset.touchs.to_matrix()
        for u_id in range(user_size):
            user_num_factors[u_id] = user_factor_matrix[u_id].sum()
        for num_factor in user_num_factors:
            if num_factor >= stacks*stack_range:
                dist[stacks-1] +=1
            else :
                index = math.floor(num_factor/stack_range)
                dist[index] +=1
        return dist, labels

    #get the distribution of number of touched factors in each items
    @classmethod
    def factor_dist_on_items(self, dataset,  stacks=21, stack_range=1):
        item_size = dataset.matrix_shape()[2]
        factor_size = dataset.matrix_shape()[1]
        item_factor_table  = numpy.zeros((item_size, factor_size), dtype=bool)
        item_num_factors = numpy.zeros(item_size, dtype=int)
        dist = numpy.zeros(stacks, dtype=int)
        labels = numpy.array(list( i*stack_range for i in range(stacks)))

        user_factor_matrix = dataset.touchs.to_matrix()
        for u_id, i_id, rating in dataset.ratings.to_list():
            for f_id in range(factor_size):
                if user_factor_matrix[u_id][f_id] ==1:
                    item_factor_table[i_id][f_id] =True
        for i_id in range(len(item_factor_table)):
            item_num_factors[i_id] = item_factor_table[i_id].sum()

        for num_factor in item_num_factors:
            if num_factor >= stacks*stack_range:
                dist[stacks-1] +=1
            else :
                index = math.floor(num_factor/stack_range)
                dist[index] +=1
        return dist, labels

    #get the distribution of number of products in each factors
    @classmethod
    def item_dist(self, dataset, stacks=11, stack_range=10):
        matrix_shape = dataset.ratings.matrix_shape()
        items  = numpy.zeros(matrix_shape[1], dtype=int)

        for u_id, i_id, rating in dataset.ratings.to_list():
            items[i_id] +=1
        dist= numpy.zeros(stacks, dtype=int)
        for item_size in items:
            if item_size >= stacks* stack_range:
                dist[stacks-1]+=1
            else:
                index = math.floor(item_size/stack_range)
                dist[index]+=1
        return dist

    #get the distribution of number of products in each factors
    @classmethod
    def item_dist_on_factors(self, dataset, stacks=11, stack_range=10):
        matrix_shape = ( dataset.matrix_shape()[1],dataset.matrix_shape()[2])
        factor_item_table  = numpy.zeros(matrix_shape, dtype=bool)
        factor_num_items = numpy.zeros(matrix_shape[0], dtype=int)
        dist = numpy.zeros(matrix_shape[1], dtype=int)
        user_factor_matrix = dataset.touchs.to_matrix()

        for u_id, i_id, rating in dataset.ratings.to_list():
            for f_id in range(matrix_shape[0]):
                if user_factor_matrix[u_id][f_id] ==1:
                    factor_item_table[f_id][i_id] =True

        for f_id in range(len(factor_item_table)):
            factor_num_items[f_id] = factor_item_table[f_id].sum()

        for num_item in factor_num_items:
            dist[num_item] +=1

        for index in reversed(range(len(dist))):
            if dist[index] >0:
                dist = dist[:index+1]
                break
        return dist

<commit_msg>refactor profile.py<commit_after>import numpy
import math
from mta.dataset import Dataset
from copy import copy

class Profile:
    @classmethod
    def rating_dist(self, dataset, stacks=11, stack_range=1000):
        rating_list = numpy.array(dataset.ratings.to_list())[:,2].flat

        dist = self._make_dist(rating_list, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    @classmethod
    def rating_std_dist_on_items(self, dataset, stacks=11, stack_range=100):
        item_size = dataset.ratings.matrix_shape()[1]
        rating_stds  = numpy.zeros(item_size, dtype=float)
        rating_matrix = dataset.ratings.to_matrix()
        for i_id in range(len(rating_matrix[0])):
            rating_stds[i_id] = rating_matrix[:,i_id].std(ddof=1)

        dist = self._make_dist(rating_stds, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    #get the distribution of number of touched factors in each users
    @classmethod
    def factor_dist_on_users(self, dataset, stacks=21, stack_range=1):
        user_size = dataset.matrix_shape()[0]
        factor_size = dataset.matrix_shape()[1]
        user_num_factors = numpy.zeros(user_size, dtype=int)
        user_factor_matrix = dataset.touchs.to_matrix()
        for u_id in range(user_size):
            user_num_factors[u_id] = user_factor_matrix[u_id].sum()

        dist = self._make_dist(user_num_factors, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    #get the distribution of number of touched factors in each items
    @classmethod
    def factor_dist_on_items(self, dataset,  stacks=21, stack_range=1):
        item_size = dataset.matrix_shape()[2]
        factor_size = dataset.matrix_shape()[1]
        item_factor_table  = numpy.zeros((item_size, factor_size), dtype=bool)
        item_num_factors = numpy.zeros(item_size, dtype=int)
        user_factor_matrix = dataset.touchs.to_matrix()
        for u_id, i_id, rating in dataset.ratings.to_list():
            for f_id in range(factor_size):
                if user_factor_matrix[u_id][f_id] ==1:
                    item_factor_table[i_id][f_id] =True
        for i_id in range(len(item_factor_table)):
            item_num_factors[i_id] = item_factor_table[i_id].sum()

        dist = self._make_dist(item_num_factors, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    @classmethod
    def item_dist(self, dataset, stacks=11, stack_range=10):
        item_size = dataset.matrix_shape()[2]
        items  = numpy.zeros(item_size, dtype=int)
        for u_id, i_id, rating in dataset.ratings.to_list():
            items[i_id] +=1

        dist = self._make_dist(items, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    #get the distribution of number of products in each factors
    @classmethod
    def item_dist_on_factors(self, dataset, stacks=11, stack_range=10):
        item_size = dataset.matrix_shape()[2]
        factor_size = dataset.matrix_shape()[1]
        factor_item_table  = numpy.zeros((factor_size, item_size), dtype=bool)
        factor_num_items = numpy.zeros(factor_size, dtype=int)
        user_factor_matrix = dataset.touchs.to_matrix()
        for u_id, i_id, rating in dataset.ratings.to_list():
            for f_id in range(factor_size):
                if user_factor_matrix[u_id][f_id] ==1:
                    factor_item_table[f_id][i_id] =True
        for f_id in range(len(factor_item_table)):
            factor_num_items[f_id] = factor_item_table[f_id].sum()

        dist = self._make_dist(factor_num_items, stacks, stack_range)
        labels = self._make_labels(stacks, stack_range)
        return dist, labels

    def _make_dist(value_list, stacks, stack_range ):
        dist = numpy.zeros(stacks, dtype=int)
        for value in value_list:
            if value >= stacks* stack_range:
                dist[stacks-1] +=1
            else:
                index = math.floor(value/stack_range)
                dist[index] +=1
        return dist

    def _make_labels(stacks, stack_range):
        return numpy.array(list( i*stack_range for i in range(stacks)))
<|endoftext|>


<commit_before>from tkinter import *
import webbrowser

# window size and position variables
windowSizeX = "300"
windowSizeY = "300"

quitButtonX = 265
quitButtonY = 270

entertainTabsButtonX = 100
entertainTabsButtonY = 100

workingTabsButtonX = 50
workingTabsButtonY = 50

# prints out the registered browsers I can use
# print(webbrowser._tryorder)

# registers firefox (i think this is done correctly)
firefoxPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe %s &"
firefoxBrowser = webbrowser.get(firefoxPath)

# web links to be opened in tabs
urlYoutube = "https://www.youtube.com/"
urlReddit = "https://us.reddit.com/"
urlWorkReddit = "https://us.reddit.com/"
urlStackOverflow = "http://stackoverflow.com/"

class Window(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.master = master
        self.init_window()

    def init_window(self):
        self.master.title("Auto-Tab-Opener")
        self.pack(fill=BOTH, expand=1)

        # exit button
        quitButton = Button(self, text="Exit", command=self.client_exit)
        quitButton.place(x=quitButtonX, y=quitButtonY)

        # start firefox / chrome entertainment tabs button
        entertainTabsButton = Button(self, text="Entertainment Mode", command=self.entertain_Button)
        entertainTabsButton.place(x=entertainTabsButtonX, y=entertainTabsButtonY)

        # start firefox / chrome working tabs button
        workingTabsButton = Button(self, text="Working Mode", command=self.working_Button)
        workingTabsButton.place(x=workingTabsButtonX, y=workingTabsButtonY)

    # browser button command
    def entertain_Button(self):
        print("entertainment button pressed")
        firefoxBrowser.open(urlYoutube)
        firefoxBrowser.open(urlReddit)

    # browser button command 2
    def working_Button(self):
        print("working button pressed")
        firefoxBrowser.open(urlWorkReddit)
        firefoxBrowser.open(urlStackOverflow)

    def client_exit(self):
        print("Exit button pressed")
        exit()

# base window variable
root = Tk()
app = Window(root)

# Sets size for window
root.geometry(windowSizeX + "x" + windowSizeY)
root.mainloop()

# Manual exit confirmation written by me
print("Program has reached this point.")
<commit_msg>Added another link<commit_after>from tkinter import *
import webbrowser

# window size and position variables
windowSizeX = "300"
windowSizeY = "300"

quitButtonX = 265
quitButtonY = 270

entertainTabsButtonX = 100
entertainTabsButtonY = 100

workingTabsButtonX = 50
workingTabsButtonY = 50

# prints out the registered browsers I can use
# print(webbrowser._tryorder)

# registers firefox (i think this is done correctly)
firefoxPath = "C:/Program Files (x86)/Mozilla Firefox/firefox.exe %s &"
firefoxBrowser = webbrowser.get(firefoxPath)

# web links to be opened in tabs
urlYoutube = "https://www.youtube.com/"
urlReddit = "https://us.reddit.com/"
urlWorkReddit = "https://us.reddit.com/"
urlStackOverflow = "http://stackoverflow.com/"
urlGithub = "https://github.com/"

class Window(Frame):
    def __init__(self, master=None):
        Frame.__init__(self, master)
        self.master = master
        self.init_window()

    def init_window(self):
        self.master.title("Auto-Tab-Opener")
        self.pack(fill=BOTH, expand=1)

        # exit button
        quitButton = Button(self, text="Exit", command=self.client_exit)
        quitButton.place(x=quitButtonX, y=quitButtonY)

        # start firefox / chrome entertainment tabs button
        entertainTabsButton = Button(self, text="Entertainment Mode", command=self.entertain_Button)
        entertainTabsButton.place(x=entertainTabsButtonX, y=entertainTabsButtonY)

        # start firefox / chrome working tabs button
        workingTabsButton = Button(self, text="Working Mode", command=self.working_Button)
        workingTabsButton.place(x=workingTabsButtonX, y=workingTabsButtonY)

        w = Label(self, text="testing")
        w.place(anchor="s")

    # browser button command
    def entertain_Button(self):
        print("entertainment button pressed")
        firefoxBrowser.open(urlYoutube)
        firefoxBrowser.open(urlReddit)

    # browser button command 2
    def working_Button(self):
        print("working button pressed")
        firefoxBrowser.open(urlWorkReddit)
        firefoxBrowser.open(urlStackOverflow)
        firefoxBrowser.open(urlGithub)

    # quit button command
    def client_exit(self):
        print("Exit button pressed")
        exit()

# base window variable
root = Tk()
app = Window(root)

# Sets size for window
root.geometry(windowSizeX + "x" + windowSizeY)
root.mainloop()

# Manual exit confirmation written by me
print("Program has reached this point.")
<|endoftext|>


<commit_before>class caesar(object):
    def __init__(self):
        self.cipherType = 'caesar'
        return
    def chrToNum(char):
		#converts to lower case
		num = ord(char)
		num -= 97
	return num

    def numToChr(num):
		#accepts numbers in lowercase ascii form
		char = ord(num + 97)
		return char
    def az(letter):
		if 97 <= ord(letter) <= 122:
    	   return True
    	else:
    	   return False
    def encrypt(self,msg,key):
        #ENCRYPTION CODE
		encrypted = ''
		for letter in msg:
			letter = letter.lower()
			if az(letter):
				encrypted += numToChr((chrToNum(letter) + key) % 26)
			else:
				continue
		return encrypted
    def decrypt(self,msg,key):
        #DECRYPTION CODE
		decrypted = ''
		for letter in msg:
			letter = letter.lower()
			if az(letter):
				decrypted += numToChr((chrToNum(letter) - key) % 26)
			else:
				continue
		return decrypted
    def frequencyAnalysis(self,msg):
        #ANALYSIS CODE
		frequency = [0] * 26
		for letter in msg:
			if 97 <= ord(letter) <= 122:
				frequency[chrToNum(letter)] += 1
			else:
				continue
		total = 0
		for index in range(0,26):
			total += frequency[index]
		total = float(total)
		for indexAlpha in range(0,len(frequency)):
			frequency[indexAlpha] = frequency[indexAlpha] / total
		return frequency
    def variance(self,frequency):
        #VARIATION CODE
		variance = 0
		englishVariance = [0.0867,0.01492,0.02782,0.04253,0.12702,0.02228,0.02015,0.06094,0.06966,0.00153,0.00772,0.04025,0.02406,0.06749,0.07507,0.01929,0.00095,0.05987,0.06327,0.09056,0.02758,0.00978,0.02361,0.00150,0.01974,0.00074]
		for index in range(0,26):
			variance += abs(englishVariance[index]-frequency[index])
		return variance
    def noKeysDecrypt(self,msg):
        #NO KEY DECRYPTION CODE # returns a list where the first item is the decrypted msg and the 2nd the key
		key = 0
		lowestVariance = 1000
		testKey = 0
		for index in range(0,26):
			testDecrypt = self.decrypt(msg,key)
			testFrequency = self.frequencyAnalysis(testDecrypt)
			testVariance = variance(testFrequency)
			if testVariance < lowestVariance:
				lowestVariance = testVariance
				testKey = key
			key += 1
		return [self.decrypt(msg,testKey), testKey]
class afine(caesarMsg):
    def __init__(self):
        self.cipherType = 'afine'
        return
    def getValidMultkeys(num):
    	# Should retrun all intigers relativly prime to num
		validKeys = []
		for indexA in range(0,num):
			for indexB in range(0,num):
				if ((indexA *indexB) % num) == 1:
					validKeys.append(indexA)
				else:
					continue
		return validKeys
	def factor(n):
		#should factor n and return a list of factors
    def isValidMultKey(key,num):
    	#check if key is relativly prime to num return true or false
    def getInvrsMultKey(key):
    	# gets the inverse of the mult key , asumes 26 , watches for keys that do not have a reverse
		inverseKey = None
		for index in range(0,26):
			if ((key * index) % 26) == 1:
				inverseKey = index
			else:
				continue
		return inverseKey
    def getInvrsAddKey(key,delta = 'pos'):
    	#or delta = 'neg', assumes 26
    	if delta == 'neg':
    	   key = 0-key
    	   return key
    	else:
    	   key = 25-key ######## THIS MAY CAUSE AN ERROR
    def encrypt(self, msg,multKey,addKey):
        #ENCRYPTION CODE
        encrypted = ''
        for letter in msg:
           letter = letter.lower()
           if az(letter):
              encrypted += self.numToChr(((self.chrToNum(letter) + addKey) * multKey) % 26)
           else:
              continue
        return encrypted
    def decrypt(self,msg,multKey,addKey):
        #DECRYPTION CODE
        decrypted = ''
        for letter in msg:
           letter = letter.lower()
           if az(letter):
              decrypted += self.numToChr(((self.chrToNum(letter) * self.getInvrsMultKey(multKey)) - getInvrsAddKey(addKey)) % 26)
           else:
              continue
        return decrypted
    def keysValidate(self,keys):
		if type(keys) != str:
			return False
		else:
			for letter in keys:
				if !(az(letter)):
					return False
		else:
			return True
    def noKeysDecrypt(self,msg):
        #NO KEY DECRYPTION CODE
		lowestVariance = 1000
		testMultKey = None
		testAddKey = None
		for indexA in getValidMultkeys(26):
			for indexB in range(0,26):
				testDecrypt = self.decrypt(msg, indexA,indexB)
				testFrequency = frequencyAnalysis(testDecrypt)
				testVariance = variance(testFrequency)
				if testVariance < lowestVariance:
					lowestVariance = testVariance
					testMultKey = indexA
					testAddKey = indexB
		return [self.decrypt(msg,testMultKey,testAddKey),testMultKey,testAddKey]
class viginere(afineMsg):
    def __init__(self):
        self.cipherType = 'viginere'
	return
    def encrypt(self,msg, keyWord):
        #ENCRYPTION CODE
        encrypted = ''
        counter = 0
        lenK = len(keyword) -1
        for letter in msg:
			letter = letter.lower()
			if az(letter):
				encrypted += self.numToChr((self.chrToNum(letter) + self.chrToNum(keyWord[counter])) % 26)
				counter += 1
				if counter == lenK:
					counter = 0
			else:
				continue
		return encrypted
    def decrypt(self, msg, keyWord):
        #DECRYPTION CODE
		decrypted = ''
        counter = 0
        lenK = len(keyword) -1
        for letter in msg:
			letter = letter.lower()
			if az(letter):
				decrypted += self.numToChr((self.chrToNum(letter) - self.chrToNum(keyWord[counter])) % 26)
				counter += 1
				if counter == lenK:
					counter = 0
			else:
				continue
		return decrypted
    def frequencyAnalysis(self):
        #ANALYSIS CODE
    def msgVariation(self):
        #MSG VARIATION CODE
    def keysValidate(self):
        #KEY VALIDATION
    def msgValidate(self):
        #MSG VALIDATION CODE
    def noKeysDecrypt(self):
        #NO KEY DECRYPTION CODE
    def searchMsg(self,arg):
        #SEARCH THE self.msg FOR THE arg STRING
class viginereOldMsg(viginereMsg):
    def __init__(self,keys,msg):
        self.keys = []
        self.cipherType = 'viginereold'
        self.msg = msg
    def encrypt(self):
        #ENCRYPTION CODE
    def decrypt(self):
        #DECRYPTION CODE
    def frequencyAnalysis(self):
        #ANALYSIS CODE
    def msgVariation(self):
        #MSG VARIATION CODE
    def keysValidate(self):
        #KEY VALIDATION
    def msgValidate(self):
        #MSG VALIDATION CODE
    def noKeysDecrypt(self):
        #NO KEY DECRYPTION CODE
    #INHERITS .searchMsg()
<commit_msg>fixed indentation errors<commit_after>def opp(t):
	if t == True:
		return False
	elif t == False:
		return True

class caesar(object):
	def __init__(self):
		self.cipherType = 'caesar'
		return
	def chrToNum(char):
		#converts to lower case
		num = ord(char)
		num -= 97
		return num
	def numToChr(num):
		#accepts numbers in lowercase ascii form
		char = ord(num + 97)
		return char
	def az(letter):
		if 97 <= ord(letter) <= 122:
			return True
		else:
			return False
	def encrypt(self,msg,key):
		#ENCRYPTION CODE
		encrypted = ''
		for letter in msg:
			letter = letter.lower()
			if self.az(letter):
				encrypted += numToChr((chrToNum(letter) + key) % 26)
			else:
				continue
		return encrypted
	def decrypt(self,msg,key):
		#DECRYPTION CODE
		decrypted = ''
		for letter in msg:
			letter = letter.lower()
			if az(letter):
				decrypted += numToChr((chrToNum(letter) - key) % 26)
			else:
				continue
		return decrypted
	def frequencyAnalysis(self,msg):
		#ANALYSIS CODE
		frequency = [0] * 26
		for letter in msg:
			if 97 <= ord(letter) <= 122:
				frequency[chrToNum(letter)] += 1
			else:
				continue
		total = 0
		for index in range(0,26):
			total += frequency[index]
		total = float(total)
		for indexAlpha in range(0,len(frequency)):
			frequency[indexAlpha] = frequency[indexAlpha] / total
		return frequency
	def variance(self,frequency):
		#VARIATION CODE
		variance = 0
		englishVariance = [0.0867,0.01492,0.02782,0.04253,0.12702,0.02228,0.02015,0.06094,0.06966,0.00153,0.00772,0.04025,0.02406,0.06749,0.07507,0.01929,0.00095,0.05987,0.06327,0.09056,0.02758,0.00978,0.02361,0.00150,0.01974,0.00074]
		for index in range(0,26):
			variance += abs(englishVariance[index]-frequency[index])
		return variance
	def noKeysDecrypt(self,msg):
		#NO KEY DECRYPTION CODE # returns a list where the first item is the decrypted msg and the 2nd the key
		key = 0
		lowestVariance = 1000
		testKey = 0
		for index in range(0,26):
			testDecrypt = self.decrypt(msg,key)
			testFrequency = self.frequencyAnalysis(testDecrypt)
			testVariance = variance(testFrequency)
			if testVariance < lowestVariance:
				lowestVariance = testVariance
				testKey = key
			key += 1
		return [self.decrypt(msg,testKey), testKey]
class afine(caesar):
	def __init__(self):
		self.cipherType = 'afine'
		return
	def phi(num):
		num = num - 1
		return num
	def GFC(num,numA):
		#Get the greatest common factor of the two numbers
		return True
	def getValidMultkeys(num):
		# Should retrun all intigers relativly prime to num
		validKeys = []
		for indexA in range(0,num):
			for indexB in range(0,num):
				if ((indexA *indexB) % num) == 1:
					validKeys.append(indexA)
				else:
					continue
		return validKeys
	def factor(n):
		#should factor n and return a list of factors
		return True
	def isValidMultKey(key,num):
		#check if key is relativly prime to num return true or false
		return True
	def getInvrsMultKey(key):
		# gets the inverse of the mult key , asumes 26 , watches for keys that do not have a reverse
		inverseKey = None
		for index in range(0,26):
			if ((key * index) % 26) == 1:
				inverseKey = index
			else:
				continue
		return inverseKey
	def getInvrsAddKey(key,delta = 'pos'):
		#or delta = 'neg', assumes 26
		if delta == 'neg':
			key = 0-key
			return key
		else:
			key = 25-key ######## THIS MAY CAUSE AN ERROR
	def encrypt(self, msg,multKey,addKey):
		#ENCRYPTION CODE
		encrypted = ''
		for letter in msg:
			letter = letter.lower()
			if self.az(letter):
				encrypted += self.numToChr(((self.chrToNum(letter) + addKey) * multKey) % 26)
			else:
				continue
		return encrypted
	def decrypt(self,msg,multKey,addKey):
		#DECRYPTION CODE
		decrypted = ''
		for letter in msg:
			letter = letter.lower()
			if self.az(letter):
				decrypted += self.numToChr(((self.chrToNum(letter) * self.getInvrsMultKey(multKey)) - getInvrsAddKey(addKey)) % 26)
			else:
				continue
		return decrypted
	def keysValidate(self,keys):
		if type(keys) != str:
			return False
		elif True:
			for letter in keys:
				if opp(self.az(letter)):
					return False
		else:
			return True
	def noKeysDecrypt(self,msg):
		#NO KEY DECRYPTION CODE
		lowestVariance = 1000
		testMultKey = None
		testAddKey = None
		for indexA in getValidMultkeys(26):
			for indexB in range(0,26):
				testDecrypt = self.decrypt(msg, indexA,indexB)
				testFrequency = frequencyAnalysis(testDecrypt)
				testVariance = variance(testFrequency)
				if testVariance < lowestVariance:
					lowestVariance = testVariance
					testMultKey = indexA
					testAddKey = indexB
		return [self.decrypt(msg,testMultKey,testAddKey),testMultKey,testAddKey]
class viginere(afine):
	def __init__(self):
		self.cipherType = 'viginere'
		return
	def encrypt(self,msg, keyWord):
		#ENCRYPTION CODE
		encrypted = ''
		counter = 0
		lenK = len(keyword) -1
		for letter in msg:
			letter = letter.lower()
			if self.az(letter):
				encrypted += self.numToChr((self.chrToNum(letter) + self.chrToNum(keyWord[counter])) % 26)
				counter += 1
				if counter == lenK:
					counter = 0
			else:
				continue
		return encrypted
	def decrypt(self, msg, keyWord):
		#DECRYPTION CODE
		decrypted = ''
		counter = 0
		lenK = len(keyword) -1
		for letter in msg:
			letter = letter.lower()
			if self.az(letter):
				decrypted += self.numToChr((self.chrToNum(letter) - self.chrToNum(keyWord[counter])) % 26)
				counter += 1
				if counter == lenK:
					counter = 0
			else:
				continue
		return decrypted
	def frequencyAnalysis(self):
		#ANALYSIS CODE
		return True
	def msgVariation(self):
		#MSG VARIATION CODE
		return True
	def keysValidate(self):
		#KEY VALIDATION
		return True
	def msgValidate(self):
		#MSG VALIDATION CODE
		return True
	def noKeysDecrypt(self):
		#NO KEY DECRYPTION CODE
		return True
	def searchMsg(self,arg):
		#SEARCH THE self.msg FOR THE arg STRING
		return True
class viginereOldMsg(viginere):
	def __init__(self,keys,msg):
		self.keys = []
		self.cipherType = 'viginereold'
		self.msg = msg
	def encrypt(self):
		#ENCRYPTION CODE
		return True
	def decrypt(self):
		#DECRYPTION CODE
		return True
	def frequencyAnalysis(self):
		#ANALYSIS CODE
		return True
	def msgVariation(self):
		#MSG VARIATION CODE
		return True
	def keysValidate(self):
		#KEY VALIDATION
		return True
	def msgValidate(self):
		#MSG VALIDATION CODE
		return True
	def noKeysDecrypt(self):
		#NO KEY DECRYPTION CODE
		return True
		#INHERITS .searchMsg()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import time

from collections import namedtuple

from exception import *
from limit import getlimit
from connection import __NetLinkConn
from priorities import nice, setnice, ioprio

class Profile(__NetLinkConn):
    '''
    Profile object: builds a <Profile> object from the "pid" provided on
    initialization.
    '''

    STAT_FIELDS = ['pid', 'tcomm', 'state', 'ppid', 'pgrp', 'sid', 'tty_nr',
                   'tty_pgrp', 'flags', 'min_flt', 'cmin_flt', 'maj_flt',
                   'cmaj_flt', 'utime', 'stime', 'cutime', 'cstime',
                   'priority', 'nice', 'num_threads', 'it_real_value',
                   'start_time', 'vsize', 'rss', 'rsslim', 'start_code',
                   'end_code', 'start_stack', 'esp', 'eip', 'pending',
                   'blocked', 'sigign', 'sigcatch', 'wchan', 'NONE', 'NONE',
                   'exit_signal', 'task_cpu', 'rt_priority', 'policy',
                   'blkio_ticks', 'gtime', 'cgtime', 'start_data', 'end_data',
                   'arg_start', 'arg_end', 'env_start', 'env_end', 'exit_code'
                  ]

    PROC_STATES = {'R':'running', 'S':'interruptible_sleep',
                   'D':'uninterruptible_disk_sleep', 'Z':'zombie',
                   'T':'traced', 'W':'paging'
                  }

    def __init__(self, pid):
        super(Profile, self).__init__()
        self.pid = pid
        self.__rst_state = self.rBytes()
        self.__wst_state = self.wBytes()
        self._last_r = self.__rst_state
        self._last_w = self.__wst_state
        self.__st_switches = self.__pid_status_attrs.get(
                                 'voluntary_ctxt_switches')
        if not self.__st_switches:
            self.__st_switches = 0

    @property
    def is_alive(self):
        '''
        Class property: returns <type 'bool'> response of processes state.
        '''
        all_processes = filter(lambda i: i.isdigit(), os.listdir('/proc'))
        if self.pid in map(int, all_processes):
            return True
        return False

    @property
    def ppid(self):
        '''
        Class property: returns <type 'int'> of the profiled pid's parent.
        '''
        try:
            with open('/proc/%s/stat' % self.pid) as f:
                pid_stats = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_stat_arr = pid_stats.split()
        ppid = int(pid_stat_arr[3])
        return ppid

    @property
    def name(self):
        '''
        Class property: returns <type 'str'> of profiled pid's name.
        '''
        try:
            with open('/proc/%s/comm' % self.pid) as f:
                pid_name = f.read()
                name = pid_name.strip('\n')
        except IOError:
            raise BadProcess(self.pid)
        return name

    @property
    def fds(self):
        '''
        Class property: returns <type 'int'> of open file-descriptors of
        profiled pid.
        '''
        try:
            fds = os.listdir('/proc/%s/fd' % self.pid)
        except OSError:
            raise BadProcess(self.pid)
        return len(fds)

    @property
    def gid(self):
        '''
        Class property: returns <type 'str'> or <type 'NoneType'> of profiled
        pids group id.
        '''
        gid = self.__pid_status_attrs.get('gid')
        if gid:
            return gid[0]
        return

    @property
    def threads(self):
        '''
        Class property: returns <type 'int'> or <type 'NoneType'> of profiled
        pids thread count.
        '''
        threads = self.__pid_status_attrs.get('threads')
        if threads:
            return int(threads[0])
        return

    @property
    def state(self):
        '''
        Class property: returns <type 'str'> or <type 'NoneType'> of profiled
        pids current state.
        '''
        state = self.__pid_stat_attrs.get('state')
        if state:
            return self.PROC_STATES[state]
        return 'Unknown'

    def nice(self):
        '''
        Class method: returns <type 'int'> of profiled pids niceness.
        '''
        return nice(self.pid)

    def setnice(self, level):
        '''
        Class method: sets the niceness of profiled pid, returns
        <type 'NoneType'>.

        @type 'level': <type 'int'>
        @param 'level': the new nice to set profiled pid to.
        '''
        setnice(self.pid, level)
        return

    def ioprio(self):
        '''
        Class method: returns <type 'tuple'> of profiled pids IO_Priority
        class and ioprio_nice.
        '''
        return ioprio(self.pid)

    def switches(self):
        '''
        Class method: returns <type 'int'> for the number of context switches
        of profiled pid.
        '''
        switches = self.__pid_status_attrs.get('voluntary_ctxt_switches')
        if switches:
            return int(switches[0])
        return

    def has_switched(self):
        '''
        Class method: returns <type 'bool'> to signal if there have voluntary
        context switches since initialization or last call of this method.
        '''
        current_switches = self.__pid_status_attrs.get(
                                'voluntary_ctxt_switches')
        if current_switches > self.__st_switches:
            self.__st_switches = current_switches
            return True
        return False

    def rBytes(self):
        '''
        Class method: returns <type 'int'> of pids total read bytes since
        startup.
        '''
        read = self._ntlnk_exchange()
        return read[0]

    def wBytes(self):
        '''
        Class method: returns <type 'int'> of the pids total write bytes since
        startup.
        '''
        write = self._ntlnk_exchange()
        return write[1]

    def has_read(self):
        '''
        Class method: returns <type 'bool'> for pid's reads since
        initialization or the last call of this method.
        '''
        current_r = self.rBytes()
        if current_r > self._last_r:
            self._last_r = current_r
            return True
        return False

    def has_write(self):
        '''
        Class method: returns <type 'bool'> for the pids writes since
        initialization or last call of this method.
        '''
        current_w = self.wBytes()
        if current_w > self._last_w:
            self._last_w = current_w
            return True
        return False

    def allread(self):
        '''
        Class method: returns <type 'int'> for pids total reads since object
         creation.
        '''
        current_r = self.rBytes()
        return current_r - self.__rst_state

    def allwrite(self):
        '''
        Class method: returns <type 'int'> for the pids total writes since
        object creation.
        '''
        current_w = self.wBytes()
        return current_w - self.__wst_state

    def smap(self):
        '''
        Class method: returns <type 'dict'> of namedtuples for the profiled
        pids mapping memory consumption.
        '''
        smap_tuple = namedtuple('smap', ['Size', 'Rss', 'Pss', 'Shared_Clean',
                                         'Shared_Dirty', 'Private_Clean',
                                         'Private_Dirty', 'Referenced',
                                         'Anonymous', 'AnonHugePages', 'Swap',
                                         'KernelPageSize', 'MMUPageSize',
                                         'Locked', 'VmFlags']
                               )
        try:
            with open('/proc/%s/smaps' % self.pid)  as f:
                proc_smap = f.readlines()
        except IOError:
            raise BadProcess(self.pid)
        proc_smaps = [proc_smap[i:i + 16] for i in
                      xrange(0, len(proc_smap), 16)]
        smappings = dict()
        rm_newline = lambda ln: ln.strip('\n')
        for smap in proc_smaps:
            path_name = smap[0].split()[-1]
            _smap = [''.join(i.split(' ')[1:]) for i in smap[1:]]
            _smap = map(rm_newline, _smap)
            smappings[path_name] = smap_tuple(*_smap)
        return smappings

    def pmap(self):
        '''
        Class method: returns <type 'tuple'> of the profiled pids memory map.
        '''
        smap = self.smap()
        pmap = smap.keys()
        return tuple(pmap)

    def wchan(self):
        '''
        Class method: returns <type 'str'> of a kernel symbol where the profiled
        pid currently sleeping(if at all).
        '''
        try:
            with open('/proc/%s/wchan' % self.pid) as f:
                wchan = f.read()
            return wchan
        except IOError:
            raise BadProcess(self.pid)

    def fdstat(self):
        '''
        Class method: returns <type 'dict'> for all file-descriptors as the keys
        and tuple for the files stats.
        '''
        path = '/proc/%s/fd/' % self.pid
        try:
            fds = [path + i for i in os.listdir(path)]
            fdstat = dict(zip(map(os.readlink, fds), map(os.stat, fds)))
        except IOError:
            raise BadProcess(self.pid)
        return fdstat

    def fdperms(self):
        '''
        Class method: returns <type 'dict'> for all file-descriptors real path
        as keys and their file permissions as octal values.
        '''
        fdstats = self.fdstat()
        permission_mask = lambda mask: oct(mask & 0777)
        modes = [fdstats[i].st_mode for i in fdstats]
        permissions = dict(zip(fdstats.keys(), map(permission_mask, modes)))
        return permissions

    def getrlimit(self, resource):
    '''
    Class method: returns <type 'int'> for the current soft-limit of
    process.
    '''
    return getlimit(self.pid, resource)

    @property
    def __pid_status_attrs(self):
        '''
        Private class property: (not meant to be called directly) populates a
        <type 'dict'> with pid attributes read from "status".
        '''
        try:
            with open('/proc/%s/status' % self.pid) as f:
                raw_attrs = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_attrs = {i[0].strip(':').lower():i[1:] for i in
                     map(str.split, raw_attrs.split('\n')) if i}
        return pid_attrs

    @property
    def __pid_stat_attrs(self):
        '''
        Private class property: (not meant to be called directly) populates a
        <type 'dict'> with pid attributes read from "stat".
        '''
        try:
            with open('/proc/%s/stat' % self.pid) as f:
                stats = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_stats = dict(zip(self.STAT_FIELDS, stats.split()))
        return pid_stats

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "<class '%s (pid: %s | name: %s)'>" % (self.__class__.__name__,
                                                      self.pid, self.name)
<commit_msg>fixed indentation error<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import time

from collections import namedtuple

from exception import *
from limit import getlimit
from connection import __NetLinkConn
from priorities import nice, setnice, ioprio

class Profile(__NetLinkConn):
    '''
    Profile object: builds a <Profile> object from the "pid" provided on
    initialization.
    '''

    STAT_FIELDS = ['pid', 'tcomm', 'state', 'ppid', 'pgrp', 'sid', 'tty_nr',
                   'tty_pgrp', 'flags', 'min_flt', 'cmin_flt', 'maj_flt',
                   'cmaj_flt', 'utime', 'stime', 'cutime', 'cstime',
                   'priority', 'nice', 'num_threads', 'it_real_value',
                   'start_time', 'vsize', 'rss', 'rsslim', 'start_code',
                   'end_code', 'start_stack', 'esp', 'eip', 'pending',
                   'blocked', 'sigign', 'sigcatch', 'wchan', 'NONE', 'NONE',
                   'exit_signal', 'task_cpu', 'rt_priority', 'policy',
                   'blkio_ticks', 'gtime', 'cgtime', 'start_data', 'end_data',
                   'arg_start', 'arg_end', 'env_start', 'env_end', 'exit_code'
                  ]

    PROC_STATES = {'R':'running', 'S':'interruptible_sleep',
                   'D':'uninterruptible_disk_sleep', 'Z':'zombie',
                   'T':'traced', 'W':'paging'
                  }

    def __init__(self, pid):
        super(Profile, self).__init__()
        self.pid = pid
        self.__rst_state = self.rBytes()
        self.__wst_state = self.wBytes()
        self._last_r = self.__rst_state
        self._last_w = self.__wst_state
        self.__st_switches = self.__pid_status_attrs.get(
                                 'voluntary_ctxt_switches')
        if not self.__st_switches:
            self.__st_switches = 0

    @property
    def is_alive(self):
        '''
        Class property: returns <type 'bool'> response of processes state.
        '''
        all_processes = filter(lambda i: i.isdigit(), os.listdir('/proc'))
        if self.pid in map(int, all_processes):
            return True
        return False

    @property
    def ppid(self):
        '''
        Class property: returns <type 'int'> of the profiled pid's parent.
        '''
        try:
            with open('/proc/%s/stat' % self.pid) as f:
                pid_stats = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_stat_arr = pid_stats.split()
        ppid = int(pid_stat_arr[3])
        return ppid

    @property
    def name(self):
        '''
        Class property: returns <type 'str'> of profiled pid's name.
        '''
        try:
            with open('/proc/%s/comm' % self.pid) as f:
                pid_name = f.read()
                name = pid_name.strip('\n')
        except IOError:
            raise BadProcess(self.pid)
        return name

    @property
    def fds(self):
        '''
        Class property: returns <type 'int'> of open file-descriptors of
        profiled pid.
        '''
        try:
            fds = os.listdir('/proc/%s/fd' % self.pid)
        except OSError:
            raise BadProcess(self.pid)
        return len(fds)

    @property
    def gid(self):
        '''
        Class property: returns <type 'str'> or <type 'NoneType'> of profiled
        pids group id.
        '''
        gid = self.__pid_status_attrs.get('gid')
        if gid:
            return gid[0]
        return

    @property
    def threads(self):
        '''
        Class property: returns <type 'int'> or <type 'NoneType'> of profiled
        pids thread count.
        '''
        threads = self.__pid_status_attrs.get('threads')
        if threads:
            return int(threads[0])
        return

    @property
    def state(self):
        '''
        Class property: returns <type 'str'> or <type 'NoneType'> of profiled
        pids current state.
        '''
        state = self.__pid_stat_attrs.get('state')
        if state:
            return self.PROC_STATES[state]
        return 'Unknown'

    def nice(self):
        '''
        Class method: returns <type 'int'> of profiled pids niceness.
        '''
        return nice(self.pid)

    def setnice(self, level):
        '''
        Class method: sets the niceness of profiled pid, returns
        <type 'NoneType'>.

        @type 'level': <type 'int'>
        @param 'level': the new nice to set profiled pid to.
        '''
        setnice(self.pid, level)
        return

    def ioprio(self):
        '''
        Class method: returns <type 'tuple'> of profiled pids IO_Priority
        class and ioprio_nice.
        '''
        return ioprio(self.pid)

    def switches(self):
        '''
        Class method: returns <type 'int'> for the number of context switches
        of profiled pid.
        '''
        switches = self.__pid_status_attrs.get('voluntary_ctxt_switches')
        if switches:
            return int(switches[0])
        return

    def has_switched(self):
        '''
        Class method: returns <type 'bool'> to signal if there have voluntary
        context switches since initialization or last call of this method.
        '''
        current_switches = self.__pid_status_attrs.get(
                                'voluntary_ctxt_switches')
        if current_switches > self.__st_switches:
            self.__st_switches = current_switches
            return True
        return False

    def rBytes(self):
        '''
        Class method: returns <type 'int'> of pids total read bytes since
        startup.
        '''
        read = self._ntlnk_exchange()
        return read[0]

    def wBytes(self):
        '''
        Class method: returns <type 'int'> of the pids total write bytes since
        startup.
        '''
        write = self._ntlnk_exchange()
        return write[1]

    def has_read(self):
        '''
        Class method: returns <type 'bool'> for pid's reads since
        initialization or the last call of this method.
        '''
        current_r = self.rBytes()
        if current_r > self._last_r:
            self._last_r = current_r
            return True
        return False

    def has_write(self):
        '''
        Class method: returns <type 'bool'> for the pids writes since
        initialization or last call of this method.
        '''
        current_w = self.wBytes()
        if current_w > self._last_w:
            self._last_w = current_w
            return True
        return False

    def allread(self):
        '''
        Class method: returns <type 'int'> for pids total reads since object
         creation.
        '''
        current_r = self.rBytes()
        return current_r - self.__rst_state

    def allwrite(self):
        '''
        Class method: returns <type 'int'> for the pids total writes since
        object creation.
        '''
        current_w = self.wBytes()
        return current_w - self.__wst_state

    def smap(self):
        '''
        Class method: returns <type 'dict'> of namedtuples for the profiled
        pids mapping memory consumption.
        '''
        smap_tuple = namedtuple('smap', ['Size', 'Rss', 'Pss', 'Shared_Clean',
                                         'Shared_Dirty', 'Private_Clean',
                                         'Private_Dirty', 'Referenced',
                                         'Anonymous', 'AnonHugePages', 'Swap',
                                         'KernelPageSize', 'MMUPageSize',
                                         'Locked', 'VmFlags']
                               )
        try:
            with open('/proc/%s/smaps' % self.pid)  as f:
                proc_smap = f.readlines()
        except IOError:
            raise BadProcess(self.pid)
        proc_smaps = [proc_smap[i:i + 16] for i in
                      xrange(0, len(proc_smap), 16)]
        smappings = dict()
        rm_newline = lambda ln: ln.strip('\n')
        for smap in proc_smaps:
            path_name = smap[0].split()[-1]
            _smap = [''.join(i.split(' ')[1:]) for i in smap[1:]]
            _smap = map(rm_newline, _smap)
            smappings[path_name] = smap_tuple(*_smap)
        return smappings

    def pmap(self):
        '''
        Class method: returns <type 'tuple'> of the profiled pids memory map.
        '''
        smap = self.smap()
        pmap = smap.keys()
        return tuple(pmap)

    def wchan(self):
        '''
        Class method: returns <type 'str'> of a kernel symbol where the profiled
        pid currently sleeping(if at all).
        '''
        try:
            with open('/proc/%s/wchan' % self.pid) as f:
                wchan = f.read()
            return wchan
        except IOError:
            raise BadProcess(self.pid)

    def fdstat(self):
        '''
        Class method: returns <type 'dict'> for all file-descriptors as the keys
        and tuple for the files stats.
        '''
        path = '/proc/%s/fd/' % self.pid
        try:
            fds = [path + i for i in os.listdir(path)]
            fdstat = dict(zip(map(os.readlink, fds), map(os.stat, fds)))
        except IOError:
            raise BadProcess(self.pid)
        return fdstat

    def fdperms(self):
        '''
        Class method: returns <type 'dict'> for all file-descriptors real path
        as keys and their file permissions as octal values.
        '''
        fdstats = self.fdstat()
        permission_mask = lambda mask: oct(mask & 0777)
        modes = [fdstats[i].st_mode for i in fdstats]
        permissions = dict(zip(fdstats.keys(), map(permission_mask, modes)))
        return permissions

    def getrlimit(self, resource):
        '''
        Class method: returns <type 'int'> for the current soft-limit of
        process.
        '''
        return getlimit(self.pid, resource)

    @property
    def __pid_status_attrs(self):
        '''
        Private class property: (not meant to be called directly) populates a
        <type 'dict'> with pid attributes read from "status".
        '''
        try:
            with open('/proc/%s/status' % self.pid) as f:
                raw_attrs = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_attrs = {i[0].strip(':').lower():i[1:] for i in
                     map(str.split, raw_attrs.split('\n')) if i}
        return pid_attrs

    @property
    def __pid_stat_attrs(self):
        '''
        Private class property: (not meant to be called directly) populates a
        <type 'dict'> with pid attributes read from "stat".
        '''
        try:
            with open('/proc/%s/stat' % self.pid) as f:
                stats = f.read()
        except IOError:
            raise BadProcess(self.pid)
        pid_stats = dict(zip(self.STAT_FIELDS, stats.split()))
        return pid_stats

    def __str__(self):
        return self.__repr__()

    def __repr__(self):
        return "<class '%s (pid: %s | name: %s)'>" % (self.__class__.__name__,
                                                      self.pid, self.name)
<|endoftext|>