<commit_before>"""
In our python interface to the backbone system, we separate the local collection
which stores models, from the http client which interacts with a remote store
In applications, we would use a class that combines both
"""

import requests
import shelve
import uuid
import contextlib
import logging
from collections import defaultdict
from six.moves import cPickle as pickle
from .. import  protocol
from .models import docs
import numpy as np
logger = logging.getLogger(__name__)

from ..objects import PlotObject, Plot
from ..session import PersistentBackboneSession, BaseJSONSession
from ..utils import encode_utf8, decode_utf8
from . import server_backends
from .app import bokeh_app

class StoreAdapter(object):
    """API modeled after Redis that other stores have to adapt to. """

    def mget(self, doc_keys):
        raise NotImplementedError("abstract method")

    def mset(self, data):
        raise NotImplementedError("abstract method")

    def sadd(self, doc_key, *keys):
        raise NotImplementedError("abstract method")

    def srem(self, doc_key, member_key):
        raise NotImplementedError("abstract method")

    def smembers(self, doc_key):
        raise NotImplementedError("abstract method")

    def set(self, key, data):
        raise NotImplementedError("abstract method")

    def delete(self, key):
        raise NotImplementedError("abstract method")

class PersistentSession(PersistentBackboneSession, BaseJSONSession, StoreAdapter):
    """Base class for `RedisSession`, `InMemorySession`, etc. """

    def __init__(self, docid, doc=None):
        super(PersistentSession, self).__init__(plot=None)
        self.doc = doc
        self.docid = docid
        self.raw_js_objs = []

    @property
    def plotcontext(self):
        pc = super(PersistentSession, self).plotcontext
        if pc:
            return pc
        if not self.doc:
            self.doc = docs.Doc.load(bokeh_app.servermodel_storage, self.docid)
            self.plotcontext = self._models[self.doc.plot_context_ref['id']]
            return self.plotcontext

    @plotcontext.setter
    def plotcontext(self, val):
        self._plotcontext = val

    def dockey(self, docid):
        docid = encode_utf8('doc:' + docid)
        return docid

    def modelkey(self, typename, docid, modelid):
        docid = encode_utf8(docid)
        modelid = encode_utf8(modelid)
        return 'bbmodel:%s:%s:%s' % (typename, docid, modelid)

    def callbackskey(self, typename, docid, modelid):
        return 'bbcallback:%s:%s:%s' % (typename, docid, modelid)

    def parse_modelkey(self, key):
        _, typename, docid, modelid = decode_utf8(key).split(":")
        return typename, docid, modelid

    def set_doc(self, doc):
        self.doc = doc
        self.docid = doc.docid

    def prune(self, delete=False):
        all_models = docs.prune_and_get_valid_models(self, delete=delete)
        to_keep = set([x._id for x in all_models])
        to_delete = set(self._models.keys()) - to_keep
        for k in to_delete:
            del self._models[k]

    def load_all(self, asdict=False):
        doc_keys = self.smembers(self.dockey(self.docid))
        attrs = self.mget(doc_keys)
        if asdict:
            return attrs
        data = []
        for k, attr in zip(doc_keys, attrs):
            typename, _, modelid = self.parse_modelkey(k)
            attr = protocol.deserialize_json(decode_utf8(attr))
            data.append({'type': typename, 'attributes': attr})
        models = self.load_broadcast_attrs(data, events=None)
        for m in models:
            m._dirty = False
        return models

    def store_broadcast_attrs(self, attrs):
        if not attrs:
            return
        keys = [self.modelkey(attr['type'], self.docid, attr['attributes']['id']) for attr in attrs]
        for attr in attrs:
            attr['attributes']['doc'] = self.docid
        attrs = [self.serialize(attr['attributes']) for attr in attrs]
        dkey = self.dockey(self.docid)
        data = dict(zip(keys, attrs))
        self.mset(data)
        self.sadd(dkey, *keys)

    def del_objs(self, to_del):
        for m in to_del:
            mkey = self.modelkey(m.__view_model__, self.docid, m._id)
            self.srem(self.dockey(self.docid), mkey)
            self.delete(mkey)

    def load_all_callbacks(self, get_json=False):
        """get_json = return json of callbacks, rather than
        loading them into models
        """
        doc_keys = self.smembers(self.dockey(self.docid))
        callback_keys = [x.replace("bbmodel", "bbcallback") for x in doc_keys]
        callbacks = self.mget(callback_keys)
        callbacks = [x for x in callbacks if x]
        callbacks = [protocol.deserialize_json(x) for x in callbacks]
        if get_json:
            return callbacks
        self.load_callbacks_json(callbacks)

    def store_callbacks(self, to_store):
        for callbacks in to_store:
            typename = callbacks['type']
            _id = callbacks['id']
            key = self.callbackskey(typename, self.docid, _id)
            data = self.serialize(callbacks)
            self.set(key, data)

    # TODO: move this to appropriate superclass
    def raw_js_snippets(self, obj):
        self.raw_js_objs.append(obj)

class RedisSession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses redis directly.
    """

    def __init__(self, redis, docid, doc=None):
        self.redis = redis
        super(RedisSession, self).__init__(docid, doc)

    def mget(self, doc_keys):
        return self.redis.mget(doc_keys)

    def mset(self, data):
        self.redis.mset(data)

    def sadd(self, doc_key, *keys):
        self.redis.sadd(doc_key, *keys)

    def srem(self, doc_key, mkey):
        self.redis.srem(doc_key, mkey)

    def smembers(self, doc_key):
        return self.redis.smembers(doc_key)

    def set(self, key, data):
        self.redis.set(key, data)

    def delete(self, mkey):
        self.redis.delete(mkey)

_inmem_data = {}
_inmem_sets = defaultdict(set)

class InMemorySession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses in memory data store directly.
    """

    def mget(self, doc_keys):
        return [_inmem_data.get(key, None) for key in doc_keys]

    def mset(self, data):
        _inmem_data.update(data)

    def sadd(self, doc_key, *keys):
        _inmem_sets[doc_key].update(keys)

    def srem(self, doc_key, member_key):
        inmem_set = _inmem_sets[doc_key]
        try: inmem_set.remove(member_key)
        except KeyError: pass

    def smembers(self, doc_key):
        return list(_inmem_sets[doc_key])

    def set(self, key, data):
        _inmem_data[key] = data

    def delete(self, key):
        del _inmem_data[key]

class ShelveSession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses shelve data store directly.
    """

    @contextlib.contextmanager
    def shelve_data(self):
        data = shelve.open('bokeh.data')
        try:
            yield data
        finally:
            data.close()

    @contextlib.contextmanager
    def shelve_sets(self):
        sets = shelve.open('bokeh.sets')
        try:
            yield sets
        finally:
            sets.close()

    def mget(self, doc_keys):
        with self.shelve_data() as _shelve_data:
            return [_shelve_data.get(key, None) for key in doc_keys]

    def mset(self, data):
        with self.shelve_data() as _shelve_data:
            for k, v in data.items():
                _shelve_data[k] = v

    def sadd(self, doc_key, *keys):
        with self.shelve_sets() as _shelve_sets:
            shelve_set = _shelve_sets.get(doc_key, set())
            shelve_set.update(keys)
            _shelve_sets[doc_key] = shelve_set

    def srem(self, doc_key, member_key):
        with self.shelve_sets() as _shelve_sets:
            shelve_set = _shelve_sets[doc_key]
            try: shelve_set.remove(member_key)
            except KeyError: pass
            _shelve_sets[doc_key] = shelve_set

    def smembers(self, doc_key):
        with self.shelve_sets() as _shelve_sets:
            return list(_shelve_sets[doc_key])

    def set(self, key, data):
        with self.shelve_data() as _shelve_data:
            _shelve_data[key] = data

    def delete(self, key):
        with self.shelve_data() as _shelve_data:
            del _shelve_data[key]
<commit_msg>Make _inmem_{data,sets} class members of InMemorySession<commit_after>"""
In our python interface to the backbone system, we separate the local collection
which stores models, from the http client which interacts with a remote store
In applications, we would use a class that combines both
"""

import requests
import shelve
import uuid
import contextlib
import logging
from collections import defaultdict
from six.moves import cPickle as pickle
from .. import  protocol
from .models import docs
import numpy as np
logger = logging.getLogger(__name__)

from ..objects import PlotObject, Plot
from ..session import PersistentBackboneSession, BaseJSONSession
from ..utils import encode_utf8, decode_utf8
from . import server_backends
from .app import bokeh_app

class StoreAdapter(object):
    """API modeled after Redis that other stores have to adapt to. """

    def mget(self, doc_keys):
        raise NotImplementedError("abstract method")

    def mset(self, data):
        raise NotImplementedError("abstract method")

    def sadd(self, doc_key, *keys):
        raise NotImplementedError("abstract method")

    def srem(self, doc_key, member_key):
        raise NotImplementedError("abstract method")

    def smembers(self, doc_key):
        raise NotImplementedError("abstract method")

    def set(self, key, data):
        raise NotImplementedError("abstract method")

    def delete(self, key):
        raise NotImplementedError("abstract method")

class PersistentSession(PersistentBackboneSession, BaseJSONSession, StoreAdapter):
    """Base class for `RedisSession`, `InMemorySession`, etc. """

    def __init__(self, docid, doc=None):
        super(PersistentSession, self).__init__(plot=None)
        self.doc = doc
        self.docid = docid
        self.raw_js_objs = []

    @property
    def plotcontext(self):
        pc = super(PersistentSession, self).plotcontext
        if pc:
            return pc
        if not self.doc:
            self.doc = docs.Doc.load(bokeh_app.servermodel_storage, self.docid)
            self.plotcontext = self._models[self.doc.plot_context_ref['id']]
            return self.plotcontext

    @plotcontext.setter
    def plotcontext(self, val):
        self._plotcontext = val

    def dockey(self, docid):
        docid = encode_utf8('doc:' + docid)
        return docid

    def modelkey(self, typename, docid, modelid):
        docid = encode_utf8(docid)
        modelid = encode_utf8(modelid)
        return 'bbmodel:%s:%s:%s' % (typename, docid, modelid)

    def callbackskey(self, typename, docid, modelid):
        return 'bbcallback:%s:%s:%s' % (typename, docid, modelid)

    def parse_modelkey(self, key):
        _, typename, docid, modelid = decode_utf8(key).split(":")
        return typename, docid, modelid

    def set_doc(self, doc):
        self.doc = doc
        self.docid = doc.docid

    def prune(self, delete=False):
        all_models = docs.prune_and_get_valid_models(self, delete=delete)
        to_keep = set([x._id for x in all_models])
        to_delete = set(self._models.keys()) - to_keep
        for k in to_delete:
            del self._models[k]

    def load_all(self, asdict=False):
        doc_keys = self.smembers(self.dockey(self.docid))
        attrs = self.mget(doc_keys)
        if asdict:
            return attrs
        data = []
        for k, attr in zip(doc_keys, attrs):
            typename, _, modelid = self.parse_modelkey(k)
            attr = protocol.deserialize_json(decode_utf8(attr))
            data.append({'type': typename, 'attributes': attr})
        models = self.load_broadcast_attrs(data, events=None)
        for m in models:
            m._dirty = False
        return models

    def store_broadcast_attrs(self, attrs):
        if not attrs:
            return
        keys = [self.modelkey(attr['type'], self.docid, attr['attributes']['id']) for attr in attrs]
        for attr in attrs:
            attr['attributes']['doc'] = self.docid
        attrs = [self.serialize(attr['attributes']) for attr in attrs]
        dkey = self.dockey(self.docid)
        data = dict(zip(keys, attrs))
        self.mset(data)
        self.sadd(dkey, *keys)

    def del_objs(self, to_del):
        for m in to_del:
            mkey = self.modelkey(m.__view_model__, self.docid, m._id)
            self.srem(self.dockey(self.docid), mkey)
            self.delete(mkey)

    def load_all_callbacks(self, get_json=False):
        """get_json = return json of callbacks, rather than
        loading them into models
        """
        doc_keys = self.smembers(self.dockey(self.docid))
        callback_keys = [x.replace("bbmodel", "bbcallback") for x in doc_keys]
        callbacks = self.mget(callback_keys)
        callbacks = [x for x in callbacks if x]
        callbacks = [protocol.deserialize_json(x) for x in callbacks]
        if get_json:
            return callbacks
        self.load_callbacks_json(callbacks)

    def store_callbacks(self, to_store):
        for callbacks in to_store:
            typename = callbacks['type']
            _id = callbacks['id']
            key = self.callbackskey(typename, self.docid, _id)
            data = self.serialize(callbacks)
            self.set(key, data)

    # TODO: move this to appropriate superclass
    def raw_js_snippets(self, obj):
        self.raw_js_objs.append(obj)

class RedisSession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses redis directly.
    """

    def __init__(self, redis, docid, doc=None):
        self.redis = redis
        super(RedisSession, self).__init__(docid, doc)

    def mget(self, doc_keys):
        return self.redis.mget(doc_keys)

    def mset(self, data):
        self.redis.mset(data)

    def sadd(self, doc_key, *keys):
        self.redis.sadd(doc_key, *keys)

    def srem(self, doc_key, mkey):
        self.redis.srem(doc_key, mkey)

    def smembers(self, doc_key):
        return self.redis.smembers(doc_key)

    def set(self, key, data):
        self.redis.set(key, data)

    def delete(self, mkey):
        self.redis.delete(mkey)

class InMemorySession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses in memory data store directly.
    """

    _inmem_data = {}
    _inmem_sets = defaultdict(set)

    def mget(self, doc_keys):
        return [self._inmem_data.get(key, None) for key in doc_keys]

    def mset(self, data):
        self._inmem_data.update(data)

    def sadd(self, doc_key, *keys):
        self._inmem_sets[doc_key].update(keys)

    def srem(self, doc_key, member_key):
        inmem_set = self._inmem_sets[doc_key]
        try: inmem_set.remove(member_key)
        except KeyError: pass

    def smembers(self, doc_key):
        return list(self._inmem_sets[doc_key])

    def set(self, key, data):
        self._inmem_data[key] = data

    def delete(self, key):
        del self._inmem_data[key]

class ShelveSession(PersistentSession):
    """session used by the webserver to work with
    a user's documents.  uses shelve data store directly.
    """

    @contextlib.contextmanager
    def shelve_data(self):
        data = shelve.open('bokeh.data')
        try:
            yield data
        finally:
            data.close()

    @contextlib.contextmanager
    def shelve_sets(self):
        sets = shelve.open('bokeh.sets')
        try:
            yield sets
        finally:
            sets.close()

    def mget(self, doc_keys):
        with self.shelve_data() as _shelve_data:
            return [_shelve_data.get(key, None) for key in doc_keys]

    def mset(self, data):
        with self.shelve_data() as _shelve_data:
            for k, v in data.items():
                _shelve_data[k] = v

    def sadd(self, doc_key, *keys):
        with self.shelve_sets() as _shelve_sets:
            shelve_set = _shelve_sets.get(doc_key, set())
            shelve_set.update(keys)
            _shelve_sets[doc_key] = shelve_set

    def srem(self, doc_key, member_key):
        with self.shelve_sets() as _shelve_sets:
            shelve_set = _shelve_sets[doc_key]
            try: shelve_set.remove(member_key)
            except KeyError: pass
            _shelve_sets[doc_key] = shelve_set

    def smembers(self, doc_key):
        with self.shelve_sets() as _shelve_sets:
            return list(_shelve_sets[doc_key])

    def set(self, key, data):
        with self.shelve_data() as _shelve_data:
            _shelve_data[key] = data

    def delete(self, key):
        with self.shelve_data() as _shelve_data:
            del _shelve_data[key]
<|endoftext|>


<commit_before>import os
import sys
import shutil

from PyQt5 import uic
from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QMessageBox

from timetable_plus import models
from timetable_plus import solver
from timetable_plus import views

MainWindowUI = uic.loadUiType(os.path.join('timetable_plus', 'ui', 'main.ui'))[0]

class MainWindow(QMainWindow, MainWindowUI):
    def __init__(self, parent=None):
        # noinspection PyArgumentList
        super().__init__(parent)

        self.setupUi(self)
        self.setWindowTitle('Timetable+')

        self.write_button.setEnabled(False)

        self.open_button.clicked.connect(self.get_students)
        self.write_button.clicked.connect(self.generate_timetable)
        self.actionCreateTemplate.triggered.connect(self.create_template)

        self.students = []

    def get_students(self):
        """
            Get the list of students from a user-selected file and store it in
            MainWindow.students
        """
        # noinspection PyCallByClass,PyArgumentList
        self.input_file_name, _ = QFileDialog.getOpenFileName(self, 'Choose File to Open',
                os.path.expanduser('~'), 'Microsoft Excel Spreadsheet Files (*.xlsx *.xls)')
        if self.input_file_name != '':
            try:
                self.datastore = models.Datastore(self.input_file_name)
                self.students = list(self.datastore
                        .get_students(self.actionIncludeTeachers.isChecked()).values())
            except models.LoadingError as e:
                # noinspection PyArgumentList
                QMessageBox.critical(self, 'Invalid input file',
                    'The file you selected had an error in cell {cell}: {msg}'
                    .format(cell=e.cell.coordinate, msg=e))
            else:
                self.statusbar.showMessage('Opened {}'.format(self.input_file_name))
                self.write_button.setEnabled(True)

    def generate_timetable(self):
        # noinspection PyCallByClass,PyArgumentList
        output_filename, _ = QFileDialog.getSaveFileName(self, 'Choose File to Save to',
                os.path.join(os.path.expanduser('~'), 'timetable.xlsx'),
                'Microsoft Excel Spreadsheet Files (*.xlsx *.xls)')

        if output_filename != '':
            self.statusbar.showMessage('Processing...')

            tt = solver.possible_timetables(self.students, 20)
            views.timetable_to_workbook(next(tt)).save(output_filename)

            self.statusbar.showMessage('Saved to {}'.format(output_filename))

    def create_template(self):
        filename, _ = QFileDialog.getSaveFileName(self, 'Choose File to Save template to', os.path.expanduser('~'))

        shutil.copyfile(os.path.join('resources', 'template.xlsx'), filename)

def main():
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<commit_msg>Add default filename for template file<commit_after>import os
import sys
import shutil

from PyQt5 import uic
from PyQt5.QtWidgets import QApplication, QMainWindow, QFileDialog, QMessageBox

from timetable_plus import models
from timetable_plus import solver
from timetable_plus import views

MainWindowUI = uic.loadUiType(os.path.join('timetable_plus', 'ui', 'main.ui'))[0]

class MainWindow(QMainWindow, MainWindowUI):
    def __init__(self, parent=None):
        # noinspection PyArgumentList
        super().__init__(parent)

        self.setupUi(self)
        self.setWindowTitle('Timetable+')

        self.write_button.setEnabled(False)

        self.open_button.clicked.connect(self.get_students)
        self.write_button.clicked.connect(self.generate_timetable)
        self.actionCreateTemplate.triggered.connect(self.create_template)

        self.students = []

    def get_students(self):
        """
            Get the list of students from a user-selected file and store it in
            MainWindow.students
        """
        # noinspection PyCallByClass,PyArgumentList
        self.input_file_name, _ = QFileDialog.getOpenFileName(self, 'Choose File to Open',
                os.path.expanduser('~'), 'Microsoft Excel Spreadsheet Files (*.xlsx *.xls)')
        if self.input_file_name != '':
            try:
                self.datastore = models.Datastore(self.input_file_name)
                self.students = list(self.datastore
                        .get_students(self.actionIncludeTeachers.isChecked()).values())
            except models.LoadingError as e:
                # noinspection PyArgumentList
                QMessageBox.critical(self, 'Invalid input file',
                    'The file you selected had an error in cell {cell}: {msg}'
                    .format(cell=e.cell.coordinate, msg=e))
            else:
                self.statusbar.showMessage('Opened {}'.format(self.input_file_name))
                self.write_button.setEnabled(True)

    def generate_timetable(self):
        # noinspection PyCallByClass,PyArgumentList
        output_filename, _ = QFileDialog.getSaveFileName(self, 'Choose File to Save to',
                os.path.join(os.path.expanduser('~'), 'timetable.xlsx'),
                'Microsoft Excel Spreadsheet Files (*.xlsx *.xls)')

        if output_filename != '':
            self.statusbar.showMessage('Processing...')

            tt = solver.possible_timetables(self.students, 20)
            views.timetable_to_workbook(next(tt)).save(output_filename)

            self.statusbar.showMessage('Saved to {}'.format(output_filename))

    def create_template(self):
        filename, _ = QFileDialog.getSaveFileName(self, 'Choose File to Save template to',
                os.path.join(os.path.expanduser('~'), 'template.xlsx'),
                'Microsoft Excel Spreadsheet Files (*.xlsx *.xls);; All Files')

        if filename != '':
            shutil.copyfile(os.path.join('resources', 'template.xlsx'), filename)

def main():
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import sys
import argparse
from collections import defaultdict
from decimal import Decimal
from toollib.files import findNumber,ParameterParser
from toollib.group import Group,run_grouping
from math import sqrt

class DistGroup(Group):
    def __init__(self, tup):
        super(DistGroup, self).__init__(tup)
        self.vals = []

    def add(self, chunks):
        val = float(findNumber(chunks[args.column]))
        self.vals.append(val)

    def done(self):
        from scipy.stats import chisquare
        args.outfile.write(self.tup + list(chisquare(self.vals)))

if __name__ == "__main__":
    pp = ParameterParser('Entropy of a column', columns = 1, append = False, labels = [None])
    pp.parser.add_argument('-d', '--dist', choices = ['chisquare'], default='chisquare', help='distribution test to run')
    args = pp.parseArgs()
    if not any(args.labels):
        args.labels = [args.column_name + '_disttest']
    args = pp.getArgs(args)

    run_grouping(args.infile, DistGroup, args.group, args.ordered)
<commit_msg>adding padding to disttest script<commit_after>#!/usr/bin/env python

import os
import sys
import argparse
from collections import defaultdict
from decimal import Decimal
from toollib.files import findNumber,ParameterParser
from toollib.group import Group,run_grouping
from math import sqrt

class DistGroup(Group):
    def __init__(self, tup):
        super(DistGroup, self).__init__(tup)
        self.vals = []

    def add(self, chunks):
        val = float(findNumber(chunks[args.column]))
        self.vals.append(val)

    def done(self):
        import numpy as np
        vals = np.array(self.vals)
        from scipy.stats import chisquare
        if args.pad is not None and args.pad > len(vals):
            vals = np.append(vals, [0.0] * (args.pad - len(vals)))
        args.outfile.write(self.tup + list(chisquare(vals)))

if __name__ == "__main__":
    pp = ParameterParser('Entropy of a column', columns = 1, append = False, labels = [None])
    pp.parser.add_argument('-d', '--dist', choices = ['chisquare'], default='chisquare', help='distribution test to run')
    pp.parser.add_argument('-p', '--pad', type=int, default=None, help='pad to number of potential values')
    args = pp.parseArgs()
    if not any(args.labels):
        args.labels = [args.column_name + '_disttest']
    args = pp.getArgs(args)

    run_grouping(args.infile, DistGroup, args.group, args.ordered)
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Azure Linux extension
# Distribution-specific actions
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
# documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
# Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import exceptions
import time
import subprocess
import re
from Utils.WAAgentUtil import waagent

omi_universal_install_cmd = 'bash scx-1.6.2-337.universal.x64.sh --upgrade'

class CommonActions:
    def __init__(self, logger):
        self.logger = logger
        pass

    def log_run_get_output(self, cmd):
        """
        Execute a command in a subshell
        :param str cmd: The command to be executed
        :return (int, str): A tuple of (subshell exit code, contents of stdout)
        """
        self.logger("RunCmd " + cmd)
        error, msg = waagent.RunGetOutput(cmd, chk_err=True)
        self.logger("Return " + str(error) + ":" + msg)
        return int(error), str(msg)

    def log_run_ignore_output(self, cmd):
        """
        Execute a command in a subshell
        :param str cmd: The command to be executed
        :return int: The subshell exit code
        """
        self.logger("RunCmd " + cmd)
        error, msg = waagent.RunGetOutput(cmd, chk_err=True)
        self.logger("Return " + str(error) + ":" + msg)
        return int(error)

    def log_run_with_timeout(self, cmd, timeout=3600):
        """
        Execute a command in a subshell, killing the subshell if it runs too long
        :param str cmd: The command to be executed
        :param int timeout: The maximum elapsed time, in seconds, to wait for the subshell to return; default 360
        :return (int, str): (1, "Process timeout\n") if timeout, else (subshell exit code, contents of stdout)
        """
        self.logger("Run with timeout: " + cmd)
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True,
                                   executable='/bin/bash')
        time.sleep(1)
        while process.poll() is None and timeout > 0:
            time.sleep(1)
            timeout -= 1
        if process.poll() is None:
            self.logger("Timeout while running:" + cmd)
            process.kill()
            return 1, "Process timeout\n"
        output, error = process.communicate()
        self.logger("Return " + str(error))
        return int(process.returncode), output

    def log_run_multiple_cmds(self, cmds, with_timeout, timeout=360):
        """
        Execute multiple commands in subshells, with optional timeout protection
        :param Iterable[str] cmds: An iterable of commands to be executed
        :param bool with_timeout: True if commands should be run with timeout
        :param int timeout: The timeout, in seconds; default 360. Ignored if with_timeout is False.
        :return (int, str): A tuple of (sum of status codes, concatenated stdout from commands)
        """
        errors = 0
        output = []
        for cmd in cmds:
            if with_timeout:
                err, msg = self.log_run_with_timeout(cmd, timeout)
            else:
                err, msg = self.log_run_get_output(cmd)
            errors += err
            output.append(msg)
        return errors, ''.join(output)

    def extract_om_path_and_version(self, results):
        """
        Get information about rsyslogd
        :param str results: Package information about omprog.so or version
        :return (str, str): (Path where rsyslogd output modules are installed, major version of rsyslogd)
        """
        match = re.search(r"(.+)omprog\.so", results)
        if not match:
            return None, ''
        path = match.group(1)
        match = re.search(r"Version\s*:\s*(\d+)\D", results)
        if not match:
            self.logger("rsyslog is present but version could not be determined")
            return None, ''
        version = match.group(1)
        return path, version

    def install_omi(self):
        """
        Install OMI
        :return (int, str): (status code, output of install command).
        """
        return self.log_run_ignore_output(omi_universal_install_cmd)

    def install_extra_packages(self, packages, with_timeout=False):
        """
        Ensure an arbitrary set of packages is installed
        :param list[str] packages: Iterable of package names
        :param bool with_timeout: true if package installations should be aborted if they take too long
        :return int:
        """
        return 0, ''

    def install_required_packages(self):
        """
        Install packages required by this distro to meet the common bar required of all distros
        :return (int, str): (status, concatenated stdout from all package installs)
        """
        return 0, "no additional packages were needed"

    def is_package_handler(self, package_manager):
        """
        Checks if the distro's package manager matches the specified tool.
        :param str package_manager: The tool to be checked against the distro's native package manager
        :return bool: True if the distro's native package manager is package_manager
        """
        return False

    def restart_rsyslog(self):
        """
        Restart rsyslogd.
        :return int: status of operation
        """
        return self.log_run_ignore_output("service rsyslog restart")

    def get_rsyslog_info(self):
        """
        Get rsyslogd configuration information
        :return (str, str): (path to output modules, major version)
        """
        return None, ''

    def prepare_for_mdsd_install(self):
        return 0, ''

    def extend_environment(self, env):
        """
        Add required environment variables to process environment
        :param dict[str, str] env: Process environment
        """
        pass

    def use_systemd(self):
        """
        Determine if the distro uses systemd as its system management tool.
        :return bool: True if the distro uses systemd as its system management tool.
        """
        return False

class DebianActions(CommonActions):
    def __init__(self, logger):
        CommonActions.__init__(self, logger)

    def is_package_handler(self, package_manager):
        return package_manager == "dpkg"

    def install_extra_packages(self, packages, with_timeout=False):
        cmd = 'dpkg-query -l PACKAGE |grep ^ii; if [ ! $? == 0 ]; then apt-get update; apt-get install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def get_rsyslog_info(self):
        cmd = r'(dpkg-query -s rsyslog;dpkg-query -L rsyslog) |grep "Version\|omprog\.so"'
        error, results = self.log_run_get_output(cmd)
        return self.extract_om_path_and_version(results)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/usr/lib/ssl/certs", "SSL_CERT_FILE ": "/usr/lib/ssl/cert.pem"})

class Ubuntu1510OrHigherActions(DebianActions):
    def __init__(self, logger):
        DebianActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        count = len(packages)
        if count == 0:
            return 0, ''
        package_list = str.join(' ', packages)
        cmd = '[ $(dpkg -l PACKAGES |grep ^ii |wc -l) -eq \'COUNT\' ] || apt-get install -y PACKAGES'
        cmd = cmd.replace('PACKAGES', package_list).replace('COUNT', str(count))
        if with_timeout:
            return self.log_run_with_timeout(cmd)
        else:
            return self.log_run_get_output(cmd)

    def use_systemd(self):
        return True

class RedhatActions(CommonActions):
    def __init__(self, logger):
        CommonActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -q PACKAGE; if [ ! $? == 0 ]; then yum install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        # policycoreutils-python missing on Oracle Linux (still needed to manipulate SELinux policy).
        # tar is really missing on Oracle Linux 7!
        return self.install_extra_packages(('policycoreutils-python', 'tar'), True)

    def is_package_handler(self, package_manager):
        return package_manager == "rpm"

    def get_rsyslog_info(self):
        cmd = r'(rpm -qi rsyslog;rpm -ql rsyslog)|grep "Version\|omprog\.so"'
        error, results = self.log_run_get_output(cmd)
        return self.extract_om_path_and_version(results)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/etc/pki/tls/certs", "SSL_CERT_FILE": "/etc/pki/tls/cert.pem"})

class Suse11Actions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)
        self.certs_file = "/etc/ssl/certs/mdsd-ca-certs.pem"

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE;  if [ ! $? == 0 ]; then zypper --non-interactive install PACKAGE;fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        return self.install_extra_packages(('rsyslog',), True)

    # For SUSE11, we need to create a CA certs file for our statically linked OpenSSL 1.0 libs
    def prepare_for_mdsd_install(self):
        commands = (
            r'cp /dev/null {0}'.format(self.certs_file),
            r'chown 0:0 {0}'.format(self.certs_file),
            r'chmod 0644 {0}'.format(self.certs_file),
            r"cat /etc/ssl/certs/????????.[0-9a-f] | sed '/^#/d' >> {0}".format(self.certs_file)
        )
        return self.log_run_multiple_cmds(commands, False)

    def extend_environment(self, env):
        env.update({"SSL_CERT_FILE": self.certs_file})

    def restart_rsyslog(self):
        return self.log_run_ignore_output("""\
if [ ! -f /etc/sysconfig/syslog.org_lad ]; then cp /etc/sysconfig/syslog /etc/sysconfig/syslog.org_lad; fi;
sed -i 's/SYSLOG_DAEMON="syslog-ng"/SYSLOG_DAEMON="rsyslogd"/g' /etc/sysconfig/syslog;
service syslog restart""")

class Suse12Actions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)
        self.certs_file = "/etc/ssl/certs/mdsd-ca-certs.pem"

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE; if [ ! $? == 0 ]; then zypper --non-interactive install PACKAGE;fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        return self.install_extra_packages(('libgthread-2_0-0', 'ca-certificates-mozilla', 'rsyslog'), True)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/var/lib/ca-certificates/openssl", "SSL_CERT_FILE": "/etc/ssl/cert.pem"})

class CentosActions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE; if [ ! $? == 0 ]; then yum install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        # policycoreutils-python missing on CentOS (still needed to manipulate SELinux policy)
        return self.install_extra_packages(('policycoreutils-python',), True)

DistroMap = {
    'debian': DebianActions, 'Kali': DebianActions,
    'Ubuntu': DebianActions, 'Ubuntu:15.10': Ubuntu1510OrHigherActions,
    'Ubuntu:16.04': Ubuntu1510OrHigherActions, 'Ubuntu:16.10': Ubuntu1510OrHigherActions,
    'redhat': RedhatActions, 'centos': CentosActions, 'oracle': RedhatActions,
    'SuSE:11': Suse11Actions, 'SuSE:12': Suse12Actions, 'SuSE': Suse12Actions
}

def get_distro_actions(name, version, logger):
    name_and_version = name + ":" + version
    if name_and_version in DistroMap:
        return DistroMap[name_and_version](logger)
    elif name in DistroMap:
        return DistroMap[name](logger)
    raise exceptions.LookupError('{} is not a supported distro'.format(name_and_version))
<commit_msg>Add optional should_log to log_run_get_output to match the wrapped method.<commit_after>#!/usr/bin/env python
#
# Azure Linux extension
# Distribution-specific actions
#
# Linux Azure Diagnostic Extension (Current version is specified in manifest.xml)
# Copyright (c) Microsoft Corporation
# All rights reserved.
# MIT License
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
# documentation files (the ""Software""), to deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
# persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
# Software.
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

import exceptions
import time
import subprocess
import re
from Utils.WAAgentUtil import waagent

omi_universal_install_cmd = 'bash scx-1.6.2-337.universal.x64.sh --upgrade'

class CommonActions:
    def __init__(self, logger):
        self.logger = logger
        pass

    def log_run_get_output(self, cmd, should_log=True):
        """
        Execute a command in a subshell
        :param str cmd: The command to be executed
        :param bool should_log: If true, log command execution
        :return (int, str): A tuple of (subshell exit code, contents of stdout)
        """
        if should_log:
            self.logger("RunCmd " + cmd)
        error, msg = waagent.RunGetOutput(cmd, chk_err=should_log)
        if should_log:
            self.logger("Return " + str(error) + ":" + msg)
        return int(error), str(msg)

    def log_run_ignore_output(self, cmd):
        """
        Execute a command in a subshell
        :param str cmd: The command to be executed
        :return int: The subshell exit code
        """
        self.logger("RunCmd " + cmd)
        error, msg = waagent.RunGetOutput(cmd, chk_err=True)
        self.logger("Return " + str(error) + ":" + msg)
        return int(error)

    def log_run_with_timeout(self, cmd, timeout=3600):
        """
        Execute a command in a subshell, killing the subshell if it runs too long
        :param str cmd: The command to be executed
        :param int timeout: The maximum elapsed time, in seconds, to wait for the subshell to return; default 360
        :return (int, str): (1, "Process timeout\n") if timeout, else (subshell exit code, contents of stdout)
        """
        self.logger("Run with timeout: " + cmd)
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True,
                                   executable='/bin/bash')
        time.sleep(1)
        while process.poll() is None and timeout > 0:
            time.sleep(1)
            timeout -= 1
        if process.poll() is None:
            self.logger("Timeout while running:" + cmd)
            process.kill()
            return 1, "Process timeout\n"
        output, error = process.communicate()
        self.logger("Return " + str(error))
        return int(process.returncode), output

    def log_run_multiple_cmds(self, cmds, with_timeout, timeout=360):
        """
        Execute multiple commands in subshells, with optional timeout protection
        :param Iterable[str] cmds: An iterable of commands to be executed
        :param bool with_timeout: True if commands should be run with timeout
        :param int timeout: The timeout, in seconds; default 360. Ignored if with_timeout is False.
        :return (int, str): A tuple of (sum of status codes, concatenated stdout from commands)
        """
        errors = 0
        output = []
        for cmd in cmds:
            if with_timeout:
                err, msg = self.log_run_with_timeout(cmd, timeout)
            else:
                err, msg = self.log_run_get_output(cmd)
            errors += err
            output.append(msg)
        return errors, ''.join(output)

    def extract_om_path_and_version(self, results):
        """
        Get information about rsyslogd
        :param str results: Package information about omprog.so or version
        :return (str, str): (Path where rsyslogd output modules are installed, major version of rsyslogd)
        """
        match = re.search(r"(.+)omprog\.so", results)
        if not match:
            return None, ''
        path = match.group(1)
        match = re.search(r"Version\s*:\s*(\d+)\D", results)
        if not match:
            self.logger("rsyslog is present but version could not be determined")
            return None, ''
        version = match.group(1)
        return path, version

    def install_omi(self):
        """
        Install OMI
        :return (int, str): (status code, output of install command).
        """
        return self.log_run_ignore_output(omi_universal_install_cmd)

    def install_extra_packages(self, packages, with_timeout=False):
        """
        Ensure an arbitrary set of packages is installed
        :param list[str] packages: Iterable of package names
        :param bool with_timeout: true if package installations should be aborted if they take too long
        :return int:
        """
        return 0, ''

    def install_required_packages(self):
        """
        Install packages required by this distro to meet the common bar required of all distros
        :return (int, str): (status, concatenated stdout from all package installs)
        """
        return 0, "no additional packages were needed"

    def is_package_handler(self, package_manager):
        """
        Checks if the distro's package manager matches the specified tool.
        :param str package_manager: The tool to be checked against the distro's native package manager
        :return bool: True if the distro's native package manager is package_manager
        """
        return False

    def restart_rsyslog(self):
        """
        Restart rsyslogd.
        :return int: status of operation
        """
        return self.log_run_ignore_output("service rsyslog restart")

    def get_rsyslog_info(self):
        """
        Get rsyslogd configuration information
        :return (str, str): (path to output modules, major version)
        """
        return None, ''

    def prepare_for_mdsd_install(self):
        return 0, ''

    def extend_environment(self, env):
        """
        Add required environment variables to process environment
        :param dict[str, str] env: Process environment
        """
        pass

    def use_systemd(self):
        """
        Determine if the distro uses systemd as its system management tool.
        :return bool: True if the distro uses systemd as its system management tool.
        """
        return False

class DebianActions(CommonActions):
    def __init__(self, logger):
        CommonActions.__init__(self, logger)

    def is_package_handler(self, package_manager):
        return package_manager == "dpkg"

    def install_extra_packages(self, packages, with_timeout=False):
        cmd = 'dpkg-query -l PACKAGE |grep ^ii; if [ ! $? == 0 ]; then apt-get update; apt-get install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def get_rsyslog_info(self):
        cmd = r'(dpkg-query -s rsyslog;dpkg-query -L rsyslog) |grep "Version\|omprog\.so"'
        error, results = self.log_run_get_output(cmd)
        return self.extract_om_path_and_version(results)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/usr/lib/ssl/certs", "SSL_CERT_FILE ": "/usr/lib/ssl/cert.pem"})

class Ubuntu1510OrHigherActions(DebianActions):
    def __init__(self, logger):
        DebianActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        count = len(packages)
        if count == 0:
            return 0, ''
        package_list = str.join(' ', packages)
        cmd = '[ $(dpkg -l PACKAGES |grep ^ii |wc -l) -eq \'COUNT\' ] || apt-get install -y PACKAGES'
        cmd = cmd.replace('PACKAGES', package_list).replace('COUNT', str(count))
        if with_timeout:
            return self.log_run_with_timeout(cmd)
        else:
            return self.log_run_get_output(cmd)

    def use_systemd(self):
        return True

class RedhatActions(CommonActions):
    def __init__(self, logger):
        CommonActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -q PACKAGE; if [ ! $? == 0 ]; then yum install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        # policycoreutils-python missing on Oracle Linux (still needed to manipulate SELinux policy).
        # tar is really missing on Oracle Linux 7!
        return self.install_extra_packages(('policycoreutils-python', 'tar'), True)

    def is_package_handler(self, package_manager):
        return package_manager == "rpm"

    def get_rsyslog_info(self):
        cmd = r'(rpm -qi rsyslog;rpm -ql rsyslog)|grep "Version\|omprog\.so"'
        error, results = self.log_run_get_output(cmd)
        return self.extract_om_path_and_version(results)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/etc/pki/tls/certs", "SSL_CERT_FILE": "/etc/pki/tls/cert.pem"})

class Suse11Actions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)
        self.certs_file = "/etc/ssl/certs/mdsd-ca-certs.pem"

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE;  if [ ! $? == 0 ]; then zypper --non-interactive install PACKAGE;fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        return self.install_extra_packages(('rsyslog',), True)

    # For SUSE11, we need to create a CA certs file for our statically linked OpenSSL 1.0 libs
    def prepare_for_mdsd_install(self):
        commands = (
            r'cp /dev/null {0}'.format(self.certs_file),
            r'chown 0:0 {0}'.format(self.certs_file),
            r'chmod 0644 {0}'.format(self.certs_file),
            r"cat /etc/ssl/certs/????????.[0-9a-f] | sed '/^#/d' >> {0}".format(self.certs_file)
        )
        return self.log_run_multiple_cmds(commands, False)

    def extend_environment(self, env):
        env.update({"SSL_CERT_FILE": self.certs_file})

    def restart_rsyslog(self):
        return self.log_run_ignore_output("""\
if [ ! -f /etc/sysconfig/syslog.org_lad ]; then cp /etc/sysconfig/syslog /etc/sysconfig/syslog.org_lad; fi;
sed -i 's/SYSLOG_DAEMON="syslog-ng"/SYSLOG_DAEMON="rsyslogd"/g' /etc/sysconfig/syslog;
service syslog restart""")

class Suse12Actions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)
        self.certs_file = "/etc/ssl/certs/mdsd-ca-certs.pem"

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE; if [ ! $? == 0 ]; then zypper --non-interactive install PACKAGE;fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        return self.install_extra_packages(('libgthread-2_0-0', 'ca-certificates-mozilla', 'rsyslog'), True)

    def extend_environment(self, env):
        env.update({"SSL_CERT_DIR": "/var/lib/ca-certificates/openssl", "SSL_CERT_FILE": "/etc/ssl/cert.pem"})

class CentosActions(RedhatActions):
    def __init__(self, logger):
        RedhatActions.__init__(self, logger)

    def install_extra_packages(self, packages, with_timeout=False):
        install_cmd = 'rpm -qi PACKAGE; if [ ! $? == 0 ]; then yum install -y PACKAGE; fi'
        return self.log_run_multiple_cmds([install_cmd.replace("PACKAGE", p) for p in packages], with_timeout)

    def install_required_packages(self):
        # policycoreutils-python missing on CentOS (still needed to manipulate SELinux policy)
        return self.install_extra_packages(('policycoreutils-python',), True)

DistroMap = {
    'debian': DebianActions, 'Kali': DebianActions,
    'Ubuntu': DebianActions, 'Ubuntu:15.10': Ubuntu1510OrHigherActions,
    'Ubuntu:16.04': Ubuntu1510OrHigherActions, 'Ubuntu:16.10': Ubuntu1510OrHigherActions,
    'redhat': RedhatActions, 'centos': CentosActions, 'oracle': RedhatActions,
    'SuSE:11': Suse11Actions, 'SuSE:12': Suse12Actions, 'SuSE': Suse12Actions
}

def get_distro_actions(name, version, logger):
    name_and_version = name + ":" + version
    if name_and_version in DistroMap:
        return DistroMap[name_and_version](logger)
    elif name in DistroMap:
        return DistroMap[name](logger)
    raise exceptions.LookupError('{} is not a supported distro'.format(name_and_version))
<|endoftext|>


<commit_before><commit_msg>Initialize Q8-Longest Repetition<commit_after><|endoftext|>


<commit_before>from copy import deepcopy as copy

from odict import OrderedDict

class Model(object):
  data = OrderedDict()
  functions = OrderedDict()
  key = ''
  name = ''
  parent = None
  def __init__(self, name, **kwargs):
    self.data = OrderedDict()
    self.functions = OrderedDict()
    self.name = name
    if 'parent' in kwargs:
      self.parent =  kwargs['parent']
      self.data = copy(parent.data)
      self.functions = copy(parent.functions)
      self.key = parent.key
    if 'data' in kwargs:
      data = kwargs['data']
      for key, value in data:
        if key in self.data:
          raise ValueError('Duplicate datum %s in %s' % (key, name))
        self.data[key] = value
    if 'functions' in kwargs:
      functions = kwargs['data']
      for key, value in functions:
        if key in self.functions:
          raise ValueError('Duplicate function %s in %s' % (key, name))
        self.functions['key'] = value
    if 'key' in kwargs:
      self.key = kwargs['key']

"""
TODO: Write these.
class Message(object):
  pass

class Function(object):
  arguments = None
  result = None

  def __init__(self, arguments, result):
    self.arguments = arguments
    self.result = result
"""<commit_msg>Fixed a dumb mistake.<commit_after>from copy import deepcopy as copy

from odict import OrderedDict

class Model(object):
  data = OrderedDict()
  functions = OrderedDict()
  key = ''
  name = ''
  parent = None
  def __init__(self, name, **kwargs):
    self.data = OrderedDict()
    self.functions = OrderedDict()
    self.name = name
    if 'parent' in kwargs:
      self.parent =  kwargs['parent']
      self.data = copy(parent.data)
      self.functions = copy(parent.functions)
      self.key = parent.key
    if 'data' in kwargs:
      data = kwargs['data']
      for key, value in data:
        if key in self.data:
          raise ValueError('Duplicate datum %s in %s' % (key, name))
        self.data[key] = value
    if 'functions' in kwargs:
      functions = kwargs['functions']
      for key, value in functions:
        if key in self.functions:
          raise ValueError('Duplicate function %s in %s' % (key, name))
        self.functions['key'] = value
    if 'key' in kwargs:
      self.key = kwargs['key']

"""
TODO: Write these.
class Message(object):
  pass

class Function(object):
  arguments = None
  result = None

  def __init__(self, arguments, result):
    self.arguments = arguments
    self.result = result
"""<|endoftext|>


<commit_before>#!/usr/bin/env python3

import collections # OrderedDict
import sys         # exit

from pass_utils import UNOPS, BINOPS

def lift_inputs_and_assigns(exp):
  """ Extracts input variables and assignments from an expression """

  # Constants
  ATOMS     = {"ConstantInterval", "PointInterval", "Float", "Integer", "Input",
               "Variable"}
  ONE_ITEM  = UNOPS.union({"Return"})

  # Function local variables
  assigns      = collections.OrderedDict() # name -> expression
  used_assigns = set()                     # assignments seen in the main exp
  inputs       = collections.OrderedDict() # name -> input range
  used_inputs  = set()                     # inputs seen in the main exp
  implicit_input_count = 0                 # number of implicit inputs

  # increase stack size for saftey
  old_limit = sys.getrecursionlimit()
  sys.setrecursionlimit(4000)

  def _lift_inputs_and_assigns(exp):
    nonlocal implicit_input_count
    tag = exp[0]

    # A leading tuple must be an assign
    if type(tag) is tuple:
      assignment = exp[0]
      assert(assignment[0] == "Assign")
      name = assignment[1]
      assert(name[0] == "Name")
      val  = assignment[2]
      # Explicit inputs
      if val[0] == "InputInterval":
        assert(name[1] not in inputs)
        inputs[name[1]] = val

      # Assignment to an expression
      else:
        assert(name[1] not in assigns)
        assigns[name[1]] = _lift_inputs_and_assigns(val)

      # Work on the rest of the expression
      return  _lift_inputs_and_assigns(exp[1])

    # Recur on arguments
    if tag in BINOPS:
      l = _lift_inputs_and_assigns(exp[1])
      r = _lift_inputs_and_assigns(exp[2])
      return (exp[0], l, r)

    # Recur on argument
    if tag in ONE_ITEM:
      arg = _lift_inputs_and_assigns(exp[1])
      return (exp[0], arg)

    # Base cases
    if tag in ATOMS:
      return exp

    # If we get here a bare range was given in the middle of an expression
    # This is turned into an implicit input
    if tag == "InputInterval":
      return ("ConstantInterval", exp[1], exp[2])
      # Temporarily disabled
      interval     = exp[:]
      name         = "$_implicit_Input_{}".format(implicit_input_count)
      new_exp      = ("Input", name)
      assert(name not in inputs)
      inputs[name] = interval
      implicit_input_count += 1
      used_inputs.add(name)
      return new_exp

    # Change generic 'Name' to specific tags
    if tag == "Name":

      # Input
      if exp[1] in inputs:
        used_inputs.add(exp[1])
        assert(exp[1] not in assigns)
        return ("Input", exp[1])

      # Assignment
      elif exp[1] in assigns:
        used_assigns.add(exp[1])
        return ("Variable", exp[1])

      # Not a valid 'Name'
      print("Use of undeclared name: {}".format(exp[1]))
      sys.exit(-1)

    # exp is not in the laguage
    print("lift_inputs_and_assigns error unknown: '{}'".format(exp))
    sys.exit(-1)

  # Call internally scoped function
  new_exp = _lift_inputs_and_assigns(exp)

  # Remove dead inputs
  dead_inputs = set(inputs).difference(used_inputs)
  for k in dead_inputs:
    del inputs[k]

  # Remove dead assigns
  dead_assigns = set(assigns).difference(used_assigns)
  for k in dead_assigns:
    del assigns[k]

  # reset stack size
  sys.setrecursionlimit(old_limit)
  return new_exp, inputs, assigns

def runmain():
  from pass_utils import get_runmain_input
  from lexed_to_parsed import parse_function
  from pass_utils import print_exp, print_inputs, print_assigns

  data = get_runmain_input()
  exp  = parse_function(data)
  exp, inputs, assigns = lift_inputs_and_assigns(exp)

  print_inputs(inputs)
  print()
  print_assigns(assigns)
  print()
  print_exp(exp)

if __name__ == "__main__":
  try:
    runmain()
  except KeyboardInterrupt:
    print("\nGoodbye")
<commit_msg>keep large stack for now<commit_after>#!/usr/bin/env python3

import collections # OrderedDict
import sys         # exit

from pass_utils import UNOPS, BINOPS

def lift_inputs_and_assigns(exp):
  """ Extracts input variables and assignments from an expression """

  # Constants
  ATOMS     = {"ConstantInterval", "PointInterval", "Float", "Integer", "Input",
               "Variable"}
  ONE_ITEM  = UNOPS.union({"Return"})

  # Function local variables
  assigns      = collections.OrderedDict() # name -> expression
  used_assigns = set()                     # assignments seen in the main exp
  inputs       = collections.OrderedDict() # name -> input range
  used_inputs  = set()                     # inputs seen in the main exp
  implicit_input_count = 0                 # number of implicit inputs

  # increase stack size for saftey
  old_limit = sys.getrecursionlimit()
  sys.setrecursionlimit(4000)

  def _lift_inputs_and_assigns(exp):
    nonlocal implicit_input_count
    tag = exp[0]

    # A leading tuple must be an assign
    if type(tag) is tuple:
      assignment = exp[0]
      assert(assignment[0] == "Assign")
      name = assignment[1]
      assert(name[0] == "Name")
      val  = assignment[2]
      # Explicit inputs
      if val[0] == "InputInterval":
        assert(name[1] not in inputs)
        inputs[name[1]] = val

      # Assignment to an expression
      else:
        assert(name[1] not in assigns)
        assigns[name[1]] = _lift_inputs_and_assigns(val)

      # Work on the rest of the expression
      return  _lift_inputs_and_assigns(exp[1])

    # Recur on arguments
    if tag in BINOPS:
      l = _lift_inputs_and_assigns(exp[1])
      r = _lift_inputs_and_assigns(exp[2])
      return (exp[0], l, r)

    # Recur on argument
    if tag in ONE_ITEM:
      arg = _lift_inputs_and_assigns(exp[1])
      return (exp[0], arg)

    # Base cases
    if tag in ATOMS:
      return exp

    # If we get here a bare range was given in the middle of an expression
    # This is turned into an implicit input
    if tag == "InputInterval":
      return ("ConstantInterval", exp[1], exp[2])
      # Temporarily disabled
      interval     = exp[:]
      name         = "$_implicit_Input_{}".format(implicit_input_count)
      new_exp      = ("Input", name)
      assert(name not in inputs)
      inputs[name] = interval
      implicit_input_count += 1
      used_inputs.add(name)
      return new_exp

    # Change generic 'Name' to specific tags
    if tag == "Name":

      # Input
      if exp[1] in inputs:
        used_inputs.add(exp[1])
        assert(exp[1] not in assigns)
        return ("Input", exp[1])

      # Assignment
      elif exp[1] in assigns:
        used_assigns.add(exp[1])
        return ("Variable", exp[1])

      # Not a valid 'Name'
      print("Use of undeclared name: {}".format(exp[1]))
      sys.exit(-1)

    # exp is not in the laguage
    print("lift_inputs_and_assigns error unknown: '{}'".format(exp))
    sys.exit(-1)

  # Call internally scoped function
  new_exp = _lift_inputs_and_assigns(exp)

  # Remove dead inputs
  dead_inputs = set(inputs).difference(used_inputs)
  for k in dead_inputs:
    del inputs[k]

  # Remove dead assigns
  dead_assigns = set(assigns).difference(used_assigns)
  for k in dead_assigns:
    del assigns[k]

  # reset stack size
  #sys.setrecursionlimit(old_limit)
  return new_exp, inputs, assigns

def runmain():
  from pass_utils import get_runmain_input
  from lexed_to_parsed import parse_function
  from pass_utils import print_exp, print_inputs, print_assigns

  data = get_runmain_input()
  exp  = parse_function(data)
  exp, inputs, assigns = lift_inputs_and_assigns(exp)

  print_inputs(inputs)
  print()
  print_assigns(assigns)
  print()
  print_exp(exp)

if __name__ == "__main__":
  try:
    runmain()
  except KeyboardInterrupt:
    print("\nGoodbye")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
*********
trunc.rnc
*********

This module defines objects for interfacing with the Russian National Corpus.
"""

from __future__ import absolute_import, print_function

import re

class RNCSource(object):
    """One source in RNC search results.

    :param source_name: the full name of the source, as it appears inside the ``<span>`` element.
    :type source_name: ``str``, ``unicode``

    If a source's name contains a date range, its ``date_middle`` is the
    average value of the two dates in the range:

    >>> a = RNCSource(u"И. Э. Кио. Иллюзии без иллюзий (1995-1999)")
    >>> a.date_begin, a.date_middle, a.date_end
    (1995.0, 1997.0, 1999,0)

    If a source's name contains only one date, all three ``date`` attributes
    are identical:

    >>> b = RNCSource(u"В. Г. Распутин. Новая профессия (1998)")
    >>> b.date_begin, b.date_middle, b.date_end
    (1998.0, 1998.0, 1998.0)

    If a source contains multiple individual dates (but not a range), the
    earliest date is chosen, as this typically occurs when a source name
    describes a reprinted work, and both the original publication year
    and the reprinted publication year are provided:

    >>> c = RNCSource(u"Фридрих Горенштейн. Куча (1982) // «Октябрь», 1996")
    >>> c.date_begin, c.date_middle, c.date_end
    (1982.0, 1982.0, 1982.0)

    If a source contains no dates at all, the default date values of ``0``
    are used. This is uncommon in the main (modern) corpus, but happens
    frequently in the historical corpora:

    >>> d = RNCSource(u"Житие Андрея Юродивого")
    >>> d.date_begin, d.date_middle, d.date_end
    (0.0, 0.0, 0.0)

    """

    def __init__(self, source_name):
        """Initialize the RNCSource object."""
        self.source_name = source_name
        # strip parentheses and their contents
        self.bare_name = re.sub(r'\([^)]*\)', '', self.source_name)
        # assign some default values to dates
        self.date_begin = 0.0
        self.date_middle = 0.0
        self.date_end = 0.0
        # parse the source string to find the dates
        self.parse_dates()

    def parse_dates(self):
        """Parse the source name and return the dates of its writing."""
        date_seq = re.findall(r'\d{4}-\d{4}', self.source_name)
        if date_seq:
            self.date_begin = float(date_seq[0].split('-')[0])
            self.date_end = float(date_seq[0].split('-')[1])
            self.date_middle = (self.date_begin + self.date_end) / 2.0
        else:
            dates = re.findall(r'\d{4}', self.source_name)
            if dates:
                self.date_begin = float(min(dates))
                self.date_middle = float(min(dates))
                self.date_end = float(min(dates))
            else:
                pass
<commit_msg>update docstring for RNCSource class<commit_after># -*- coding: utf-8 -*-

"""
*********
trunc.rnc
*********

This module defines objects for interfacing with the Russian National Corpus.
"""

from __future__ import absolute_import, print_function

import re

class RNCSource(object):
    """One source in RNC search results.

    :param source_name: the full name of the source, as it appears inside the ``<span>`` element.
    :type source_name: ``str``, ``unicode``

    """

    def __init__(self, source_name):
        """Initialize the RNCSource object."""
        self.source_name = source_name
        # strip parentheses and their contents
        self.bare_name = re.sub(r'\([^)]*\)', '', self.source_name)
        # assign some default values to dates
        self.date_begin = 0.0
        self.date_middle = 0.0
        self.date_end = 0.0
        # parse the source string to find the dates
        self.parse_dates()

    def parse_dates(self):
        """Parse the source name and return the dates of its writing.

        If a source's name contains a date range, its ``date_middle`` is the
        average value of the two dates in the range:

        >>> a = RNCSource(u"И. Э. Кио. Иллюзии без иллюзий (1995-1999)")
        >>> a.date_begin, a.date_middle, a.date_end
        (1995.0, 1997.0, 1999,0)

        If a source's name contains only one date, all three ``date``
        attributes are identical:

        >>> b = RNCSource(u"В. Г. Распутин. Новая профессия (1998)")
        >>> b.date_begin, b.date_middle, b.date_end
        (1998.0, 1998.0, 1998.0)

        If a source contains multiple individual dates (but not a range), the
        earliest date is chosen, as this typically occurs when a source name
        describes a reprinted work, and both the original publication year
        and the reprinted publication year are provided:

        >>> c = RNCSource(u"Фридрих Горенштейн. Куча (1982) // «Октябрь», 1996")
        >>> c.date_begin, c.date_middle, c.date_end
        (1982.0, 1982.0, 1982.0)

        If a source contains no dates at all, the default date values of
        ``0.0`` are used. This is uncommon in the main (modern) corpus, but
        happens frequently in the historical corpora:

        >>> d = RNCSource(u"Житие Андрея Юродивого")
        >>> d.date_begin, d.date_middle, d.date_end
        (0.0, 0.0, 0.0)

        """
        date_seq = re.findall(r'\d{4}-\d{4}', self.source_name)
        if date_seq:
            self.date_begin = float(date_seq[0].split('-')[0])
            self.date_end = float(date_seq[0].split('-')[1])
            self.date_middle = (self.date_begin + self.date_end) / 2.0
        else:
            dates = re.findall(r'\d{4}', self.source_name)
            if dates:
                self.date_begin = float(min(dates))
                self.date_middle = float(min(dates))
                self.date_end = float(min(dates))
            else:
                pass
<|endoftext|>


<commit_before>"""
Tests for the worker process utility.
"""

from __future__ import division, print_function, with_statement

import uuid
import logging
import multiprocessing
from multiprocessing import Manager
from multiprocessing.process import Process

import nose
from nose.tools.trivial import eq_
from nose.tools.nontrivial import raises

from orges.core.args import ArgsCreator
from orges.invoker.util.worker_provider import Task, Result, Status, Worker, \
    WorkerProcess
from orges.tests.util.integer_functions import INTEGER_FUNCTIONS
from orges.invoker.util.determine_package import determine_package

def _no_nose_setup():
    manager = Manager()
    worker_process = WorkerProcess(worker_id=uuid.uuid4(),
                                   queue_results=manager.Queue(),
                                   queue_status=manager.Queue(),
                                   queue_tasks=manager.Queue())
    return worker_process

def test_instanciation():
    _no_nose_setup()

def test_superclasses():
    worker_process = _no_nose_setup()
    isinstance(worker_process, Process)
    isinstance(worker_process, Worker)

def test_properties_return_instanciation_values():
    worker_id = uuid.uuid4()
    queue_results = multiprocessing.Queue()
    queue_status = multiprocessing.Queue()
    queue_tasks = multiprocessing.Queue()
    worker_process = WorkerProcess(worker_id=worker_id,
                                   queue_results=queue_results,
                                   queue_status=queue_status,
                                   queue_tasks=queue_tasks)
    assert worker_process.queue_results is queue_results
    assert worker_process.queue_status is queue_status
    assert worker_process.queue_tasks is queue_tasks
    assert worker_process.worker_id is worker_id

def test_initialization_is_sane():
    worker_process = _no_nose_setup()
    assert not worker_process.current_task_id
    assert not worker_process.busy

def test_start_terminate():
    worker_process = _no_nose_setup()
    worker_process.start()
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_notask_terminate():
    worker_process = _no_nose_setup()
    worker_process.start()
    worker_process.queue_tasks.put(None)
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_task_terminate():
    worker_process = _no_nose_setup()
    worker_process.start()
    worker_process.queue_tasks.put(Task(task_id=uuid.uuid4,
                                        function=print,
                                        args=None,
                                        vargs=None,
                                        kwargs=None))
    worker_process.queue_tasks.put(None)
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_task_status_results_terminate():
    #multiprocessing.log_to_stderr()
    #logger = multiprocessing.get_logger()
    #logger.setLevel(logging.DEBUG)

    for function in INTEGER_FUNCTIONS:
        # log
        print(function)

        # send task to worker process
        worker_process = _no_nose_setup()
        worker_process.start()
        worker_process.queue_tasks.put(Task(task_id=uuid.uuid4,
                                            function=determine_package(function),
                                            args=ArgsCreator(function.param_spec).args(),
                                            vargs=None,
                                            kwargs=None))
        #worker_process.queue_tasks.close()

        # check results
        status = worker_process.queue_status.get()
        #worker_process.queue_status.close()
        assert status
        assert isinstance(status, Status)
        result = worker_process.queue_results.get()
        #worker_process.queue_results.close()
        assert result
        assert isinstance(result, Result)

        # tear down
        worker_process.terminate()
        worker_process.join()

        # check postcondition
        assert not worker_process.is_alive()
        assert not worker_process.current_task_id
        assert not worker_process.busy

if __name__ == '__main__':
    nose.runmodule()
<commit_msg>clean up, docs, pylintfy<commit_after>"""
Tests for the worker process utility.
"""

from __future__ import division, print_function, with_statement

import uuid
import multiprocessing
from multiprocessing import Manager
from multiprocessing.process import Process

import nose

from orges.core.args import ArgsCreator
from orges.invoker.util.worker_provider import Task, Result, Status, Worker, \
    WorkerProcess
from orges.tests.util.integer_functions import INTEGER_FUNCTIONS
from orges.invoker.util.determine_package import determine_package

def get_default_worker_process():
    """Returns a worker process with attached queues and set worker id."""
    manager = Manager()
    worker_process = WorkerProcess(worker_id=uuid.uuid4(),
                                   queue_results=manager.Queue(),
                                   queue_status=manager.Queue(),
                                   queue_tasks=manager.Queue())
    return worker_process

def test_instanciation():
    get_default_worker_process()

def test_superclasses():
    worker_process = get_default_worker_process()
    isinstance(worker_process, Process)
    isinstance(worker_process, Worker)

def test_properties_return_instanciation_values():
    worker_id = uuid.uuid4()
    queue_results = multiprocessing.Queue()
    queue_status = multiprocessing.Queue()
    queue_tasks = multiprocessing.Queue()
    worker_process = WorkerProcess(worker_id=worker_id,
                                   queue_results=queue_results,
                                   queue_status=queue_status,
                                   queue_tasks=queue_tasks)
    assert worker_process.queue_results is queue_results
    assert worker_process.queue_status is queue_status
    assert worker_process.queue_tasks is queue_tasks
    assert worker_process.worker_id is worker_id

def test_initialization_is_sane():
    worker_process = get_default_worker_process()
    assert not worker_process.current_task_id
    assert not worker_process.busy

def test_start_terminate():
    worker_process = get_default_worker_process()
    worker_process.start()
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_notask_terminate():
    worker_process = get_default_worker_process()
    worker_process.start()
    worker_process.queue_tasks.put(None)
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_task_terminate():
    worker_process = get_default_worker_process()
    worker_process.start()
    worker_process.queue_tasks.put(Task(task_id=uuid.uuid4,
                                        function=print,
                                        args=None,
                                        vargs=None,
                                        kwargs=None))
    worker_process.queue_tasks.put(None)
    worker_process.terminate()
    worker_process.join()
    assert not worker_process.is_alive()

def test_start_task_status_results_terminate():
    #multiprocessing.log_to_stderr()
    #logger = multiprocessing.get_logger()
    #logger.setLevel(logging.DEBUG)

    for function in INTEGER_FUNCTIONS:
        # log
        print(function)

        # send task to worker process
        worker_process = get_default_worker_process()
        worker_process.start()
        task = Task(task_id=uuid.uuid4,
                    function=determine_package(function),
                    args=ArgsCreator(function.param_spec).args(),
                    vargs=None,
                    kwargs=None)
        worker_process.queue_tasks.put(task)

        # check results
        status = worker_process.queue_status.get()
        assert status
        assert isinstance(status, Status)
        result = worker_process.queue_results.get()
        assert result
        assert isinstance(result, Result)

        # tear down
        worker_process.terminate()
        worker_process.join()

        # check postcondition
        assert not worker_process.is_alive()
        assert not worker_process.current_task_id
        assert not worker_process.busy

if __name__ == '__main__':
    nose.runmodule()
<|endoftext|>